ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2025-12-06 17:27:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix Linux/gcc compilation (#795)

Browse Source 
Fixes #803
This commit is contained in:
Alex Kremer
2023-08-02 13:44:03 +01:00
committed by GitHub
parent 98d0a963dc
commit 24f69acd9e
81 changed files with 1259 additions and 1282 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/backend/BackendInterface.cpp
View File

@@ -51,9 +51,8 @@ BackendInterface::writeLedgerObject(std::string&& key, std::uint32_t const seq,
}
std::optional<LedgerRange>
BackendInterface::hardFetchLedgerRangeNoThrow(boost::asio::yield_context& yield) const
BackendInterface::hardFetchLedgerRangeNoThrow(boost::asio::yield_context yield) const
{
gLog.trace() << "called";
while (true)
{
try
@@ -70,7 +69,6 @@ BackendInterface::hardFetchLedgerRangeNoThrow(boost::asio::yield_context& yield)
std::optional<LedgerRange>
BackendInterface::hardFetchLedgerRangeNoThrow() const
{
gLog.trace() << "called";
return retryOnTimeout([&]() { return hardFetchLedgerRange(); });
}
@@ -79,7 +77,7 @@ std::optional<Blob>
BackendInterface::fetchLedgerObject(
ripple::uint256 const& key,
std::uint32_t const sequence,
boost::asio::yield_context& yield) const
boost::asio::yield_context yield) const
{
auto obj = cache_.get(key, sequence);
if (obj)
@@ -103,7 +101,7 @@ std::vector<Blob>
BackendInterface::fetchLedgerObjects(
std::vector<ripple::uint256> const& keys,
std::uint32_t const sequence,
boost::asio::yield_context& yield) const
boost::asio::yield_context yield) const
{
std::vector<Blob> results;
results.resize(keys.size());
@@ -138,7 +136,7 @@ std::optional<ripple::uint256>
BackendInterface::fetchSuccessorKey(
ripple::uint256 key,
std::uint32_t const ledgerSequence,
boost::asio::yield_context& yield) const
boost::asio::yield_context yield) const
{
auto succ = cache_.getSuccessor(key, ledgerSequence);
if (succ)
@@ -152,7 +150,7 @@ std::optional<LedgerObject>
BackendInterface::fetchSuccessorObject(
ripple::uint256 key,
std::uint32_t const ledgerSequence,
boost::asio::yield_context& yield) const
boost::asio::yield_context yield) const
{
auto succ = fetchSuccessorKey(key, ledgerSequence, yield);
if (succ)
@@ -171,7 +169,7 @@ BackendInterface::fetchBookOffers(
ripple::uint256 const& book,
std::uint32_t const ledgerSequence,
std::uint32_t const limit,
boost::asio::yield_context& yield) const
boost::asio::yield_context yield) const
{
// TODO try to speed this up. This can take a few seconds. The goal is
// to get it down to a few hundred milliseconds.
@@ -248,7 +246,7 @@ BackendInterface::fetchLedgerPage(
std::uint32_t const ledgerSequence,
std::uint32_t const limit,
bool outOfOrder,
boost::asio::yield_context& yield) const
boost::asio::yield_context yield) const
{
LedgerPage page;
@@ -289,7 +287,7 @@ BackendInterface::fetchLedgerPage(
}
std::optional<ripple::Fees>
BackendInterface::fetchFees(std::uint32_t const seq, boost::asio::yield_context& yield) const
BackendInterface::fetchFees(std::uint32_t const seq, boost::asio::yield_context yield) const
{
ripple::Fees fees;

62
src/backend/BackendInterface.h
View File

@@ -94,28 +94,19 @@ auto
synchronous(F&& f)
{
boost::asio::io_context ctx;
boost::asio::strand<boost::asio::io_context::executor_type> strand(ctx.get_executor());
using R = typename boost::result_of<F(boost::asio::yield_context&)>::type;
using R = typename boost::result_of<F(boost::asio::yield_context)>::type;
if constexpr (!std::is_same<R, void>::value)
{
R res;
boost::asio::spawn(
strand, [&f, &res, _ = boost::asio::make_work_guard(strand)](boost::asio::yield_context yield) {
res = f(yield);
;
});
boost::asio::spawn(ctx, [&f, &res](boost::asio::yield_context yield) { res = f(yield); });
ctx.run();
return res;
}
else
{
boost::asio::spawn(strand, [&f, _ = boost::asio::make_work_guard(strand)](boost::asio::yield_context yield) {
f(yield);
;
});
boost::asio::spawn(ctx, [&f](boost::asio::yield_context yield) { f(yield); });
ctx.run();
}
}
@@ -183,15 +174,15 @@ public:
/*! @brief Fetches a specific ledger by sequence number. */
virtual std::optional<ripple::LedgerHeader>
fetchLedgerBySequence(std::uint32_t const sequence, boost::asio::yield_context& yield) const = 0;
fetchLedgerBySequence(std::uint32_t const sequence, boost::asio::yield_context yield) const = 0;
/*! @brief Fetches a specific ledger by hash. */
virtual std::optional<ripple::LedgerHeader>
fetchLedgerByHash(ripple::uint256 const& hash, boost::asio::yield_context& yield) const = 0;
fetchLedgerByHash(ripple::uint256 const& hash, boost::asio::yield_context yield) const = 0;
/*! @brief Fetches the latest ledger sequence. */
virtual std::optional<std::uint32_t>
fetchLatestLedgerSequence(boost::asio::yield_context& yield) const = 0;
fetchLatestLedgerSequence(boost::asio::yield_context yield) const = 0;
/*! @brief Fetches the current ledger range while locking that process */
std::optional<LedgerRange>
@@ -228,7 +219,7 @@ public:
* @return std::optional<ripple::Fees>
*/
std::optional<ripple::Fees>
fetchFees(std::uint32_t const seq, boost::asio::yield_context& yield) const;
fetchFees(std::uint32_t const seq, boost::asio::yield_context yield) const;
/*! @brief TRANSACTION METHODS */
/**
@@ -239,7 +230,7 @@ public:
* @return std::optional<TransactionAndMetadata>
*/
virtual std::optional<TransactionAndMetadata>
fetchTransaction(ripple::uint256 const& hash, boost::asio::yield_context& yield) const = 0;
fetchTransaction(ripple::uint256 const& hash, boost::asio::yield_context yield) const = 0;
/**
* @brief Fetches multiple transactions.
@@ -249,7 +240,7 @@ public:
* @return std::vector<TransactionAndMetadata>
*/
virtual std::vector<TransactionAndMetadata>
fetchTransactions(std::vector<ripple::uint256> const& hashes, boost::asio::yield_context& yield) const = 0;
fetchTransactions(std::vector<ripple::uint256> const& hashes, boost::asio::yield_context yield) const = 0;
/**
* @brief Fetches all transactions for a specific account
@@ -269,7 +260,7 @@ public:
std::uint32_t const limit,
bool forward,
std::optional<TransactionsCursor> const& cursor,
boost::asio::yield_context& yield) const = 0;
boost::asio::yield_context yield) const = 0;
/**
* @brief Fetches all transactions from a specific ledger.
@@ -280,7 +271,7 @@ public:
* @return std::vector<TransactionAndMetadata>
*/
virtual std::vector<TransactionAndMetadata>
fetchAllTransactionsInLedger(std::uint32_t const ledgerSequence, boost::asio::yield_context& yield) const = 0;
fetchAllTransactionsInLedger(std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const = 0;
/**
* @brief Fetches all transaction hashes from a specific ledger.
@@ -290,7 +281,7 @@ public:
* @return std::vector<ripple::uint256>
*/
virtual std::vector<ripple::uint256>
fetchAllTransactionHashesInLedger(std::uint32_t const ledgerSequence, boost::asio::yield_context& yield) const = 0;
fetchAllTransactionHashesInLedger(std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const = 0;
/*! @brief NFT methods */
/**
@@ -302,7 +293,7 @@ public:
* @return std::optional<NFT>
*/
virtual std::optional<NFT>
fetchNFT(ripple::uint256 const& tokenID, std::uint32_t const ledgerSequence, boost::asio::yield_context& yield)
fetchNFT(ripple::uint256 const& tokenID, std::uint32_t const ledgerSequence, boost::asio::yield_context yield)
const = 0;
/**
@@ -321,7 +312,7 @@ public:
std::uint32_t const limit,
bool const forward,
std::optional<TransactionsCursor> const& cursorIn,
boost::asio::yield_context& yield) const = 0;
boost::asio::yield_context yield) const = 0;
/*! @brief STATE DATA METHODS */
/**
@@ -333,8 +324,7 @@ public:
* @return std::optional<Blob>
*/
std::optional<Blob>
fetchLedgerObject(ripple::uint256 const& key, std::uint32_t const sequence, boost::asio::yield_context& yield)
const;
fetchLedgerObject(ripple::uint256 const& key, std::uint32_t const sequence, boost::asio::yield_context yield) const;
/**
* @brief Fetches all ledger objects: a vector of vectors of unsigned chars.
@@ -348,11 +338,11 @@ public:
fetchLedgerObjects(
std::vector<ripple::uint256> const& keys,
std::uint32_t const sequence,
boost::asio::yield_context& yield) const;
boost::asio::yield_context yield) const;
/*! @brief Virtual function version of fetchLedgerObject */
virtual std::optional<Blob>
doFetchLedgerObject(ripple::uint256 const& key, std::uint32_t const sequence, boost::asio::yield_context& yield)
doFetchLedgerObject(ripple::uint256 const& key, std::uint32_t const sequence, boost::asio::yield_context yield)
const = 0;
/*! @brief Virtual function version of fetchLedgerObjects */
@@ -360,7 +350,7 @@ public:
doFetchLedgerObjects(
std::vector<ripple::uint256> const& keys,
std::uint32_t const sequence,
boost::asio::yield_context& yield) const = 0;
boost::asio::yield_context yield) const = 0;
/**
* @brief Returns the difference between ledgers: vector of objects
@@ -373,7 +363,7 @@ public:
* @return std::vector<LedgerObject>
*/
virtual std::vector<LedgerObject>
fetchLedgerDiff(std::uint32_t const ledgerSequence, boost::asio::yield_context& yield) const = 0;
fetchLedgerDiff(std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const = 0;
/**
* @brief Fetches a page of ledger objects, ordered by key/index.
@@ -391,20 +381,20 @@ public:
std::uint32_t const ledgerSequence,
std::uint32_t const limit,
bool outOfOrder,
boost::asio::yield_context& yield) const;
boost::asio::yield_context yield) const;
/*! @brief Fetches successor object from key/index. */
std::optional<LedgerObject>
fetchSuccessorObject(ripple::uint256 key, std::uint32_t const ledgerSequence, boost::asio::yield_context& yield)
fetchSuccessorObject(ripple::uint256 key, std::uint32_t const ledgerSequence, boost::asio::yield_context yield)
const;
/*! @brief Fetches successor key from key/index. */
std::optional<ripple::uint256>
fetchSuccessorKey(ripple::uint256 key, std::uint32_t const ledgerSequence, boost::asio::yield_context& yield) const;
fetchSuccessorKey(ripple::uint256 key, std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const;
/*! @brief Virtual function version of fetchSuccessorKey. */
virtual std::optional<ripple::uint256>
doFetchSuccessorKey(ripple::uint256 key, std::uint32_t const ledgerSequence, boost::asio::yield_context& yield)
doFetchSuccessorKey(ripple::uint256 key, std::uint32_t const ledgerSequence, boost::asio::yield_context yield)
const = 0;
/**
@@ -422,7 +412,7 @@ public:
ripple::uint256 const& book,
std::uint32_t const ledgerSequence,
std::uint32_t const limit,
boost::asio::yield_context& yield) const;
boost::asio::yield_context yield) const;
/**
* @brief Returns a ledger range
@@ -442,14 +432,14 @@ public:
/*! @brief Virtual function equivalent of hardFetchLedgerRange. */
virtual std::optional<LedgerRange>
hardFetchLedgerRange(boost::asio::yield_context& yield) const = 0;
hardFetchLedgerRange(boost::asio::yield_context yield) const = 0;
/*! @brief Fetches ledger range but doesn't throw timeout. Use with care. */
std::optional<LedgerRange>
hardFetchLedgerRangeNoThrow() const;
/*! @brief Fetches ledger range but doesn't throw timeout. Use with care. */
std::optional<LedgerRange>
hardFetchLedgerRangeNoThrow(boost::asio::yield_context& yield) const;
hardFetchLedgerRangeNoThrow(boost::asio::yield_context yield) const;
/**
* @brief Writes to a specific ledger.

52
src/backend/CassandraBackend.h
View File

@@ -106,7 +106,7 @@ public:
std::uint32_t const limit,
bool forward,
std::optional<TransactionsCursor> const& cursorIn,
boost::asio::yield_context& yield) const override
boost::asio::yield_context yield) const override
{
auto rng = fetchLedgerRange();
if (!rng)
@@ -210,7 +210,7 @@ public:
}
std::optional<std::uint32_t>
fetchLatestLedgerSequence(boost::asio::yield_context& yield) const override
fetchLatestLedgerSequence(boost::asio::yield_context yield) const override
{
if (auto const res = executor_.read(yield, schema_->selectLatestLedger); res)
{
@@ -234,10 +234,8 @@ public:
}
std::optional<ripple::LedgerHeader>
fetchLedgerBySequence(std::uint32_t const sequence, boost::asio::yield_context& yield) const override
fetchLedgerBySequence(std::uint32_t const sequence, boost::asio::yield_context yield) const override
{
log_.trace() << __func__ << " call for seq " << sequence;
auto const res = executor_.read(yield, schema_->selectLedgerBySeq, sequence);
if (res)
{
@@ -263,10 +261,8 @@ public:
}
std::optional<ripple::LedgerHeader>
fetchLedgerByHash(ripple::uint256 const& hash, boost::asio::yield_context& yield) const override
fetchLedgerByHash(ripple::uint256 const& hash, boost::asio::yield_context yield) const override
{
log_.trace() << __func__ << " call";
if (auto const res = executor_.read(yield, schema_->selectLedgerByHash, hash); res)
{
if (auto const& result = res.value(); result)
@@ -289,10 +285,8 @@ public:
}
std::optional<LedgerRange>
hardFetchLedgerRange(boost::asio::yield_context& yield) const override
hardFetchLedgerRange(boost::asio::yield_context yield) const override
{
log_.trace() << __func__ << " call";
if (auto const res = executor_.read(yield, schema_->selectLedgerRange); res)
{
auto const& results = res.value();
@@ -332,18 +326,16 @@ public:
}
std::vector<TransactionAndMetadata>
fetchAllTransactionsInLedger(std::uint32_t const ledgerSequence, boost::asio::yield_context& yield) const override
fetchAllTransactionsInLedger(std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const override
{
log_.trace() << __func__ << " call";
auto hashes = fetchAllTransactionHashesInLedger(ledgerSequence, yield);
return fetchTransactions(hashes, yield);
}
std::vector<ripple::uint256>
fetchAllTransactionHashesInLedger(std::uint32_t const ledgerSequence, boost::asio::yield_context& yield)
fetchAllTransactionHashesInLedger(std::uint32_t const ledgerSequence, boost::asio::yield_context yield)
const override
{
log_.trace() << __func__ << " call";
auto start = std::chrono::system_clock::now();
auto const res = executor_.read(yield, schema_->selectAllTransactionHashesInLedger, ledgerSequence);
@@ -373,11 +365,9 @@ public:
}
std::optional<NFT>
fetchNFT(ripple::uint256 const& tokenID, std::uint32_t const ledgerSequence, boost::asio::yield_context& yield)
fetchNFT(ripple::uint256 const& tokenID, std::uint32_t const ledgerSequence, boost::asio::yield_context yield)
const override
{
log_.trace() << __func__ << " call";
auto const res = executor_.read(yield, schema_->selectNFT, tokenID, ledgerSequence);
if (not res)
return std::nullopt;
@@ -418,10 +408,8 @@ public:
std::uint32_t const limit,
bool const forward,
std::optional<TransactionsCursor> const& cursorIn,
boost::asio::yield_context& yield) const override
boost::asio::yield_context yield) const override
{
log_.trace() << __func__ << " call";
auto rng = fetchLedgerRange();
if (!rng)
return {{}, {}};
@@ -491,7 +479,7 @@ public:
}
std::optional<Blob>
doFetchLedgerObject(ripple::uint256 const& key, std::uint32_t const sequence, boost::asio::yield_context& yield)
doFetchLedgerObject(ripple::uint256 const& key, std::uint32_t const sequence, boost::asio::yield_context yield)
const override
{
log_.debug() << "Fetching ledger object for seq " << sequence << ", key = " << ripple::to_string(key);
@@ -516,10 +504,8 @@ public:
}
std::optional<TransactionAndMetadata>
fetchTransaction(ripple::uint256 const& hash, boost::asio::yield_context& yield) const override
fetchTransaction(ripple::uint256 const& hash, boost::asio::yield_context yield) const override
{
log_.trace() << __func__ << " call";
if (auto const res = executor_.read(yield, schema_->selectTransaction, hash); res)
{
if (auto const maybeValue = res->template get<Blob, Blob, uint32_t, uint32_t>(); maybeValue)
@@ -541,11 +527,9 @@ public:
}
std::optional<ripple::uint256>
doFetchSuccessorKey(ripple::uint256 key, std::uint32_t const ledgerSequence, boost::asio::yield_context& yield)
doFetchSuccessorKey(ripple::uint256 key, std::uint32_t const ledgerSequence, boost::asio::yield_context yield)
const override
{
log_.trace() << __func__ << " call";
if (auto const res = executor_.read(yield, schema_->selectSuccessor, key, ledgerSequence); res)
{
if (auto const result = res->template get<ripple::uint256>(); result)
@@ -568,10 +552,8 @@ public:
}
std::vector<TransactionAndMetadata>
fetchTransactions(std::vector<ripple::uint256> const& hashes, boost::asio::yield_context& yield) const override
fetchTransactions(std::vector<ripple::uint256> const& hashes, boost::asio::yield_context yield) const override
{
log_.trace() << __func__ << " call";
if (hashes.size() == 0)
return {};
@@ -611,10 +593,8 @@ public:
doFetchLedgerObjects(
std::vector<ripple::uint256> const& keys,
std::uint32_t const sequence,
boost::asio::yield_context& yield) const override
boost::asio::yield_context yield) const override
{
log_.trace() << __func__ << " call";
if (keys.size() == 0)
return {};
@@ -647,10 +627,8 @@ public:
}
std::vector<LedgerObject>
fetchLedgerDiff(std::uint32_t const ledgerSequence, boost::asio::yield_context& yield) const override
fetchLedgerDiff(std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const override
{
log_.trace() << __func__ << " call";
auto const [keys, timeDiff] = util::timed([this, &ledgerSequence, &yield]() -> std::vector<ripple::uint256> {
auto const res = executor_.read(yield, schema_->selectDiff, ledgerSequence);
if (not res)

8
src/backend/Types.h
View File

@@ -87,8 +87,8 @@ struct TransactionAndMetadata
struct TransactionsCursor
{
std::uint32_t ledgerSequence;
std::uint32_t transactionIndex;
std::uint32_t ledgerSequence = 0;
std::uint32_t transactionIndex = 0;
TransactionsCursor() = default;
TransactionsCursor(std::uint32_t ledgerSequence, std::uint32_t transactionIndex)
@@ -155,8 +155,8 @@ struct NFT
struct LedgerRange
{
std::uint32_t minSequence;
std::uint32_t maxSequence;
std::uint32_t minSequence = 0;
std::uint32_t maxSequence = 0;
};
constexpr ripple::uint256 firstKey{"0000000000000000000000000000000000000000000000000000000000000000"};
constexpr ripple::uint256 lastKey{"FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"};

18
src/backend/cassandra/impl/ExecutionStrategy.h
View File

@@ -231,9 +231,13 @@ public:
auto init = [this, &statements, &future]<typename Self>(Self& self) {
future.emplace(handle_.get().asyncExecute(
statements, [sself = std::make_shared<Self>(std::move(self))](auto&& res) mutable {
// Note: explicit work below needed on linux/gcc11
auto executor = boost::asio::get_associated_executor(*sself);
boost::asio::post(
boost::asio::get_associated_executor(*sself),
[sself = std::move(sself), res = std::move(res)]() mutable {
executor,
[sself = std::move(sself),
res = std::move(res),
_ = boost::asio::make_work_guard(executor)]() mutable {
sself->complete(std::move(res));
sself.reset();
});
@@ -279,8 +283,10 @@ public:
auto init = [this, &statement, &future]<typename Self>(Self& self) {
future.emplace(handle_.get().asyncExecute(
statement, [sself = std::make_shared<Self>(std::move(self))](auto&&) mutable {
// Note: explicit work below needed on linux/gcc11
auto executor = boost::asio::get_associated_executor(*sself);
boost::asio::post(
boost::asio::get_associated_executor(*sself), [sself = std::move(sself)]() mutable {
executor, [sself = std::move(sself), _ = boost::asio::make_work_guard(executor)]() mutable {
sself->complete();
sself.reset();
});
@@ -332,11 +338,15 @@ public:
// when all async operations complete unblock the result
if (--numOutstanding == 0)
{
// Note: explicit work below needed on linux/gcc11
auto executor = boost::asio::get_associated_executor(*sself);
boost::asio::post(
boost::asio::get_associated_executor(*sself), [sself = std::move(sself)]() mutable {
executor, [sself = std::move(sself), _ = boost::asio::make_work_guard(executor)]() mutable {
sself->complete();
sself.reset();
});
}
};
std::transform(

2
src/etl/ETLService.cpp
View File

@@ -25,7 +25,7 @@ using namespace clio;
// Database must be populated when this starts
std::optional<uint32_t>
ETLService::runETLPipeline(uint32_t startSequence, int numExtractors)
ETLService::runETLPipeline(uint32_t startSequence, uint32_t numExtractors)
{
if (finishSequence_ && startSequence > *finishSequence_)
return {};

3
src/etl/ETLService.h
View File

@@ -188,10 +188,11 @@ private:
* @note database must already be populated when this function is called
*
* @param startSequence the first ledger to extract
* @param numExtractors number of extractors to use
* @return the last ledger written to the database, if any
*/
std::optional<uint32_t>
runETLPipeline(uint32_t startSequence, int offset);
runETLPipeline(uint32_t startSequence, uint32_t numExtractors);
/**
* @brief Monitor the network for newly validated ledgers.

4
src/etl/LoadBalancer.cpp
View File

@@ -140,11 +140,11 @@ std::optional<boost::json::object>
LoadBalancer::forwardToRippled(
boost::json::object const& request,
std::string const& clientIp,
boost::asio::yield_context& yield) const
boost::asio::yield_context yield) const
{
srand((unsigned)time(0));
auto sourceIdx = rand() % sources_.size();
auto numAttempts = 0;
auto numAttempts = 0u;
while (numAttempts < sources_.size())
{

2
src/etl/LoadBalancer.h
View File

@@ -139,7 +139,7 @@ public:
* @return response received from rippled node
*/
std::optional<boost::json::object>
forwardToRippled(boost::json::object const& request, std::string const& clientIp, boost::asio::yield_context& yield)
forwardToRippled(boost::json::object const& request, std::string const& clientIp, boost::asio::yield_context yield)
const;
private:

4
src/etl/ProbingSource.cpp
View File

@@ -125,7 +125,7 @@ std::optional<boost::json::object>
ProbingSource::forwardToRippled(
boost::json::object const& request,
std::string const& clientIp,
boost::asio::yield_context& yield) const
boost::asio::yield_context yield) const
{
if (!currentSrc_)
return {};
@@ -136,7 +136,7 @@ std::optional<boost::json::object>
ProbingSource::requestFromRippled(
boost::json::object const& request,
std::string const& clientIp,
boost::asio::yield_context& yield) const
boost::asio::yield_context yield) const
{
if (!currentSrc_)
return {};

4
src/etl/ProbingSource.h
View File

@@ -103,7 +103,7 @@ public:
fetchLedger(uint32_t ledgerSequence, bool getObjects = true, bool getObjectNeighbors = false) override;
std::optional<boost::json::object>
forwardToRippled(boost::json::object const& request, std::string const& clientIp, boost::asio::yield_context& yield)
forwardToRippled(boost::json::object const& request, std::string const& clientIp, boost::asio::yield_context yield)
const override;
boost::uuids::uuid
@@ -114,7 +114,7 @@ private:
requestFromRippled(
boost::json::object const& request,
std::string const& clientIp,
boost::asio::yield_context& yield) const override;
boost::asio::yield_context yield) const override;
SourceHooks
make_SSLHooks() noexcept;

631
src/etl/Source.cpp
View File

@@ -20,7 +20,6 @@
#include <backend/DBHelpers.h>
#include <etl/ETLService.h>
#include <etl/LoadBalancer.h>
#include <etl/NFTHelpers.h>
#include <etl/ProbingSource.h>
#include <etl/Source.h>
#include <log/Logger.h>
@@ -60,59 +59,11 @@ make_TimeoutOption()
}
}
template <class Derived>
void
SourceImpl<Derived>::reconnect(boost::beast::error_code ec)
{
if (paused_)
return;
if (connected_)
hooks_.onDisconnected(ec);
connected_ = false;
// These are somewhat normal errors. operation_aborted occurs on shutdown,
// when the timer is cancelled. connection_refused will occur repeatedly
std::string err = ec.message();
// if we cannot connect to the transaction processing process
if (ec.category() == boost::asio::error::get_ssl_category())
{
err = std::string(" (") + boost::lexical_cast<std::string>(ERR_GET_LIB(ec.value())) + "," +
boost::lexical_cast<std::string>(ERR_GET_REASON(ec.value())) + ") ";
// ERR_PACK /* crypto/err/err.h */
char buf[128];
::ERR_error_string_n(ec.value(), buf, sizeof(buf));
err += buf;
log_.error() << err;
}
if (ec != boost::asio::error::operation_aborted && ec != boost::asio::error::connection_refused)
{
log_.error() << "error code = " << ec << " - " << toString();
}
else
{
log_.warn() << "error code = " << ec << " - " << toString();
}
// exponentially increasing timeouts, with a max of 30 seconds
size_t waitTime = std::min(pow(2, numFailures_), 30.0);
numFailures_++;
timer_.expires_after(boost::asio::chrono::seconds(waitTime));
timer_.async_wait([this](auto ec) {
bool startAgain = (ec != boost::asio::error::operation_aborted);
log_.trace() << "async_wait : ec = " << ec;
derived().close(startAgain);
});
}
void
PlainSource::close(bool startAgain)
{
timer_.cancel();
ioc_.post([this, startAgain]() {
boost::asio::post(strand_, [this, startAgain]() {
if (closing_)
return;
@@ -131,18 +82,14 @@ PlainSource::close(bool startAgain)
closing_ = false;
if (startAgain)
{
ws_ = std::make_unique<boost::beast::websocket::stream<boost::beast::tcp_stream>>(
boost::asio::make_strand(ioc_));
ws_ = std::make_unique<StreamType>(strand_);
run();
}
});
}
else if (startAgain)
{
ws_ = std::make_unique<boost::beast::websocket::stream<boost::beast::tcp_stream>>(
boost::asio::make_strand(ioc_));
ws_ = std::make_unique<StreamType>(strand_);
run();
}
});
@@ -152,15 +99,14 @@ void
SslSource::close(bool startAgain)
{
timer_.cancel();
ioc_.post([this, startAgain]() {
boost::asio::post(strand_, [this, startAgain]() {
if (closing_)
return;
if (derived().ws().is_open())
{
// onStop() also calls close(). If the async_close is called twice,
// an assertion fails. Using closing_ makes sure async_close is only
// called once
// onStop() also calls close(). If the async_close is called twice, an assertion fails. Using closing_ makes
// sure async_close is only called once
closing_ = true;
derived().ws().async_close(boost::beast::websocket::close_code::normal, [this, startAgain](auto ec) {
if (ec)
@@ -171,49 +117,24 @@ SslSource::close(bool startAgain)
closing_ = false;
if (startAgain)
{
ws_ = std::make_unique<
boost::beast::websocket::stream<boost::beast::ssl_stream<boost::beast::tcp_stream>>>(
boost::asio::make_strand(ioc_), *sslCtx_);
ws_ = std::make_unique<StreamType>(strand_, *sslCtx_);
run();
}
});
}
else if (startAgain)
{
ws_ = std::make_unique<boost::beast::websocket::stream<boost::beast::ssl_stream<boost::beast::tcp_stream>>>(
boost::asio::make_strand(ioc_), *sslCtx_);
ws_ = std::make_unique<StreamType>(strand_, *sslCtx_);
run();
}
});
}
template <class Derived>
void
SourceImpl<Derived>::onResolve(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type results)
{
log_.trace() << "ec = " << ec << " - " << toString();
if (ec)
{
// try again
reconnect(ec);
}
else
{
boost::beast::get_lowest_layer(derived().ws()).expires_after(std::chrono::seconds(30));
boost::beast::get_lowest_layer(derived().ws()).async_connect(results, [this](auto ec, auto ep) {
derived().onConnect(ec, ep);
});
}
}
void
PlainSource::onConnect(
boost::beast::error_code ec,
boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint)
{
log_.trace() << "ec = " << ec << " - " << toString();
if (ec)
{
// start over
@@ -222,18 +143,14 @@ PlainSource::onConnect(
else
{
numFailures_ = 0;
// Turn off timeout on the tcp stream, because websocket stream has it's
// own timeout system
// Websocket stream has it's own timeout system
boost::beast::get_lowest_layer(derived().ws()).expires_never();
// Set a desired timeout for the websocket stream
derived().ws().set_option(make_TimeoutOption());
// Set a decorator to change the User-Agent of the handshake
derived().ws().set_option(
boost::beast::websocket::stream_base::decorator([](boost::beast::websocket::request_type& req) {
req.set(boost::beast::http::field::user_agent, "clio-client");
req.set("X-User", "clio-client");
}));
@@ -241,7 +158,6 @@ PlainSource::onConnect(
// Host HTTP header during the WebSocket handshake.
// See https://tools.ietf.org/html/rfc7230#section-5.4
auto host = ip_ + ':' + std::to_string(endpoint.port());
// Perform the websocket handshake
derived().ws().async_handshake(host, "/", [this](auto ec) { onHandshake(ec); });
}
}
@@ -249,7 +165,6 @@ PlainSource::onConnect(
void
SslSource::onConnect(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint)
{
log_.trace() << "ec = " << ec << " - " << toString();
if (ec)
{
// start over
@@ -258,18 +173,14 @@ SslSource::onConnect(boost::beast::error_code ec, boost::asio::ip::tcp::resolver
else
{
numFailures_ = 0;
// Turn off timeout on the tcp stream, because websocket stream has it's
// own timeout system
// Websocket stream has it's own timeout system
boost::beast::get_lowest_layer(derived().ws()).expires_never();
// Set a desired timeout for the websocket stream
derived().ws().set_option(make_TimeoutOption());
// Set a decorator to change the User-Agent of the handshake
derived().ws().set_option(
boost::beast::websocket::stream_base::decorator([](boost::beast::websocket::request_type& req) {
req.set(boost::beast::http::field::user_agent, "clio-client");
req.set("X-User", "clio-client");
}));
@@ -277,7 +188,6 @@ SslSource::onConnect(boost::beast::error_code ec, boost::asio::ip::tcp::resolver
// Host HTTP header during the WebSocket handshake.
// See https://tools.ietf.org/html/rfc7230#section-5.4
auto host = ip_ + ':' + std::to_string(endpoint.port());
// Perform the websocket handshake
ws().next_layer().async_handshake(
boost::asio::ssl::stream_base::client, [this, endpoint](auto ec) { onSslHandshake(ec, endpoint); });
}
@@ -294,524 +204,7 @@ SslSource::onSslHandshake(
}
else
{
// Perform the websocket handshake
auto host = ip_ + ':' + std::to_string(endpoint.port());
// Perform the websocket handshake
ws().async_handshake(host, "/", [this](auto ec) { onHandshake(ec); });
}
}
template <class Derived>
void
SourceImpl<Derived>::onHandshake(boost::beast::error_code ec)
{
log_.trace() << "ec = " << ec << " - " << toString();
if (auto action = hooks_.onConnected(ec); action == SourceHooks::Action::STOP)
return;
if (ec)
{
// start over
reconnect(ec);
}
else
{
boost::json::object jv{
{"command", "subscribe"}, {"streams", {"ledger", "manifests", "validations", "transactions_proposed"}}};
std::string s = boost::json::serialize(jv);
log_.trace() << "Sending subscribe stream message";
derived().ws().set_option(
boost::beast::websocket::stream_base::decorator([](boost::beast::websocket::request_type& req) {
req.set(
boost::beast::http::field::user_agent, std::string(BOOST_BEAST_VERSION_STRING) + " clio-client");
req.set("X-User", "coro-client");
}));
// Send the message
derived().ws().async_write(boost::asio::buffer(s), [this](auto ec, size_t size) { onWrite(ec, size); });
}
}
template <class Derived>
void
SourceImpl<Derived>::onWrite(boost::beast::error_code ec, size_t bytesWritten)
{
log_.trace() << "ec = " << ec << " - " << toString();
if (ec)
{
// start over
reconnect(ec);
}
else
{
derived().ws().async_read(readBuffer_, [this](auto ec, size_t size) { onRead(ec, size); });
}
}
template <class Derived>
void
SourceImpl<Derived>::onRead(boost::beast::error_code ec, size_t size)
{
log_.trace() << "ec = " << ec << " - " << toString();
// if error or error reading message, start over
if (ec)
{
reconnect(ec);
}
else
{
handleMessage(size);
log_.trace() << "calling async_read - " << toString();
derived().ws().async_read(readBuffer_, [this](auto ec, size_t size) { onRead(ec, size); });
}
}
template <class Derived>
bool
SourceImpl<Derived>::handleMessage(size_t size)
{
log_.trace() << toString();
setLastMsgTime();
connected_ = true;
try
{
auto const msg = boost::beast::buffers_to_string(readBuffer_.data());
readBuffer_.consume(size);
auto const raw = boost::json::parse(msg);
auto const response = raw.as_object();
uint32_t ledgerIndex = 0;
if (response.contains("result"))
{
auto const& result = response.at("result").as_object();
if (result.contains("ledger_index"))
ledgerIndex = result.at("ledger_index").as_int64();
if (result.contains("validated_ledgers"))
{
auto const& validatedLedgers = result.at("validated_ledgers").as_string();
setValidatedRange({validatedLedgers.data(), validatedLedgers.size()});
}
log_.info() << "Received a message on ledger "
<< " subscription stream. Message : " << response << " - " << toString();
}
else if (response.contains("type") && response.at("type") == "ledgerClosed")
{
log_.info() << "Received a message on ledger "
<< " subscription stream. Message : " << response << " - " << toString();
if (response.contains("ledger_index"))
{
ledgerIndex = response.at("ledger_index").as_int64();
}
if (response.contains("validated_ledgers"))
{
auto const& validatedLedgers = response.at("validated_ledgers").as_string();
setValidatedRange({validatedLedgers.data(), validatedLedgers.size()});
}
}
else
{
if (balancer_.shouldPropagateTxnStream(this))
{
if (response.contains("transaction"))
{
forwardCache_.freshen();
subscriptions_->forwardProposedTransaction(response);
}
else if (response.contains("type") && response.at("type") == "validationReceived")
{
subscriptions_->forwardValidation(response);
}
else if (response.contains("type") && response.at("type") == "manifestReceived")
{
subscriptions_->forwardManifest(response);
}
}
}
if (ledgerIndex != 0)
{
log_.trace() << "Pushing ledger sequence = " << ledgerIndex << " - " << toString();
networkValidatedLedgers_->push(ledgerIndex);
}
return true;
}
catch (std::exception const& e)
{
log_.error() << "Exception in handleMessage : " << e.what();
return false;
}
}
// TODO: move to detail
class AsyncCallData
{
clio::Logger log_{"ETL"};
std::unique_ptr<org::xrpl::rpc::v1::GetLedgerDataResponse> cur_;
std::unique_ptr<org::xrpl::rpc::v1::GetLedgerDataResponse> next_;
org::xrpl::rpc::v1::GetLedgerDataRequest request_;
std::unique_ptr<grpc::ClientContext> context_;
grpc::Status status_;
unsigned char nextPrefix_;
std::string lastKey_;
public:
AsyncCallData(uint32_t seq, ripple::uint256 const& marker, std::optional<ripple::uint256> const& nextMarker)
{
request_.mutable_ledger()->set_sequence(seq);
if (marker.isNonZero())
{
request_.set_marker(marker.data(), marker.size());
}
request_.set_user("ETL");
nextPrefix_ = 0x00;
if (nextMarker)
nextPrefix_ = nextMarker->data()[0];
unsigned char prefix = marker.data()[0];
log_.debug() << "Setting up AsyncCallData. marker = " << ripple::strHex(marker)
<< " . prefix = " << ripple::strHex(std::string(1, prefix))
<< " . nextPrefix_ = " << ripple::strHex(std::string(1, nextPrefix_));
assert(nextPrefix_ > prefix || nextPrefix_ == 0x00);
cur_ = std::make_unique<org::xrpl::rpc::v1::GetLedgerDataResponse>();
next_ = std::make_unique<org::xrpl::rpc::v1::GetLedgerDataResponse>();
context_ = std::make_unique<grpc::ClientContext>();
}
enum class CallStatus { MORE, DONE, ERRORED };
CallStatus
process(
std::unique_ptr<org::xrpl::rpc::v1::XRPLedgerAPIService::Stub>& stub,
grpc::CompletionQueue& cq,
BackendInterface& backend,
bool abort,
bool cacheOnly = false)
{
log_.trace() << "Processing response. "
<< "Marker prefix = " << getMarkerPrefix();
if (abort)
{
log_.error() << "AsyncCallData aborted";
return CallStatus::ERRORED;
}
if (!status_.ok())
{
log_.error() << "AsyncCallData status_ not ok: "
<< " code = " << status_.error_code() << " message = " << status_.error_message();
return CallStatus::ERRORED;
}
if (!next_->is_unlimited())
{
log_.warn() << "AsyncCallData is_unlimited is false. Make sure "
"secure_gateway is set correctly at the ETL source";
}
std::swap(cur_, next_);
bool more = true;
// if no marker returned, we are done
if (cur_->marker().size() == 0)
more = false;
// if returned marker is greater than our end, we are done
unsigned char prefix = cur_->marker()[0];
if (nextPrefix_ != 0x00 && prefix >= nextPrefix_)
more = false;
// if we are not done, make the next async call
if (more)
{
request_.set_marker(std::move(cur_->marker()));
call(stub, cq);
}
auto const numObjects = cur_->ledger_objects().objects_size();
log_.debug() << "Writing " << numObjects << " objects";
std::vector<Backend::LedgerObject> cacheUpdates;
cacheUpdates.reserve(numObjects);
for (int i = 0; i < numObjects; ++i)
{
auto& obj = *(cur_->mutable_ledger_objects()->mutable_objects(i));
if (!more && nextPrefix_ != 0x00)
{
if (((unsigned char)obj.key()[0]) >= nextPrefix_)
continue;
}
cacheUpdates.push_back(
{*ripple::uint256::fromVoidChecked(obj.key()),
{obj.mutable_data()->begin(), obj.mutable_data()->end()}});
if (!cacheOnly)
{
if (lastKey_.size())
backend.writeSuccessor(std::move(lastKey_), request_.ledger().sequence(), std::string{obj.key()});
lastKey_ = obj.key();
backend.writeNFTs(getNFTDataFromObj(request_.ledger().sequence(), obj.key(), obj.data()));
backend.writeLedgerObject(
std::move(*obj.mutable_key()), request_.ledger().sequence(), std::move(*obj.mutable_data()));
}
}
backend.cache().update(cacheUpdates, request_.ledger().sequence(), cacheOnly);
log_.debug() << "Wrote " << numObjects << " objects. Got more: " << (more ? "YES" : "NO");
return more ? CallStatus::MORE : CallStatus::DONE;
}
void
call(std::unique_ptr<org::xrpl::rpc::v1::XRPLedgerAPIService::Stub>& stub, grpc::CompletionQueue& cq)
{
context_ = std::make_unique<grpc::ClientContext>();
std::unique_ptr<grpc::ClientAsyncResponseReader<org::xrpl::rpc::v1::GetLedgerDataResponse>> rpc(
stub->PrepareAsyncGetLedgerData(context_.get(), request_, &cq));
rpc->StartCall();
rpc->Finish(next_.get(), &status_, this);
}
std::string
getMarkerPrefix()
{
if (next_->marker().size() == 0)
return "";
else
return ripple::strHex(std::string{next_->marker().data()[0]});
}
std::string
getLastKey()
{
return lastKey_;
}
};
template <class Derived>
std::pair<std::vector<std::string>, bool>
SourceImpl<Derived>::loadInitialLedger(uint32_t sequence, uint32_t numMarkers, bool cacheOnly)
{
if (!stub_)
return {{}, false};
grpc::CompletionQueue cq;
void* tag;
bool ok = false;
std::vector<AsyncCallData> calls;
auto markers = getMarkers(numMarkers);
for (size_t i = 0; i < markers.size(); ++i)
{
std::optional<ripple::uint256> nextMarker;
if (i + 1 < markers.size())
nextMarker = markers[i + 1];
calls.emplace_back(sequence, markers[i], nextMarker);
}
log_.debug() << "Starting data download for ledger " << sequence << ". Using source = " << toString();
for (auto& c : calls)
c.call(stub_, cq);
size_t numFinished = 0;
bool abort = false;
size_t incr = 500000;
size_t progress = incr;
std::vector<std::string> edgeKeys;
while (numFinished < calls.size() && cq.Next(&tag, &ok))
{
assert(tag);
auto ptr = static_cast<AsyncCallData*>(tag);
if (!ok)
{
log_.error() << "loadInitialLedger - ok is false";
return {{}, false}; // handle cancelled
}
else
{
log_.trace() << "Marker prefix = " << ptr->getMarkerPrefix();
auto result = ptr->process(stub_, cq, *backend_, abort, cacheOnly);
if (result != AsyncCallData::CallStatus::MORE)
{
numFinished++;
log_.debug() << "Finished a marker. "
<< "Current number of finished = " << numFinished;
std::string lastKey = ptr->getLastKey();
if (lastKey.size())
edgeKeys.push_back(ptr->getLastKey());
}
if (result == AsyncCallData::CallStatus::ERRORED)
abort = true;
if (backend_->cache().size() > progress)
{
log_.info() << "Downloaded " << backend_->cache().size() << " records from rippled";
progress += incr;
}
}
}
log_.info() << "Finished loadInitialLedger. cache size = " << backend_->cache().size();
return {std::move(edgeKeys), !abort};
}
template <class Derived>
std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>
SourceImpl<Derived>::fetchLedger(uint32_t ledgerSequence, bool getObjects, bool getObjectNeighbors)
{
org::xrpl::rpc::v1::GetLedgerResponse response;
if (!stub_)
return {{grpc::StatusCode::INTERNAL, "No Stub"}, response};
// ledger header with txns and metadata
org::xrpl::rpc::v1::GetLedgerRequest request;
grpc::ClientContext context;
request.mutable_ledger()->set_sequence(ledgerSequence);
request.set_transactions(true);
request.set_expand(true);
request.set_get_objects(getObjects);
request.set_get_object_neighbors(getObjectNeighbors);
request.set_user("ETL");
grpc::Status status = stub_->GetLedger(&context, request, &response);
if (status.ok() && !response.is_unlimited())
{
log_.warn() << "SourceImpl::fetchLedger - is_unlimited is "
"false. Make sure secure_gateway is set "
"correctly on the ETL source. source = "
<< toString() << " status = " << status.error_message();
}
return {status, std::move(response)};
}
template <class Derived>
std::optional<boost::json::object>
SourceImpl<Derived>::forwardToRippled(
boost::json::object const& request,
std::string const& clientIp,
boost::asio::yield_context& yield) const
{
if (auto resp = forwardCache_.get(request); resp)
{
log_.debug() << "request hit forwardCache";
return resp;
}
return requestFromRippled(request, clientIp, yield);
}
template <class Derived>
std::optional<boost::json::object>
SourceImpl<Derived>::requestFromRippled(
boost::json::object const& request,
std::string const& clientIp,
boost::asio::yield_context& yield) const
{
log_.trace() << "Attempting to forward request to tx. "
<< "request = " << boost::json::serialize(request);
boost::json::object response;
if (!connected_)
{
log_.error() << "Attempted to proxy but failed to connect to tx";
return {};
}
namespace beast = boost::beast; // from <boost/beast.hpp>
namespace http = beast::http; // from <boost/beast/http.hpp>
namespace websocket = beast::websocket; // from
namespace net = boost::asio; // from
using tcp = boost::asio::ip::tcp; // from
try
{
boost::beast::error_code ec;
// These objects perform our I/O
tcp::resolver resolver{ioc_};
log_.trace() << "Creating websocket";
auto ws = std::make_unique<websocket::stream<beast::tcp_stream>>(ioc_);
// Look up the domain name
auto const results = resolver.async_resolve(ip_, wsPort_, yield[ec]);
if (ec)
return {};
ws->next_layer().expires_after(std::chrono::seconds(3));
log_.trace() << "Connecting websocket";
// Make the connection on the IP address we get from a lookup
ws->next_layer().async_connect(results, yield[ec]);
if (ec)
return {};
// Set a decorator to change the User-Agent of the handshake
// and to tell rippled to charge the client IP for RPC
// resources. See "secure_gateway" in
//
// https://github.com/ripple/rippled/blob/develop/cfg/rippled-example.cfg
ws->set_option(websocket::stream_base::decorator([&clientIp](websocket::request_type& req) {
req.set(http::field::user_agent, std::string(BOOST_BEAST_VERSION_STRING) + " websocket-client-coro");
req.set(http::field::forwarded, "for=" + clientIp);
}));
log_.trace() << "client ip: " << clientIp;
log_.trace() << "Performing websocket handshake";
// Perform the websocket handshake
ws->async_handshake(ip_, "/", yield[ec]);
if (ec)
return {};
log_.trace() << "Sending request";
// Send the message
ws->async_write(net::buffer(boost::json::serialize(request)), yield[ec]);
if (ec)
return {};
beast::flat_buffer buffer;
ws->async_read(buffer, yield[ec]);
if (ec)
return {};
auto begin = static_cast<char const*>(buffer.data().data());
auto end = begin + buffer.data().size();
auto parsed = boost::json::parse(std::string(begin, end));
if (!parsed.is_object())
{
log_.error() << "Error parsing response: " << std::string{begin, end};
return {};
}
log_.trace() << "Successfully forward request";
response = parsed.as_object();
response["forwarded"] = true;
return response;
}
catch (std::exception const& e)
{
log_.error() << "Encountered exception : " << e.what();
return {};
}
}

449
src/etl/Source.h
View File

@@ -22,6 +22,8 @@
#include <backend/BackendInterface.h>
#include <config/Config.h>
#include <etl/ETLHelpers.h>
#include <etl/LoadBalancer.h>
#include <etl/impl/AsyncData.h>
#include <etl/impl/ForwardCache.h>
#include <log/Logger.h>
#include <subscriptions/SubscriptionManager.h>
@@ -37,8 +39,6 @@
#include <boost/uuid/uuid_generators.hpp>
#include <grpcpp/grpcpp.h>
class LoadBalancer;
class Source;
class ProbingSource;
class SubscriptionManager;
@@ -82,7 +82,7 @@ public:
loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, bool cacheOnly = false) = 0;
virtual std::optional<boost::json::object>
forwardToRippled(boost::json::object const& request, std::string const& clientIp, boost::asio::yield_context& yield)
forwardToRippled(boost::json::object const& request, std::string const& clientIp, boost::asio::yield_context yield)
const = 0;
virtual boost::uuids::uuid
@@ -107,7 +107,7 @@ private:
requestFromRippled(
boost::json::object const& request,
std::string const& clientIp,
boost::asio::yield_context& yield) const = 0;
boost::asio::yield_context yield) const = 0;
};
/**
@@ -130,10 +130,6 @@ class SourceImpl : public Source
std::string wsPort_;
std::string grpcPort_;
std::unique_ptr<org::xrpl::rpc::v1::XRPLedgerAPIService::Stub> stub_;
boost::asio::ip::tcp::resolver resolver_;
boost::beast::flat_buffer readBuffer_;
std::vector<std::pair<uint32_t, uint32_t>> validatedLedgers_;
std::string validatedLedgersRaw_{"N/A"};
std::shared_ptr<NetworkValidatedLedgers> networkValidatedLedgers_;
@@ -160,8 +156,12 @@ protected:
std::string ip_;
size_t numFailures_ = 0;
boost::asio::io_context& ioc_;
boost::asio::strand<boost::asio::io_context::executor_type> strand_;
boost::asio::steady_timer timer_;
boost::asio::ip::tcp::resolver resolver_;
boost::beast::flat_buffer readBuffer_;
std::unique_ptr<org::xrpl::rpc::v1::XRPLedgerAPIService::Stub> stub_;
std::atomic_bool closing_{false};
std::atomic_bool paused_{false};
@@ -183,14 +183,14 @@ public:
std::shared_ptr<NetworkValidatedLedgers> networkValidatedLedgers,
LoadBalancer& balancer,
SourceHooks hooks)
: resolver_(boost::asio::make_strand(ioContext))
, networkValidatedLedgers_(networkValidatedLedgers)
: networkValidatedLedgers_(networkValidatedLedgers)
, backend_(backend)
, subscriptions_(subscriptions)
, balancer_(balancer)
, forwardCache_(config, ioContext, *this)
, ioc_(ioContext)
, timer_(boost::asio::make_strand(ioContext))
, strand_(boost::asio::make_strand(ioContext))
, timer_(strand_)
, resolver_(strand_)
, hooks_(hooks)
{
static boost::uuids::random_generator uuidGenerator;
@@ -255,7 +255,83 @@ public:
requestFromRippled(
boost::json::object const& request,
std::string const& clientIp,
boost::asio::yield_context& yield) const override;
boost::asio::yield_context yield) const override
{
log_.trace() << "Attempting to forward request to tx. "
<< "request = " << boost::json::serialize(request);
boost::json::object response;
if (!connected_)
{
log_.error() << "Attempted to proxy but failed to connect to tx";
return {};
}
namespace beast = boost::beast;
namespace http = beast::http;
namespace websocket = beast::websocket;
namespace net = boost::asio;
using tcp = boost::asio::ip::tcp;
try
{
auto executor = boost::asio::get_associated_executor(yield);
boost::beast::error_code ec;
tcp::resolver resolver{executor};
auto ws = std::make_unique<websocket::stream<beast::tcp_stream>>(executor);
auto const results = resolver.async_resolve(ip_, wsPort_, yield[ec]);
if (ec)
return {};
ws->next_layer().expires_after(std::chrono::seconds(3));
ws->next_layer().async_connect(results, yield[ec]);
if (ec)
return {};
// Set a decorator to change the User-Agent of the handshake and to tell rippled to charge the client IP for
// RPC resources. See "secure_gateway" in
// https://github.com/ripple/rippled/blob/develop/cfg/rippled-example.cfg
ws->set_option(websocket::stream_base::decorator([&clientIp](websocket::request_type& req) {
req.set(http::field::user_agent, std::string(BOOST_BEAST_VERSION_STRING) + " websocket-client-coro");
req.set(http::field::forwarded, "for=" + clientIp);
}));
ws->async_handshake(ip_, "/", yield[ec]);
if (ec)
return {};
ws->async_write(net::buffer(boost::json::serialize(request)), yield[ec]);
if (ec)
return {};
beast::flat_buffer buffer;
ws->async_read(buffer, yield[ec]);
if (ec)
return {};
auto begin = static_cast<char const*>(buffer.data().data());
auto end = begin + buffer.data().size();
auto parsed = boost::json::parse(std::string(begin, end));
if (!parsed.is_object())
{
log_.error() << "Error parsing response: " << std::string{begin, end};
return {};
}
response = parsed.as_object();
response["forwarded"] = true;
return response;
}
catch (std::exception const& e)
{
log_.error() << "Encountered exception : " << e.what();
return {};
}
}
/**
* @param sequence ledger sequence to check for
@@ -339,7 +415,34 @@ public:
* @return the extracted data and the result status
*/
std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>
fetchLedger(uint32_t ledgerSequence, bool getObjects = true, bool getObjectNeighbors = false) override;
fetchLedger(uint32_t ledgerSequence, bool getObjects = true, bool getObjectNeighbors = false) override
{
org::xrpl::rpc::v1::GetLedgerResponse response;
if (!stub_)
return {{grpc::StatusCode::INTERNAL, "No Stub"}, response};
// Ledger header with txns and metadata
org::xrpl::rpc::v1::GetLedgerRequest request;
grpc::ClientContext context;
request.mutable_ledger()->set_sequence(ledgerSequence);
request.set_transactions(true);
request.set_expand(true);
request.set_get_objects(getObjects);
request.set_get_object_neighbors(getObjectNeighbors);
request.set_user("ETL");
grpc::Status status = stub_->GetLedger(&context, request, &response);
if (status.ok() && !response.is_unlimited())
{
log_.warn()
<< "is_unlimited is false. Make sure secure_gateway is set correctly on the ETL source. source = "
<< toString() << "; status = " << status.error_message();
}
return {status, std::move(response)};
}
/**
* @brief Produces a human-readable string with info about the source
@@ -383,13 +486,130 @@ public:
* @return true if the download was successful
*/
std::pair<std::vector<std::string>, bool>
loadInitialLedger(std::uint32_t ledgerSequence, std::uint32_t numMarkers, bool cacheOnly = false) override;
loadInitialLedger(std::uint32_t ledgerSequence, std::uint32_t numMarkers, bool cacheOnly = false) override
{
if (!stub_)
return {{}, false};
grpc::CompletionQueue cq;
void* tag;
bool ok = false;
std::vector<clio::detail::AsyncCallData> calls;
auto markers = getMarkers(numMarkers);
for (size_t i = 0; i < markers.size(); ++i)
{
std::optional<ripple::uint256> nextMarker;
if (i + 1 < markers.size())
nextMarker = markers[i + 1];
calls.emplace_back(ledgerSequence, markers[i], nextMarker);
}
log_.debug() << "Starting data download for ledger " << ledgerSequence << ". Using source = " << toString();
for (auto& c : calls)
c.call(stub_, cq);
size_t numFinished = 0;
bool abort = false;
size_t incr = 500000;
size_t progress = incr;
std::vector<std::string> edgeKeys;
while (numFinished < calls.size() && cq.Next(&tag, &ok))
{
assert(tag);
auto ptr = static_cast<clio::detail::AsyncCallData*>(tag);
if (!ok)
{
log_.error() << "loadInitialLedger - ok is false";
return {{}, false}; // handle cancelled
}
else
{
log_.trace() << "Marker prefix = " << ptr->getMarkerPrefix();
auto result = ptr->process(stub_, cq, *backend_, abort, cacheOnly);
if (result != clio::detail::AsyncCallData::CallStatus::MORE)
{
++numFinished;
log_.debug() << "Finished a marker. "
<< "Current number of finished = " << numFinished;
std::string lastKey = ptr->getLastKey();
if (lastKey.size())
edgeKeys.push_back(ptr->getLastKey());
}
if (result == clio::detail::AsyncCallData::CallStatus::ERRORED)
abort = true;
if (backend_->cache().size() > progress)
{
log_.info() << "Downloaded " << backend_->cache().size() << " records from rippled";
progress += incr;
}
}
}
log_.info() << "Finished loadInitialLedger. cache size = " << backend_->cache().size();
return {std::move(edgeKeys), !abort};
}
/**
* @brief Attempt to reconnect to the ETL source
*/
void
reconnect(boost::beast::error_code ec);
reconnect(boost::beast::error_code ec)
{
if (paused_)
return;
if (connected_)
hooks_.onDisconnected(ec);
connected_ = false;
readBuffer_ = {};
// These are somewhat normal errors. operation_aborted occurs on shutdown,
// when the timer is cancelled. connection_refused will occur repeatedly
std::string err = ec.message();
// if we cannot connect to the transaction processing process
if (ec.category() == boost::asio::error::get_ssl_category())
{
err = std::string(" (") + boost::lexical_cast<std::string>(ERR_GET_LIB(ec.value())) + "," +
boost::lexical_cast<std::string>(ERR_GET_REASON(ec.value())) + ") ";
// ERR_PACK /* crypto/err/err.h */
char buf[128];
::ERR_error_string_n(ec.value(), buf, sizeof(buf));
err += buf;
log_.error() << err;
}
if (ec != boost::asio::error::operation_aborted && ec != boost::asio::error::connection_refused)
{
log_.error() << "error code = " << ec << " - " << toString();
}
else
{
log_.warn() << "error code = " << ec << " - " << toString();
}
// exponentially increasing timeouts, with a max of 30 seconds
size_t waitTime = std::min(pow(2, numFailures_), 30.0);
numFailures_++;
timer_.expires_after(boost::asio::chrono::seconds(waitTime));
timer_.async_wait([this](auto ec) {
bool startAgain = (ec != boost::asio::error::operation_aborted);
derived().close(startAgain);
});
}
/**
* @brief Pause the source effectively stopping it from trying to reconnect
@@ -415,46 +635,186 @@ public:
* @brief Callback for resolving the server host
*/
void
onResolve(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type results);
/**
* @brief Callback for connection to the server
*/
virtual void
onConnect(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint) = 0;
onResolve(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type results)
{
if (ec)
{
// try again
reconnect(ec);
}
else
{
boost::beast::get_lowest_layer(derived().ws()).expires_after(std::chrono::seconds(30));
boost::beast::get_lowest_layer(derived().ws()).async_connect(results, [this](auto ec, auto ep) {
derived().onConnect(ec, ep);
});
}
}
/**
* @brief Callback for handshake with the server
*/
void
onHandshake(boost::beast::error_code ec);
onHandshake(boost::beast::error_code ec)
{
if (auto action = hooks_.onConnected(ec); action == SourceHooks::Action::STOP)
return;
if (ec)
{
// start over
reconnect(ec);
}
else
{
boost::json::object jv{
{"command", "subscribe"},
{"streams", {"ledger", "manifests", "validations", "transactions_proposed"}},
};
std::string s = boost::json::serialize(jv);
log_.trace() << "Sending subscribe stream message";
derived().ws().set_option(
boost::beast::websocket::stream_base::decorator([](boost::beast::websocket::request_type& req) {
req.set(
boost::beast::http::field::user_agent,
std::string(BOOST_BEAST_VERSION_STRING) + " clio-client");
req.set("X-User", "coro-client");
}));
// Send subscription message
derived().ws().async_write(boost::asio::buffer(s), [this](auto ec, size_t size) { onWrite(ec, size); });
}
}
/**
* @brief Callback for writing data
*/
void
onWrite(boost::beast::error_code ec, size_t size);
onWrite(boost::beast::error_code ec, size_t size)
{
if (ec)
reconnect(ec);
else
derived().ws().async_read(readBuffer_, [this](auto ec, size_t size) { onRead(ec, size); });
}
/**
* @brief Callback for data available to read
*/
void
onRead(boost::beast::error_code ec, size_t size);
onRead(boost::beast::error_code ec, size_t size)
{
if (ec)
{
reconnect(ec);
}
else
{
handleMessage(size);
derived().ws().async_read(readBuffer_, [this](auto ec, size_t size) { onRead(ec, size); });
}
}
/**
* @brief Handle the most recently received message
* @return true if the message was handled successfully. false on error
*/
bool
handleMessage(size_t size);
handleMessage(size_t size)
{
setLastMsgTime();
connected_ = true;
try
{
auto const msg = boost::beast::buffers_to_string(readBuffer_.data());
readBuffer_.consume(size);
auto const raw = boost::json::parse(msg);
auto const response = raw.as_object();
uint32_t ledgerIndex = 0;
if (response.contains("result"))
{
auto const& result = response.at("result").as_object();
if (result.contains("ledger_index"))
ledgerIndex = result.at("ledger_index").as_int64();
if (result.contains("validated_ledgers"))
{
auto const& validatedLedgers = result.at("validated_ledgers").as_string();
setValidatedRange({validatedLedgers.data(), validatedLedgers.size()});
}
log_.info() << "Received a message on ledger "
<< " subscription stream. Message : " << response << " - " << toString();
}
else if (response.contains("type") && response.at("type") == "ledgerClosed")
{
log_.info() << "Received a message on ledger "
<< " subscription stream. Message : " << response << " - " << toString();
if (response.contains("ledger_index"))
{
ledgerIndex = response.at("ledger_index").as_int64();
}
if (response.contains("validated_ledgers"))
{
auto const& validatedLedgers = response.at("validated_ledgers").as_string();
setValidatedRange({validatedLedgers.data(), validatedLedgers.size()});
}
}
else
{
if (balancer_.shouldPropagateTxnStream(this))
{
if (response.contains("transaction"))
{
forwardCache_.freshen();
subscriptions_->forwardProposedTransaction(response);
}
else if (response.contains("type") && response.at("type") == "validationReceived")
{
subscriptions_->forwardValidation(response);
}
else if (response.contains("type") && response.at("type") == "manifestReceived")
{
subscriptions_->forwardManifest(response);
}
}
}
if (ledgerIndex != 0)
{
log_.trace() << "Pushing ledger sequence = " << ledgerIndex << " - " << toString();
networkValidatedLedgers_->push(ledgerIndex);
}
return true;
}
catch (std::exception const& e)
{
log_.error() << "Exception in handleMessage : " << e.what();
return false;
}
}
/**
* @brief Forward a request to rippled
* @return response wrapped in an optional on success; nullopt otherwise
*/
std::optional<boost::json::object>
forwardToRippled(boost::json::object const& request, std::string const& clientIp, boost::asio::yield_context& yield)
const override;
forwardToRippled(boost::json::object const& request, std::string const& clientIp, boost::asio::yield_context yield)
const override
{
if (auto resp = forwardCache_.get(request); resp)
{
log_.debug() << "request hit forwardCache";
return resp;
}
return requestFromRippled(request, clientIp, yield);
}
protected:
Derived&
@@ -466,18 +826,14 @@ protected:
void
run() override
{
log_.trace() << toString();
auto const host = ip_;
auto const port = wsPort_;
resolver_.async_resolve(host, port, [this](auto ec, auto results) { onResolve(ec, results); });
resolver_.async_resolve(ip_, wsPort_, [this](auto ec, auto results) { onResolve(ec, results); });
}
};
class PlainSource : public SourceImpl<PlainSource>
{
std::unique_ptr<boost::beast::websocket::stream<boost::beast::tcp_stream>> ws_;
using StreamType = boost::beast::websocket::stream<boost::beast::tcp_stream>;
std::unique_ptr<StreamType> ws_;
public:
PlainSource(
@@ -489,8 +845,7 @@ public:
LoadBalancer& balancer,
SourceHooks hooks)
: SourceImpl(config, ioc, backend, subscriptions, nwvl, balancer, std::move(hooks))
, ws_(std::make_unique<boost::beast::websocket::stream<boost::beast::tcp_stream>>(
boost::asio::make_strand(ioc)))
, ws_(std::make_unique<StreamType>(strand_))
{
}
@@ -498,8 +853,7 @@ public:
* @brief Callback for connection to the server
*/
void
onConnect(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint)
override;
onConnect(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint);
/**
* @brief Close the websocket
@@ -517,9 +871,9 @@ public:
class SslSource : public SourceImpl<SslSource>
{
using StreamType = boost::beast::websocket::stream<boost::beast::ssl_stream<boost::beast::tcp_stream>>;
std::optional<std::reference_wrapper<boost::asio::ssl::context>> sslCtx_;
std::unique_ptr<boost::beast::websocket::stream<boost::beast::ssl_stream<boost::beast::tcp_stream>>> ws_;
std::unique_ptr<StreamType> ws_;
public:
SslSource(
@@ -533,9 +887,7 @@ public:
SourceHooks hooks)
: SourceImpl(config, ioc, backend, subscriptions, nwvl, balancer, std::move(hooks))
, sslCtx_(sslCtx)
, ws_(std::make_unique<boost::beast::websocket::stream<boost::beast::ssl_stream<boost::beast::tcp_stream>>>(
boost::asio::make_strand(ioc_),
*sslCtx_))
, ws_(std::make_unique<StreamType>(strand_, *sslCtx_))
{
}
@@ -543,8 +895,7 @@ public:
* @brief Callback for connection to the server
*/
void
onConnect(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint)
override;
onConnect(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint);
/**
* @brief Callback for SSL handshake completion
@@ -559,7 +910,7 @@ public:
void
close(bool startAgain);
boost::beast::websocket::stream<boost::beast::ssl_stream<boost::beast::tcp_stream>>&
StreamType&
ws()
{
return *ws_;

182
src/etl/impl/AsyncData.h Normal file
View File

@@ -0,0 +1,182 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2023, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include <etl/NFTHelpers.h>
#include <log/Logger.h>
#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
#include <grpcpp/grpcpp.h>
namespace clio::detail {
class AsyncCallData
{
clio::Logger log_{"ETL"};
std::unique_ptr<org::xrpl::rpc::v1::GetLedgerDataResponse> cur_;
std::unique_ptr<org::xrpl::rpc::v1::GetLedgerDataResponse> next_;
org::xrpl::rpc::v1::GetLedgerDataRequest request_;
std::unique_ptr<grpc::ClientContext> context_;
grpc::Status status_;
unsigned char nextPrefix_;
std::string lastKey_;
public:
AsyncCallData(uint32_t seq, ripple::uint256 const& marker, std::optional<ripple::uint256> const& nextMarker)
{
request_.mutable_ledger()->set_sequence(seq);
if (marker.isNonZero())
{
request_.set_marker(marker.data(), marker.size());
}
request_.set_user("ETL");
nextPrefix_ = 0x00;
if (nextMarker)
nextPrefix_ = nextMarker->data()[0];
unsigned char prefix = marker.data()[0];
log_.debug() << "Setting up AsyncCallData. marker = " << ripple::strHex(marker)
<< " . prefix = " << ripple::strHex(std::string(1, prefix))
<< " . nextPrefix_ = " << ripple::strHex(std::string(1, nextPrefix_));
assert(nextPrefix_ > prefix || nextPrefix_ == 0x00);
cur_ = std::make_unique<org::xrpl::rpc::v1::GetLedgerDataResponse>();
next_ = std::make_unique<org::xrpl::rpc::v1::GetLedgerDataResponse>();
context_ = std::make_unique<grpc::ClientContext>();
}
enum class CallStatus { MORE, DONE, ERRORED };
CallStatus
process(
std::unique_ptr<org::xrpl::rpc::v1::XRPLedgerAPIService::Stub>& stub,
grpc::CompletionQueue& cq,
BackendInterface& backend,
bool abort,
bool cacheOnly = false)
{
log_.trace() << "Processing response. "
<< "Marker prefix = " << getMarkerPrefix();
if (abort)
{
log_.error() << "AsyncCallData aborted";
return CallStatus::ERRORED;
}
if (!status_.ok())
{
log_.error() << "AsyncCallData status_ not ok: "
<< " code = " << status_.error_code() << " message = " << status_.error_message();
return CallStatus::ERRORED;
}
if (!next_->is_unlimited())
{
log_.warn() << "AsyncCallData is_unlimited is false. Make sure "
"secure_gateway is set correctly at the ETL source";
}
std::swap(cur_, next_);
bool more = true;
// if no marker returned, we are done
if (cur_->marker().size() == 0)
more = false;
// if returned marker is greater than our end, we are done
unsigned char prefix = cur_->marker()[0];
if (nextPrefix_ != 0x00 && prefix >= nextPrefix_)
more = false;
// if we are not done, make the next async call
if (more)
{
request_.set_marker(std::move(cur_->marker()));
call(stub, cq);
}
auto const numObjects = cur_->ledger_objects().objects_size();
log_.debug() << "Writing " << numObjects << " objects";
std::vector<Backend::LedgerObject> cacheUpdates;
cacheUpdates.reserve(numObjects);
for (int i = 0; i < numObjects; ++i)
{
auto& obj = *(cur_->mutable_ledger_objects()->mutable_objects(i));
if (!more && nextPrefix_ != 0x00)
{
if (((unsigned char)obj.key()[0]) >= nextPrefix_)
continue;
}
cacheUpdates.push_back(
{*ripple::uint256::fromVoidChecked(obj.key()),
{obj.mutable_data()->begin(), obj.mutable_data()->end()}});
if (!cacheOnly)
{
if (lastKey_.size())
backend.writeSuccessor(std::move(lastKey_), request_.ledger().sequence(), std::string{obj.key()});
lastKey_ = obj.key();
backend.writeNFTs(getNFTDataFromObj(request_.ledger().sequence(), obj.key(), obj.data()));
backend.writeLedgerObject(
std::move(*obj.mutable_key()), request_.ledger().sequence(), std::move(*obj.mutable_data()));
}
}
backend.cache().update(cacheUpdates, request_.ledger().sequence(), cacheOnly);
log_.debug() << "Wrote " << numObjects << " objects. Got more: " << (more ? "YES" : "NO");
return more ? CallStatus::MORE : CallStatus::DONE;
}
void
call(std::unique_ptr<org::xrpl::rpc::v1::XRPLedgerAPIService::Stub>& stub, grpc::CompletionQueue& cq)
{
context_ = std::make_unique<grpc::ClientContext>();
std::unique_ptr<grpc::ClientAsyncResponseReader<org::xrpl::rpc::v1::GetLedgerDataResponse>> rpc(
stub->PrepareAsyncGetLedgerData(context_.get(), request_, &cq));
rpc->StartCall();
rpc->Finish(next_.get(), &status_, this);
}
std::string
getMarkerPrefix()
{
if (next_->marker().size() == 0)
return "";
else
return ripple::strHex(std::string{next_->marker().data()[0]});
}
std::string
getLastKey()
{
return lastKey_;
}
};
} // namespace clio::detail

3
src/etl/impl/CacheLoader.h
View File

@@ -19,6 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <log/Logger.h>
#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
@@ -183,7 +184,7 @@ private:
uint32_t ledgerIndex,
std::string const& ip,
std::string const& port,
boost::asio::yield_context& yield)
boost::asio::yield_context yield)
{
log_.info() << "Loading cache from peer. ip = " << ip << " . port = " << port;
namespace beast = boost::beast; // from <boost/beast.hpp>

2
src/etl/impl/ForwardCache.h
View File

@@ -55,7 +55,7 @@ class ForwardCache
public:
ForwardCache(clio::Config const& config, boost::asio::io_context& ioc, Source const& source)
: strand_(ioc.get_executor()), source_(source)
: strand_(boost::asio::make_strand(ioc)), source_(source)
{
if (config.contains("cache"))
{

5
src/etl/impl/LedgerPublisher.h
View File

@@ -71,7 +71,10 @@ public:
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,
SystemState const& state)
: publishStrand_{ioc.get_executor()}, backend_{backend}, subscriptions_{subscriptions}, state_{std::cref(state)}
: publishStrand_{boost::asio::make_strand(ioc)}
, backend_{backend}
, subscriptions_{subscriptions}
, state_{std::cref(state)}
{
}

6
src/log/Logger.cpp
View File

@@ -88,8 +88,7 @@ LogService::init(Config const& config)
boost::log::add_console_log(std::cout, keywords::format = format);
}
auto logDir = config.maybeValue<std::string>("log_directory");
if (logDir)
if (auto logDir = config.maybeValue<std::string>("log_directory"); logDir)
{
boost::filesystem::path dirPath{logDir.value()};
if (!boost::filesystem::exists(dirPath))
@@ -111,8 +110,7 @@ LogService::init(Config const& config)
fileSink->locked_backend()->scan_for_files();
}
// get default severity, can be overridden per channel using
// the `log_channels` array
// get default severity, can be overridden per channel using the `log_channels` array
auto defaultSeverity = config.valueOr<Severity>("log_level", Severity::NFO);
static constexpr std::array<const char*, 7> channels = {
"General",

2
src/main/Main.cpp
View File

@@ -168,7 +168,7 @@ try
auto const threads = config.valueOr("io_threads", 2);
if (threads <= 0)
{
LogService::fatal() << "io_threads is less than 0";
LogService::fatal() << "io_threads is less than 1";
return EXIT_FAILURE;
}
LogService::info() << "Number of io threads = " << threads;

4
src/rpc/Factories.cpp
View File

@@ -27,7 +27,7 @@ namespace RPC {
util::Expected<Web::Context, Status>
make_WsContext(
boost::asio::yield_context& yc,
boost::asio::yield_context yc,
boost::json::object const& request,
shared_ptr<Server::ConnectionBase> const& session,
util::TagDecoratorFactory const& tagFactory,
@@ -56,7 +56,7 @@ make_WsContext(
util::Expected<Web::Context, Status>
make_HttpContext(
boost::asio::yield_context& yc,
boost::asio::yield_context yc,
boost::json::object const& request,
util::TagDecoratorFactory const& tagFactory,
Backend::LedgerRange const& range,

4
src/rpc/Factories.h
View File

@@ -43,7 +43,7 @@ namespace RPC {
util::Expected<Web::Context, Status>
make_WsContext(
boost::asio::yield_context& yc,
boost::asio::yield_context yc,
boost::json::object const& request,
std::shared_ptr<Server::ConnectionBase> const& session,
util::TagDecoratorFactory const& tagFactory,
@@ -53,7 +53,7 @@ make_WsContext(
util::Expected<Web::Context, Status>
make_HttpContext(
boost::asio::yield_context& yc,
boost::asio::yield_context yc,
boost::json::object const& request,
util::TagDecoratorFactory const& tagFactory,
Backend::LedgerRange const& range,

26
src/rpc/RPCHelpers.cpp
View File

@@ -357,7 +357,7 @@ ledgerInfoFromRequest(std::shared_ptr<Backend::BackendInterface const> const& ba
std::variant<Status, ripple::LedgerHeader>
getLedgerInfoFromHashOrSeq(
BackendInterface const& backend,
boost::asio::yield_context& yield,
boost::asio::yield_context yield,
std::optional<std::string> ledgerHash,
std::optional<uint32_t> ledgerIndex,
uint32_t maxSeq)
@@ -432,7 +432,7 @@ traverseNFTObjects(
ripple::AccountID const& accountID,
ripple::uint256 nextPage,
std::uint32_t limit,
boost::asio::yield_context& yield,
boost::asio::yield_context yield,
std::function<void(ripple::SLE&&)> atOwnedNode)
{
auto const firstNFTPage = ripple::keylet::nftpage_min(accountID);
@@ -459,7 +459,7 @@ traverseNFTObjects(
// the object exists and the key is in right range, must be nft page
ripple::SLE pageSLE{ripple::SLE{ripple::SerialIter{page->data(), page->size()}, currentPage}};
auto count = 0;
auto count = 0u;
// traverse the nft page linked list until the start of the list or reach the limit
while (true)
{
@@ -484,7 +484,7 @@ traverseOwnedNodes(
std::uint32_t sequence,
std::uint32_t limit,
std::optional<std::string> jsonCursor,
boost::asio::yield_context& yield,
boost::asio::yield_context yield,
std::function<void(ripple::SLE&&)> atOwnedNode,
bool nftIncluded)
{
@@ -539,7 +539,7 @@ traverseOwnedNodes(
std::uint32_t const startHint,
std::uint32_t sequence,
std::uint32_t limit,
boost::asio::yield_context& yield,
boost::asio::yield_context yield,
std::function<void(ripple::SLE&&)> atOwnedNode)
{
auto cursor = AccountCursor({beast::zero, 0});
@@ -663,7 +663,7 @@ traverseOwnedNodes(
gLog.debug() << "Time loading owned entries: " << timeDiff << " milliseconds";
for (auto i = 0; i < objects.size(); ++i)
for (auto i = 0u; i < objects.size(); ++i)
{
ripple::SerialIter it{objects[i].data(), objects[i].size()};
atOwnedNode(ripple::SLE{it, keys[i]});
@@ -847,7 +847,7 @@ isGlobalFrozen(
BackendInterface const& backend,
std::uint32_t sequence,
ripple::AccountID const& issuer,
boost::asio::yield_context& yield)
boost::asio::yield_context yield)
{
if (ripple::isXRP(issuer))
return false;
@@ -871,7 +871,7 @@ isFrozen(
ripple::AccountID const& account,
ripple::Currency const& currency,
ripple::AccountID const& issuer,
boost::asio::yield_context& yield)
boost::asio::yield_context yield)
{
if (ripple::isXRP(currency))
return false;
@@ -913,7 +913,7 @@ xrpLiquid(
BackendInterface const& backend,
std::uint32_t sequence,
ripple::AccountID const& id,
boost::asio::yield_context& yield)
boost::asio::yield_context yield)
{
auto key = ripple::keylet::account(id).key;
auto blob = backend.fetchLedgerObject(key, sequence, yield);
@@ -943,7 +943,7 @@ accountFunds(
std::uint32_t const sequence,
ripple::STAmount const& amount,
ripple::AccountID const& id,
boost::asio::yield_context& yield)
boost::asio::yield_context yield)
{
if (!amount.native() && amount.getIssuer() == id)
{
@@ -963,7 +963,7 @@ accountHolds(
ripple::Currency const& currency,
ripple::AccountID const& issuer,
bool const zeroIfFrozen,
boost::asio::yield_context& yield)
boost::asio::yield_context yield)
{
ripple::STAmount amount;
if (ripple::isXRP(currency))
@@ -1006,7 +1006,7 @@ transferRate(
BackendInterface const& backend,
std::uint32_t sequence,
ripple::AccountID const& issuer,
boost::asio::yield_context& yield)
boost::asio::yield_context yield)
{
auto key = ripple::keylet::account(issuer).key;
auto blob = backend.fetchLedgerObject(key, sequence, yield);
@@ -1030,7 +1030,7 @@ postProcessOrderBook(
ripple::AccountID const& takerID,
Backend::BackendInterface const& backend,
std::uint32_t const ledgerSequence,
boost::asio::yield_context& yield)
boost::asio::yield_context yield)
{
boost::json::array jsonOffers;

20
src/rpc/RPCHelpers.h
View File

@@ -99,7 +99,7 @@ ledgerInfoFromRequest(std::shared_ptr<Backend::BackendInterface const> const& ba
std::variant<Status, ripple::LedgerHeader>
getLedgerInfoFromHashOrSeq(
BackendInterface const& backend,
boost::asio::yield_context& yield,
boost::asio::yield_context yield,
std::optional<std::string> ledgerHash,
std::optional<uint32_t> ledgerIndex,
uint32_t maxSeq);
@@ -112,7 +112,7 @@ traverseOwnedNodes(
std::uint32_t const startHint,
std::uint32_t sequence,
std::uint32_t limit,
boost::asio::yield_context& yield,
boost::asio::yield_context yield,
std::function<void(ripple::SLE&&)> atOwnedNode);
// Remove the account check from traverseOwnedNodes
@@ -124,7 +124,7 @@ traverseOwnedNodes(
std::uint32_t sequence,
std::uint32_t limit,
std::optional<std::string> jsonCursor,
boost::asio::yield_context& yield,
boost::asio::yield_context yield,
std::function<void(ripple::SLE&&)> atOwnedNode,
bool nftIncluded = false);
@@ -149,7 +149,7 @@ isGlobalFrozen(
BackendInterface const& backend,
std::uint32_t seq,
ripple::AccountID const& issuer,
boost::asio::yield_context& yield);
boost::asio::yield_context yield);
bool
isFrozen(
@@ -158,7 +158,7 @@ isFrozen(
ripple::AccountID const& account,
ripple::Currency const& currency,
ripple::AccountID const& issuer,
boost::asio::yield_context& yield);
boost::asio::yield_context yield);
ripple::STAmount
accountFunds(
@@ -166,7 +166,7 @@ accountFunds(
std::uint32_t sequence,
ripple::STAmount const& amount,
ripple::AccountID const& id,
boost::asio::yield_context& yield);
boost::asio::yield_context yield);
ripple::STAmount
accountHolds(
@@ -176,21 +176,21 @@ accountHolds(
ripple::Currency const& currency,
ripple::AccountID const& issuer,
bool zeroIfFrozen,
boost::asio::yield_context& yield);
boost::asio::yield_context yield);
ripple::Rate
transferRate(
BackendInterface const& backend,
std::uint32_t sequence,
ripple::AccountID const& issuer,
boost::asio::yield_context& yield);
boost::asio::yield_context yield);
ripple::XRPAmount
xrpLiquid(
BackendInterface const& backend,
std::uint32_t sequence,
ripple::AccountID const& id,
boost::asio::yield_context& yield);
boost::asio::yield_context yield);
boost::json::array
postProcessOrderBook(
@@ -199,7 +199,7 @@ postProcessOrderBook(
ripple::AccountID const& takerID,
Backend::BackendInterface const& backend,
std::uint32_t ledgerSequence,
boost::asio::yield_context& yield);
boost::asio::yield_context yield);
std::variant<Status, ripple::Book>
parseBook(ripple::Currency pays, ripple::AccountID payIssuer, ripple::Currency gets, ripple::AccountID getIssuer);

9
src/rpc/WorkQueue.cpp
View File

@@ -19,18 +19,13 @@
#include <rpc/WorkQueue.h>
WorkQueue::WorkQueue(std::uint32_t numWorkers, uint32_t maxSize)
WorkQueue::WorkQueue(std::uint32_t numWorkers, uint32_t maxSize) : ioc_{numWorkers}
{
if (maxSize != 0)
maxSize_ = maxSize;
while (--numWorkers)
threads_.emplace_back([this] { ioc_.run(); });
}
WorkQueue::~WorkQueue()
{
work_.reset();
for (auto& thread : threads_)
thread.join();
ioc_.join();
}

30
src/rpc/WorkQueue.h
View File

@@ -40,11 +40,9 @@ class WorkQueue
std::atomic_uint64_t curSize_ = 0;
uint32_t maxSize_ = std::numeric_limits<uint32_t>::max();
clio::Logger log_{"RPC"};
std::vector<std::thread> threads_ = {};
boost::asio::io_context ioc_ = {};
std::optional<boost::asio::io_context::work> work_{ioc_};
clio::Logger log_{"RPC"};
boost::asio::thread_pool ioc_;
public:
WorkQueue(std::uint32_t numWorkers, uint32_t maxSize = 0);
@@ -68,28 +66,26 @@ public:
{
if (curSize_ >= maxSize_ && !isWhiteListed)
{
log_.warn() << "Queue is full. rejecting job. current size = " << curSize_ << " max size = " << maxSize_;
log_.warn() << "Queue is full. rejecting job. current size = " << curSize_ << "; max size = " << maxSize_;
return false;
}
++curSize_;
auto start = std::chrono::system_clock::now();
// Each time we enqueue a job, we want to post a symmetrical job that will dequeue and run the job at the front
// of the job queue.
boost::asio::spawn(ioc_, [this, f = std::move(f), start](auto yield) mutable {
auto const run = std::chrono::system_clock::now();
auto const wait = std::chrono::duration_cast<std::chrono::microseconds>(run - start).count();
boost::asio::spawn(
ioc_, [this, f = std::forward<F>(f), start = std::chrono::system_clock::now()](auto yield) mutable {
auto const run = std::chrono::system_clock::now();
auto const wait = std::chrono::duration_cast<std::chrono::microseconds>(run - start).count();
// increment queued_ here, in the same place we implement durationUs_
++queued_;
durationUs_ += wait;
log_.info() << "WorkQueue wait time = " << wait << " queue size = " << curSize_;
++queued_;
durationUs_ += wait;
log_.info() << "WorkQueue wait time = " << wait << " queue size = " << curSize_;
f(yield);
--curSize_;
});
f(yield);
--curSize_;
});
return true;
}

4
src/rpc/common/Types.h
View File

@@ -72,9 +72,7 @@ struct VoidOutput
struct Context
{
// TODO: we shall change yield_context to const yield_context after we
// update backend interfaces to use const& yield
std::reference_wrapper<boost::asio::yield_context> yield;
boost::asio::yield_context yield;
std::shared_ptr<Server::ConnectionBase> session;
bool isAdmin = false;
std::string clientIp;

2
src/rpc/handlers/NFTOffersCommon.cpp
View File

@@ -58,7 +58,7 @@ NFTOffersHandlerBase::iterateOfferDirectory(
Input input,
ripple::uint256 const& tokenID,
ripple::Keylet const& directory,
boost::asio::yield_context& yield) const
boost::asio::yield_context yield) const
{
auto const range = sharedPtrBackend_->fetchLedgerRange();
auto const lgrInfoOrStatus =

2
src/rpc/handlers/NFTOffersCommon.h
View File

@@ -83,7 +83,7 @@ protected:
Input input,
ripple::uint256 const& tokenID,
ripple::Keylet const& directory,
boost::asio::yield_context& yield) const;
boost::asio::yield_context yield) const;
private:
friend void

4
src/rpc/handlers/Subscribe.h
View File

@@ -141,7 +141,7 @@ public:
private:
boost::json::object
subscribeToStreams(
boost::asio::yield_context& yield,
boost::asio::yield_context yield,
std::vector<std::string> const& streams,
std::shared_ptr<Server::ConnectionBase> const& session) const
{
@@ -194,7 +194,7 @@ private:
subscribeToBooks(
std::vector<OrderBook> const& books,
std::shared_ptr<Server::ConnectionBase> const& session,
boost::asio::yield_context& yield,
boost::asio::yield_context yield,
Output& output) const
{
static auto constexpr fetchLimit = 200;

4
src/subscriptions/SubscriptionManager.cpp
View File

@@ -63,7 +63,7 @@ getLedgerPubMessage(
}
boost::json::object
SubscriptionManager::subLedger(boost::asio::yield_context& yield, SessionPtrType session)
SubscriptionManager::subLedger(boost::asio::yield_context yield, SessionPtrType session)
{
subscribeHelper(session, ledgerSubscribers_, [this](SessionPtrType session) { unsubLedger(session); });
@@ -184,7 +184,7 @@ SubscriptionManager::pubTransaction(Backend::TransactionAndMetadata const& blobs
{
ripple::STAmount ownerFunds;
auto fetchFundsSynchronous = [&]() {
Backend::synchronous([&](boost::asio::yield_context& yield) {
Backend::synchronous([&](boost::asio::yield_context yield) {
ownerFunds = RPC::accountFunds(*backend_, lgrInfo.seq, amount, account, yield);
});
};

6
src/subscriptions/SubscriptionManager.h
View File

@@ -41,7 +41,7 @@ public:
Subscription(Subscription&) = delete;
Subscription(Subscription&&) = delete;
explicit Subscription(boost::asio::io_context& ioc) : strand_(ioc.get_executor())
explicit Subscription(boost::asio::io_context& ioc) : strand_(boost::asio::make_strand(ioc))
{
}
@@ -83,7 +83,7 @@ public:
SubscriptionMap(SubscriptionMap&) = delete;
SubscriptionMap(SubscriptionMap&&) = delete;
explicit SubscriptionMap(boost::asio::io_context& ioc) : strand_(ioc.get_executor())
explicit SubscriptionMap(boost::asio::io_context& ioc) : strand_(boost::asio::make_strand(ioc))
{
}
@@ -251,7 +251,7 @@ public:
}
boost::json::object
subLedger(boost::asio::yield_context& yield, SessionPtrType session);
subLedger(boost::asio::yield_context yield, SessionPtrType session);
void
pubLedger(

6
src/webserver/Context.h
View File

@@ -34,7 +34,7 @@ namespace Web {
struct Context : util::Taggable
{
std::reference_wrapper<boost::asio::yield_context> yield;
boost::asio::yield_context yield;
std::string method;
std::uint32_t apiVersion;
boost::json::object params;
@@ -43,7 +43,7 @@ struct Context : util::Taggable
std::string clientIp;
Context(
boost::asio::yield_context& yield,
boost::asio::yield_context yield,
std::string const& command,
std::uint32_t apiVersion,
boost::json::object params,
@@ -52,7 +52,7 @@ struct Context : util::Taggable
Backend::LedgerRange const& range,
std::string const& clientIp)
: Taggable(tagFactory)
, yield(std::ref(yield))
, yield(yield)
, method(command)
, apiVersion(apiVersion)
, params(std::move(params))

15
src/webserver/DOSGuard.h
View File

@@ -117,8 +117,8 @@ public:
auto [transferedByte, requests] = ipState_.at(ip);
if (transferedByte > maxFetches_ || requests > maxRequestCount_)
{
log_.warn() << "Dosguard:Client surpassed the rate limit. ip = " << ip
<< " Transfered Byte:" << transferedByte << " Requests:" << requests;
log_.warn() << "Dosguard: Client surpassed the rate limit. ip = " << ip
<< " Transfered Byte: " << transferedByte << "; Requests: " << requests;
return false;
}
}
@@ -127,8 +127,8 @@ public:
{
if (it->second > maxConnCount_)
{
log_.warn() << "Dosguard:Client surpassed the rate limit. ip = " << ip
<< " Concurrent connection:" << it->second;
log_.warn() << "Dosguard: Client surpassed the rate limit. ip = " << ip
<< " Concurrent connection: " << it->second;
return false;
}
}
@@ -249,9 +249,9 @@ class IntervalSweepHandler
{
std::chrono::milliseconds sweepInterval_;
std::reference_wrapper<boost::asio::io_context> ctx_;
BaseDOSGuard* dosGuard_ = nullptr;
boost::asio::steady_timer timer_;
boost::asio::steady_timer timer_{boost::asio::make_strand(ctx_.get())};
BaseDOSGuard* dosGuard_ = nullptr;
public:
/**
@@ -263,6 +263,7 @@ public:
IntervalSweepHandler(clio::Config const& config, boost::asio::io_context& ctx)
: sweepInterval_{std::max(1u, static_cast<uint32_t>(config.valueOr("dos_guard.sweep_interval", 1.0) * 1000.0))}
, ctx_{std::ref(ctx)}
, timer_{ctx.get_executor()}
{
}
@@ -297,7 +298,7 @@ private:
return;
dosGuard_->clear();
createTimer();
boost::asio::post(ctx_.get().get_executor(), [this] { createTimer(); });
});
}
};

8
src/webserver/HttpSession.h
View File

@@ -59,12 +59,6 @@ public:
return stream_;
}
boost::beast::tcp_stream
releaseStream()
{
return std::move(stream_);
}
void
run()
{
@@ -76,10 +70,8 @@ public:
void
doClose()
{
// Send a TCP shutdown
boost::beast::error_code ec;
stream_.socket().shutdown(tcp::socket::shutdown_send, ec);
// At this point the connection is closed gracefully
}
void

12
src/webserver/RPCServerHandler.h
View File

@@ -81,8 +81,8 @@ public:
req["params"] = boost::json::array({boost::json::object{}});
if (!rpcEngine_->post(
[request = std::move(req), connection, this](boost::asio::yield_context yc) mutable {
handleRequest(yc, std::move(request), connection);
[this, request = std::move(req), connection](boost::asio::yield_context yield) mutable {
handleRequest(yield, std::move(request), connection);
},
connection->clientIp))
{
@@ -126,9 +126,9 @@ public:
private:
void
handleRequest(
boost::asio::yield_context& yc,
boost::asio::yield_context yield,
boost::json::object&& request,
std::shared_ptr<Server::ConnectionBase> connection)
std::shared_ptr<Server::ConnectionBase> const& connection)
{
log_.info() << connection->tag() << (connection->upgraded ? "ws" : "http")
<< " received request from work queue: " << util::removeSecret(request)
@@ -147,7 +147,7 @@ private:
auto const context = [&] {
if (connection->upgraded)
return RPC::make_WsContext(
yc,
yield,
request,
connection,
tagFactory_.with(connection->tag()),
@@ -156,7 +156,7 @@ private:
std::cref(apiVersionParser_));
else
return RPC::make_HttpContext(
yc,
yield,
request,
tagFactory_.with(connection->tag()),
*range,

31
src/webserver/Server.h
View File

@@ -41,7 +41,6 @@ class Detector : public std::enable_shared_from_this<Detector<PlainSession, SslS
using std::enable_shared_from_this<Detector<PlainSession, SslSession, Handler>>::shared_from_this;
clio::Logger log_{"WebServer"};
std::reference_wrapper<boost::asio::io_context> ioc_;
boost::beast::tcp_stream stream_;
std::optional<std::reference_wrapper<boost::asio::ssl::context>> ctx_;
std::reference_wrapper<util::TagDecoratorFactory const> tagFactory_;
@@ -51,14 +50,12 @@ class Detector : public std::enable_shared_from_this<Detector<PlainSession, SslS
public:
Detector(
std::reference_wrapper<boost::asio::io_context> ioc,
tcp::socket&& socket,
std::optional<std::reference_wrapper<boost::asio::ssl::context>> ctx,
std::reference_wrapper<util::TagDecoratorFactory const> tagFactory,
std::reference_wrapper<clio::DOSGuard> dosGuard,
std::shared_ptr<Handler> const& handler)
: ioc_(ioc)
, stream_(std::move(socket))
: stream_(std::move(socket))
, ctx_(ctx)
, tagFactory_(std::cref(tagFactory))
, dosGuard_(dosGuard)
@@ -79,7 +76,6 @@ public:
run()
{
boost::beast::get_lowest_layer(stream_).expires_after(std::chrono::seconds(30));
// Detect a TLS handshake
async_detect_ssl(stream_, buffer_, boost::beast::bind_front_handler(&Detector::onDetect, shared_from_this()));
}
@@ -89,7 +85,6 @@ public:
if (ec)
return fail(ec, "detect");
// would not create session if can not get ip
std::string ip;
try
{
@@ -103,14 +98,14 @@ public:
if (result)
{
if (!ctx_)
return fail(ec, "ssl not supported by this server");
// Launch SSL session
return fail(ec, "SSL is not supported by this server");
std::make_shared<SslSession<Handler>>(
stream_.release_socket(), ip, *ctx_, tagFactory_, dosGuard_, handler_, std::move(buffer_))
->run();
return;
}
// Launch plain session
std::make_shared<PlainSession<Handler>>(
stream_.release_socket(), ip, tagFactory_, dosGuard_, handler_, std::move(buffer_))
->run();
@@ -130,11 +125,11 @@ class Server : public std::enable_shared_from_this<Server<PlainSession, SslSessi
using std::enable_shared_from_this<Server<PlainSession, SslSession, Handler>>::shared_from_this;
clio::Logger log_{"WebServer"};
std::reference_wrapper<boost::asio::io_context> const ioc_;
std::optional<std::reference_wrapper<boost::asio::ssl::context>> const ctx_;
util::TagDecoratorFactory const tagFactory_;
std::reference_wrapper<clio::DOSGuard> const dosGuard_;
std::shared_ptr<Handler> const handler_;
std::reference_wrapper<boost::asio::io_context> ioc_;
std::optional<std::reference_wrapper<boost::asio::ssl::context>> ctx_;
util::TagDecoratorFactory tagFactory_;
std::reference_wrapper<clio::DOSGuard> dosGuard_;
std::shared_ptr<Handler> handler_;
tcp::acceptor acceptor_;
public:
@@ -150,7 +145,7 @@ public:
, tagFactory_(std::move(tagFactory))
, dosGuard_(std::ref(dosGuard))
, handler_(callback)
, acceptor_(boost::asio::make_strand(ioc.get_executor()))
, acceptor_(boost::asio::make_strand(ioc))
{
boost::beast::error_code ec;
@@ -190,7 +185,7 @@ private:
doAccept()
{
acceptor_.async_accept(
boost::asio::make_strand(ioc_.get().get_executor()),
boost::asio::make_strand(ioc_.get()),
boost::beast::bind_front_handler(&Server::onAccept, shared_from_this()));
}
@@ -201,9 +196,9 @@ private:
{
auto ctxRef =
ctx_ ? std::optional<std::reference_wrapper<boost::asio::ssl::context>>{ctx_.value()} : std::nullopt;
// Create the detector session and run it
std::make_shared<Detector<PlainSession, SslSession, Handler>>(
ioc_, std::move(socket), ctxRef, std::cref(tagFactory_), dosGuard_, handler_)
std::move(socket), ctxRef, std::cref(tagFactory_), dosGuard_, handler_)
->run();
}

6
src/webserver/SslHttpSession.h
View File

@@ -61,12 +61,6 @@ public:
return stream_;
}
boost::beast::ssl_stream<boost::beast::tcp_stream>
releaseStream()
{
return std::move(stream_);
}
void
run()
{

5
src/webserver/SslWsSession.h
View File

@@ -30,7 +30,8 @@ namespace Server {
template <ServerHandler Handler>
class SslWsSession : public WsBase<SslWsSession, Handler>
{
boost::beast::websocket::stream<boost::beast::ssl_stream<boost::beast::tcp_stream>> ws_;
using StreamType = boost::beast::websocket::stream<boost::beast::ssl_stream<boost::beast::tcp_stream>>;
StreamType ws_;
public:
explicit SslWsSession(
@@ -44,7 +45,7 @@ public:
{
}
boost::beast::websocket::stream<boost::beast::ssl_stream<boost::beast::tcp_stream>>&
StreamType&
ws()
{
return ws_;

14
src/webserver/details/HttpBase.h
View File

@@ -52,6 +52,7 @@ class HttpBase : public ConnectionBase
return static_cast<Derived<Handler>&>(*this);
}
// TODO: this should be rewritten using http::message_generator instead
struct SendLambda
{
HttpBase& self_;
@@ -154,8 +155,8 @@ public:
{
if (dead())
return;
// Make the request empty before reading,
// otherwise the operation behavior is undefined.
// Make the request empty before reading, otherwise the operation behavior is undefined.
req_ = {};
// Set the timeout.
@@ -169,10 +170,8 @@ public:
}
void
onRead(boost::beast::error_code ec, std::size_t bytes_transferred)
onRead(boost::beast::error_code ec, [[maybe_unused]] std::size_t bytes_transferred)
{
boost::ignore_unused(bytes_transferred);
if (ec == http::error::end_of_stream)
return derived().doClose();
@@ -236,7 +235,8 @@ public:
jsonResponse["warnings"].as_array().push_back(RPC::makeWarning(RPC::warnRPC_RATE_LIMIT));
else
jsonResponse["warnings"] = boost::json::array{RPC::makeWarning(RPC::warnRPC_RATE_LIMIT)};
// reserialize when we need to include this warning
// Reserialize when we need to include this warning
msg = boost::json::serialize(jsonResponse);
}
sender_(httpResponse(status, "application/json", std::move(msg)));
@@ -255,9 +255,7 @@ public:
if (close)
return derived().doClose();
// We're done with the response so delete it
res_ = nullptr;
doRead();
}

2
src/webserver/details/WsBase.h
View File

@@ -162,7 +162,7 @@ public:
else
jsonResponse["warnings"] = boost::json::array{RPC::makeWarning(RPC::warnRPC_RATE_LIMIT)};
// reserialize when we need to include this warning
// Reserialize when we need to include this warning
msg = boost::json::serialize(jsonResponse);
}
auto sharedMsg = std::make_shared<std::string>(std::move(msg));