ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2026-04-29 15:37:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move to clang-format-16 (#908)

Browse Source 
Fixes #848
This commit is contained in:
Alex Kremer
2023-10-19 16:55:04 +01:00
committed by GitHub
parent 5de87b9ef8
commit 1aab2b94b1
265 changed files with 3950 additions and 4743 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
.clang-format
View File

@@ -1,7 +1,7 @@
---
Language: Cpp
AccessModifierOffset: -4
AlignAfterOpenBracket: AlwaysBreak
AlignAfterOpenBracket: BlockIndent
AlignConsecutiveAssignments: false
AlignConsecutiveDeclarations: false
AlignEscapedNewlinesLeft: true
@@ -18,20 +18,8 @@ AlwaysBreakBeforeMultilineStrings: true
AlwaysBreakTemplateDeclarations: true
BinPackArguments: false
BinPackParameters: false
BraceWrapping:
AfterClass: true
AfterControlStatement: true
AfterEnum: false
AfterFunction: true
AfterNamespace: false
AfterObjCDeclaration: true
AfterStruct: true
AfterUnion: true
BeforeCatch: true
BeforeElse: true
IndentBraces: false
BreakBeforeBinaryOperators: false
BreakBeforeBraces: Custom
BreakBeforeBraces: WebKit
BreakBeforeTernaryOperators: true
BreakConstructorInitializersBeforeComma: true
ColumnLimit: 120
@@ -43,6 +31,7 @@ Cpp11BracedListStyle: true
DerivePointerAlignment: false
DisableFormat: false
ExperimentalAutoDetectBinPacking: false
FixNamespaceComments: true
ForEachMacros: [ Q_FOREACH, BOOST_FOREACH ]
IncludeCategories:
- Regex: '^<(BeastConfig)'
@@ -58,6 +47,8 @@ IndentCaseLabels: true
IndentFunctionDeclarationAfterType: false
IndentWidth: 4
IndentWrappedFunctionNames: false
IndentRequiresClause: true
RequiresClausePosition: OwnLine
KeepEmptyLinesAtTheStartOfBlocks: false
MaxEmptyLinesToKeep: 1
NamespaceIndentation: None
@@ -70,6 +61,7 @@ PenaltyBreakString: 1000
PenaltyExcessCharacter: 1000000
PenaltyReturnTypeOnItsOwnLine: 200
PointerAlignment: Left
QualifierAlignment: Right
ReflowComments: true
SortIncludes: true
SpaceAfterCStyleCast: false

16
.githooks/pre-commit
View File

@@ -4,7 +4,21 @@ exec 1>&2
# paths to check and re-format
sources="src unittests"
formatter="clang-format-11 -i"
formatter="clang-format -i"
version=$($formatter --version | awk -F'[[:space:]]' '{print $4}')
if [[ "16.0.0" > "$version" ]]; then
cat <<\EOF
ERROR
-----------------------------------------------------------------------------
A minimum of version 16 of `clang-format` is required.
Please fix paths and run again.
-----------------------------------------------------------------------------
EOF
exit 2
fi
first=$(git diff $sources)
find $sources -type f \( -name '*.cpp' -o -name '*.h' -o -name '*.ipp' \) -print0 | xargs -0 $formatter

20
.github/actions/clang_format/action.yml vendored
View File

@@ -1,11 +1,23 @@
name: Check format
description: Check format using clang-format-11
description: Check format using clang-format-16
runs:
using: composite
steps:
# Github's ubuntu-20.04 image already has clang-format-11 installed
- run: |
find src unittests -type f \( -name '*.cpp' -o -name '*.h' -o -name '*.ipp' \) -print0 | xargs -0 clang-format-11 -i
- name: Add llvm repo
run: |
echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-16 main' | sudo tee -a /etc/apt/sources.list
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
shell: bash
- name: Install packages
run: |
sudo apt update -qq
sudo apt install -y jq clang-format-16
shell: bash
- name: Run formatter
run: |
find src unittests -type f \( -name '*.cpp' -o -name '*.h' -o -name '*.ipp' \) -print0 | xargs -0 clang-format-16 -i
shell: bash
- name: Check for differences

2
CONTRIBUTING.md
View File

@@ -91,7 +91,7 @@ The button for that is near the bottom of the PR's page on GitHub.
This is a non-exhaustive list of recommended style guidelines. These are not always strictly enforced and serve as a way to keep the codebase coherent.
## Formatting
Code must conform to `clang-format` version 10, unless the result would be unreasonably difficult to read or maintain.
Code must conform to `clang-format` version 16, unless the result would be unreasonably difficult to read or maintain.
To change your code to conform use `clang-format -i <your changed files>`.
## Avoid

4
src/data/BackendCounters.cpp
View File

@@ -24,9 +24,7 @@ namespace data {
BackendCounters::PtrType
BackendCounters::make()
{
struct EnableMakeShared : public BackendCounters
{
};
struct EnableMakeShared : public BackendCounters {};
return std::make_shared<EnableMakeShared>();
}

6
src/data/BackendCounters.h
View File

@@ -52,8 +52,7 @@ concept SomeBackendCounters = requires(T a) {
* @brief Holds statistics about the backend.
* @note This class is thread-safe.
*/
class BackendCounters
{
class BackendCounters {
public:
using PtrType = std::shared_ptr<BackendCounters>;
@@ -96,8 +95,7 @@ public:
private:
BackendCounters() = default;
class AsyncOperationCounters
{
class AsyncOperationCounters {
public:
AsyncOperationCounters(std::string name);

6
src/data/BackendFactory.h
View File

@@ -46,8 +46,7 @@ make_Backend(util::Config const& config)
std::shared_ptr<BackendInterface> backend = nullptr;
// TODO: retire `cassandra-new` by next release after 2.0
if (boost::iequals(type, "cassandra") or boost::iequals(type, "cassandra-new"))
{
if (boost::iequals(type, "cassandra") or boost::iequals(type, "cassandra-new")) {
auto cfg = config.section("database." + type);
backend = std::make_shared<data::cassandra::CassandraBackend>(data::cassandra::SettingsProvider{cfg}, readOnly);
}
@@ -56,8 +55,7 @@ make_Backend(util::Config const& config)
throw std::runtime_error("Invalid database type");
auto const rng = backend->hardFetchLedgerRangeNoThrow();
if (rng)
{
if (rng) {
backend->updateRange(rng->minSequence);
backend->updateRange(rng->maxSequence);
}

108
src/data/BackendInterface.cpp
View File

@@ -34,8 +34,7 @@ BackendInterface::finishWrites(std::uint32_t const ledgerSequence)
{
LOG(gLog.debug()) << "Want finish writes for " << ledgerSequence;
auto commitRes = doFinishWrites();
if (commitRes)
{
if (commitRes) {
LOG(gLog.debug()) << "Successfully commited. Updating range now to " << ledgerSequence;
updateRange(ledgerSequence);
}
@@ -59,23 +58,20 @@ std::optional<Blob>
BackendInterface::fetchLedgerObject(
ripple::uint256 const& key,
std::uint32_t const sequence,
boost::asio::yield_context yield) const
boost::asio::yield_context yield
) const
{
auto obj = cache_.get(key, sequence);
if (obj)
{
if (obj) {
LOG(gLog.trace()) << "Cache hit - " << ripple::strHex(key);
return *obj;
}
LOG(gLog.trace()) << "Cache miss - " << ripple::strHex(key);
auto dbObj = doFetchLedgerObject(key, sequence, yield);
if (!dbObj)
{
if (!dbObj) {
LOG(gLog.trace()) << "Missed cache and missed in db";
}
else
{
} else {
LOG(gLog.trace()) << "Missed cache but found in db";
}
return dbObj;
@@ -85,32 +81,26 @@ std::vector<Blob>
BackendInterface::fetchLedgerObjects(
std::vector<ripple::uint256> const& keys,
std::uint32_t const sequence,
boost::asio::yield_context yield) const
boost::asio::yield_context yield
) const
{
std::vector<Blob> results;
results.resize(keys.size());
std::vector<ripple::uint256> misses;
for (size_t i = 0; i < keys.size(); ++i)
{
for (size_t i = 0; i < keys.size(); ++i) {
auto obj = cache_.get(keys[i], sequence);
if (obj)
{
if (obj) {
results[i] = *obj;
}
else
{
} else {
misses.push_back(keys[i]);
}
}
LOG(gLog.trace()) << "Cache hits = " << keys.size() - misses.size() << " - cache misses = " << misses.size();
if (!misses.empty())
{
if (!misses.empty()) {
auto objs = doFetchLedgerObjects(misses, sequence, yield);
for (size_t i = 0, j = 0; i < results.size(); ++i)
{
if (results[i].empty())
{
for (size_t i = 0, j = 0; i < results.size(); ++i) {
if (results[i].empty()) {
results[i] = objs[j];
++j;
}
@@ -124,15 +114,13 @@ std::optional<ripple::uint256>
BackendInterface::fetchSuccessorKey(
ripple::uint256 key,
std::uint32_t const ledgerSequence,
boost::asio::yield_context yield) const
boost::asio::yield_context yield
) const
{
auto succ = cache_.getSuccessor(key, ledgerSequence);
if (succ)
{
if (succ) {
LOG(gLog.trace()) << "Cache hit - " << ripple::strHex(key);
}
else
{
} else {
LOG(gLog.trace()) << "Cache miss - " << ripple::strHex(key);
}
return succ ? succ->key : doFetchSuccessorKey(key, ledgerSequence, yield);
@@ -142,11 +130,11 @@ std::optional<LedgerObject>
BackendInterface::fetchSuccessorObject(
ripple::uint256 key,
std::uint32_t const ledgerSequence,
boost::asio::yield_context yield) const
boost::asio::yield_context yield
) const
{
auto succ = fetchSuccessorKey(key, ledgerSequence, yield);
if (succ)
{
if (succ) {
auto obj = fetchLedgerObject(*succ, ledgerSequence, yield);
if (!obj)
return {{*succ, {}}};
@@ -161,7 +149,8 @@ BackendInterface::fetchBookOffers(
ripple::uint256 const& book,
std::uint32_t const ledgerSequence,
std::uint32_t const limit,
boost::asio::yield_context yield) const
boost::asio::yield_context yield
) const
{
// TODO try to speed this up. This can take a few seconds. The goal is
// to get it down to a few hundred milliseconds.
@@ -175,29 +164,25 @@ BackendInterface::fetchBookOffers(
std::uint32_t numPages = 0;
long succMillis = 0;
long pageMillis = 0;
while (keys.size() < limit)
{
while (keys.size() < limit) {
auto mid1 = std::chrono::system_clock::now();
auto offerDir = fetchSuccessorObject(uTipIndex, ledgerSequence, yield);
auto mid2 = std::chrono::system_clock::now();
numSucc++;
succMillis += getMillis(mid2 - mid1);
if (!offerDir || offerDir->key >= bookEnd)
{
if (!offerDir || offerDir->key >= bookEnd) {
LOG(gLog.trace()) << "offerDir.has_value() " << offerDir.has_value() << " breaking";
break;
}
uTipIndex = offerDir->key;
while (keys.size() < limit)
{
while (keys.size() < limit) {
++numPages;
ripple::STLedgerEntry const sle{
ripple::SerialIter{offerDir->blob.data(), offerDir->blob.size()}, offerDir->key};
auto indexes = sle.getFieldV256(ripple::sfIndexes);
keys.insert(keys.end(), indexes.begin(), indexes.end());
auto next = sle.getFieldU64(ripple::sfIndexNext);
if (next == 0u)
{
if (next == 0u) {
LOG(gLog.trace()) << "Next is empty. breaking";
break;
}
@@ -212,8 +197,7 @@ BackendInterface::fetchBookOffers(
}
auto mid = std::chrono::system_clock::now();
auto objs = fetchLedgerObjects(keys, ledgerSequence, yield);
for (size_t i = 0; i < keys.size() && i < limit; ++i)
{
for (size_t i = 0; i < keys.size() && i < limit; ++i) {
LOG(gLog.trace()) << "Key = " << ripple::strHex(keys[i]) << " blob = " << ripple::strHex(objs[i])
<< " ledgerSequence = " << ledgerSequence;
assert(objs[i].size());
@@ -251,12 +235,9 @@ BackendInterface::updateRange(uint32_t newMax)
{
std::scoped_lock const lck(rngMtx_);
assert(!range || newMax >= range->maxSequence);
if (!range)
{
if (!range) {
range = {newMax, newMax};
}
else
{
} else {
range->maxSequence = newMax;
}
}
@@ -267,41 +248,33 @@ BackendInterface::fetchLedgerPage(
std::uint32_t const ledgerSequence,
std::uint32_t const limit,
bool outOfOrder,
boost::asio::yield_context yield) const
boost::asio::yield_context yield
) const
{
LedgerPage page;
std::vector<ripple::uint256> keys;
bool reachedEnd = false;
while (keys.size() < limit && !reachedEnd)
{
while (keys.size() < limit && !reachedEnd) {
ripple::uint256 const& curCursor = !keys.empty() ? keys.back() : (cursor ? *cursor : firstKey);
std::uint32_t const seq = outOfOrder ? range->maxSequence : ledgerSequence;
auto succ = fetchSuccessorKey(curCursor, seq, yield);
if (!succ)
{
if (!succ) {
reachedEnd = true;
}
else
{
} else {
keys.push_back(*succ);
}
}
auto objects = fetchLedgerObjects(keys, ledgerSequence, yield);
for (size_t i = 0; i < objects.size(); ++i)
{
if (!objects[i].empty())
{
for (size_t i = 0; i < objects.size(); ++i) {
if (!objects[i].empty()) {
page.objects.push_back({keys[i], std::move(objects[i])});
}
else if (!outOfOrder)
{
} else if (!outOfOrder) {
LOG(gLog.error()) << "Deleted or non-existent object in successor table. key = " << ripple::strHex(keys[i])
<< " - seq = " << ledgerSequence;
std::stringstream msg;
for (size_t j = 0; j < objects.size(); ++j)
{
for (size_t j = 0; j < objects.size(); ++j) {
msg << " - " << ripple::strHex(keys[j]);
}
LOG(gLog.error()) << msg.str();
@@ -321,8 +294,7 @@ BackendInterface::fetchFees(std::uint32_t const seq, boost::asio::yield_context
auto key = ripple::keylet::fees().key;
auto bytes = fetchLedgerObject(key, seq, yield);
if (!bytes)
{
if (!bytes) {
LOG(gLog.error()) << "Could not find fees";
return {};
}

51
src/data/BackendInterface.h
View File

@@ -38,10 +38,9 @@ namespace data {
/**
* @brief Represents a database timeout error.
*/
class DatabaseTimeout : public std::exception
{
class DatabaseTimeout : public std::exception {
public:
const char*
char const*
what() const throw() override
{
return "Database read timed out. Please retry the request";
@@ -63,14 +62,10 @@ retryOnTimeout(FnType func, size_t waitMs = DEFAULT_WAIT_BETWEEN_RETRY)
{
static util::Logger const log{"Backend"};
while (true)
{
try
{
while (true) {
try {
return func();
}
catch (DatabaseTimeout const&)
{
} catch (DatabaseTimeout const&) {
LOG(log.error()) << "Database request timed out. Sleeping and retrying ... ";
std::this_thread::sleep_for(std::chrono::milliseconds(waitMs));
}
@@ -91,17 +86,15 @@ synchronous(FnType&& func)
boost::asio::io_context ctx;
using R = typename boost::result_of<FnType(boost::asio::yield_context)>::type;
if constexpr (!std::is_same<R, void>::value)
{
if constexpr (!std::is_same<R, void>::value) {
R res;
boost::asio::spawn(
ctx, [_ = boost::asio::make_work_guard(ctx), &func, &res](auto yield) { res = func(yield); });
boost::asio::spawn(ctx, [_ = boost::asio::make_work_guard(ctx), &func, &res](auto yield) {
res = func(yield);
});
ctx.run();
return res;
}
else
{
} else {
boost::asio::spawn(ctx, [_ = boost::asio::make_work_guard(ctx), &func](auto yield) { func(yield); });
ctx.run();
}
@@ -124,8 +117,7 @@ synchronousAndRetryOnTimeout(FnType&& func)
/**
* @brief The interface to the database used by Clio.
*/
class BackendInterface
{
class BackendInterface {
protected:
mutable std::shared_mutex rngMtx_;
std::optional<LedgerRange> range;
@@ -245,7 +237,8 @@ public:
std::uint32_t limit,
bool forward,
std::optional<TransactionsCursor> const& cursor,
boost::asio::yield_context yield) const = 0;
boost::asio::yield_context yield
) const = 0;
/**
* @brief Fetches all transactions from a specific ledger.
@@ -294,7 +287,8 @@ public:
std::uint32_t limit,
bool forward,
std::optional<TransactionsCursor> const& cursorIn,
boost::asio::yield_context yield) const = 0;
boost::asio::yield_context yield
) const = 0;
/**
* @brief Fetches a specific ledger object.
@@ -325,7 +319,8 @@ public:
fetchLedgerObjects(
std::vector<ripple::uint256> const& keys,
std::uint32_t sequence,
boost::asio::yield_context yield) const;
boost::asio::yield_context yield
) const;
/**
* @brief The database-specific implementation for fetching a ledger object.
@@ -350,7 +345,8 @@ public:
doFetchLedgerObjects(
std::vector<ripple::uint256> const& keys,
std::uint32_t sequence,
boost::asio::yield_context yield) const = 0;
boost::asio::yield_context yield
) const = 0;
/**
* @brief Returns the difference between ledgers.
@@ -378,7 +374,8 @@ public:
std::uint32_t ledgerSequence,
std::uint32_t limit,
bool outOfOrder,
boost::asio::yield_context yield) const;
boost::asio::yield_context yield
) const;
/**
* @brief Fetches the successor object.
@@ -430,7 +427,8 @@ public:
ripple::uint256 const& book,
std::uint32_t ledgerSequence,
std::uint32_t limit,
boost::asio::yield_context yield) const;
boost::asio::yield_context yield
) const;
/**
* @brief Synchronously fetches the ledger range from DB.
@@ -492,7 +490,8 @@ public:
std::uint32_t seq,
std::uint32_t date,
std::string&& transaction,
std::string&& metadata) = 0;
std::string&& metadata
) = 0;
/**
* @brief Writes NFTs to the database.

243
src/data/CassandraBackend.h
View File

@@ -44,8 +44,7 @@ namespace data::cassandra {
* @tparam ExecutionStrategyType The execution strategy type to use
*/
template <SomeSettingsProvider SettingsProviderType, SomeExecutionStrategy ExecutionStrategyType>
class BasicCassandraBackend : public BackendInterface
{
class BasicCassandraBackend : public BackendInterface {
util::Logger log_{"Backend"};
SettingsProviderType settingsProvider_;
@@ -73,12 +72,10 @@ public:
if (auto const res = handle_.connect(); not res)
throw std::runtime_error("Could not connect to Cassandra: " + res.error());
if (not readOnly)
{
if (auto const res = handle_.execute(schema_.createKeyspace); not res)
{
// on datastax, creation of keyspaces can be configured to only be done thru the admin interface.
// this does not mean that the keyspace does not already exist tho.
if (not readOnly) {
if (auto const res = handle_.execute(schema_.createKeyspace); not res) {
// on datastax, creation of keyspaces can be configured to only be done thru the admin
// interface. this does not mean that the keyspace does not already exist tho.
if (res.error().code() != CASS_ERROR_SERVER_UNAUTHORIZED)
throw std::runtime_error("Could not create keyspace: " + res.error());
}
@@ -87,12 +84,9 @@ public:
throw std::runtime_error("Could not create schema: " + res.error());
}
try
{
try {
schema_.prepareStatements(handle_);
}
catch (std::runtime_error const& ex)
{
} catch (std::runtime_error const& ex) {
LOG(log_.error()) << "Failed to prepare the statements: " << ex.what() << "; readOnly: " << readOnly;
throw;
}
@@ -106,7 +100,8 @@ public:
std::uint32_t const limit,
bool forward,
std::optional<TransactionsCursor> const& cursorIn,
boost::asio::yield_context yield) const override
boost::asio::yield_context yield
) const override
{
auto rng = fetchLedgerRange();
if (!rng)
@@ -120,14 +115,11 @@ public:
}();
auto cursor = cursorIn;
if (cursor)
{
if (cursor) {
statement.bindAt(1, cursor->asTuple());
LOG(log_.debug()) << "account = " << ripple::strHex(account) << " tuple = " << cursor->ledgerSequence
<< cursor->transactionIndex;
}
else
{
} else {
auto const seq = forward ? rng->minSequence : rng->maxSequence;
auto const placeHolder = forward ? 0u : std::numeric_limits<std::uint32_t>::max();
@@ -142,8 +134,7 @@ public:
statement.bindAt(2, Limit{limit});
auto const res = executor_.read(yield, statement);
auto const& results = res.value();
if (not results.hasRows())
{
if (not results.hasRows()) {
LOG(log_.debug()) << "No rows returned";
return {};
}
@@ -152,11 +143,9 @@ public:
auto numRows = results.numRows();
LOG(log_.info()) << "num_rows = " << numRows;
for (auto [hash, data] : extract<ripple::uint256, std::tuple<uint32_t, uint32_t>>(results))
{
for (auto [hash, data] : extract<ripple::uint256, std::tuple<uint32_t, uint32_t>>(results)) {
hashes.push_back(hash);
if (--numRows == 0)
{
if (--numRows == 0) {
LOG(log_.debug()) << "Setting cursor";
cursor = data;
@@ -170,8 +159,7 @@ public:
auto const txns = fetchTransactions(hashes, yield);
LOG(log_.debug()) << "Txns = " << txns.size();
if (txns.size() == limit)
{
if (txns.size() == limit) {
LOG(log_.debug()) << "Returning cursor";
return {txns, cursor};
}
@@ -185,13 +173,11 @@ public:
// wait for other threads to finish their writes
executor_.sync();
if (!range)
{
if (!range) {
executor_.writeSync(schema_->updateLedgerRange, ledgerSequence_, false, ledgerSequence_);
}
if (not executeSyncUpdate(schema_->updateLedgerRange.bind(ledgerSequence_, true, ledgerSequence_ - 1)))
{
if (not executeSyncUpdate(schema_->updateLedgerRange.bind(ledgerSequence_, true, ledgerSequence_ - 1))) {
LOG(log_.warn()) << "Update failed for ledger " << ledgerSequence_;
return false;
}
@@ -213,10 +199,8 @@ public:
std::optional<std::uint32_t>
fetchLatestLedgerSequence(boost::asio::yield_context yield) const override
{
if (auto const res = executor_.read(yield, schema_->selectLatestLedger); res)
{
if (auto const& result = res.value(); result)
{
if (auto const res = executor_.read(yield, schema_->selectLatestLedger); res) {
if (auto const& result = res.value(); result) {
if (auto const maybeValue = result.template get<uint32_t>(); maybeValue)
return maybeValue;
@@ -225,9 +209,7 @@ public:
}
LOG(log_.error()) << "Could not fetch latest ledger - no result";
}
else
{
} else {
LOG(log_.error()) << "Could not fetch latest ledger: " << res.error();
}
@@ -238,12 +220,9 @@ public:
fetchLedgerBySequence(std::uint32_t const sequence, boost::asio::yield_context yield) const override
{
auto const res = executor_.read(yield, schema_->selectLedgerBySeq, sequence);
if (res)
{
if (auto const& result = res.value(); result)
{
if (auto const maybeValue = result.template get<std::vector<unsigned char>>(); maybeValue)
{
if (res) {
if (auto const& result = res.value(); result) {
if (auto const maybeValue = result.template get<std::vector<unsigned char>>(); maybeValue) {
return util::deserializeHeader(ripple::makeSlice(*maybeValue));
}
@@ -252,9 +231,7 @@ public:
}
LOG(log_.error()) << "Could not fetch ledger by sequence - no result";
}
else
{
} else {
LOG(log_.error()) << "Could not fetch ledger by sequence: " << res.error();
}
@@ -264,10 +241,8 @@ public:
std::optional<ripple::LedgerHeader>
fetchLedgerByHash(ripple::uint256 const& hash, boost::asio::yield_context yield) const override
{
if (auto const res = executor_.read(yield, schema_->selectLedgerByHash, hash); res)
{
if (auto const& result = res.value(); result)
{
if (auto const res = executor_.read(yield, schema_->selectLedgerByHash, hash); res) {
if (auto const& result = res.value(); result) {
if (auto const maybeValue = result.template get<uint32_t>(); maybeValue)
return fetchLedgerBySequence(*maybeValue, yield);
@@ -276,9 +251,7 @@ public:
}
LOG(log_.error()) << "Could not fetch ledger by hash - no result";
}
else
{
} else {
LOG(log_.error()) << "Could not fetch ledger by hash: " << res.error();
}
@@ -289,11 +262,9 @@ public:
hardFetchLedgerRange(boost::asio::yield_context yield) const override
{
auto const res = executor_.read(yield, schema_->selectLedgerRange);
if (res)
{
if (res) {
auto const& results = res.value();
if (not results.hasRows())
{
if (not results.hasRows()) {
LOG(log_.debug()) << "Could not fetch ledger range - no rows";
return std::nullopt;
}
@@ -303,14 +274,10 @@ public:
// least use tuple<int, int>?
LedgerRange range;
std::size_t idx = 0;
for (auto [seq] : extract<uint32_t>(results))
{
if (idx == 0)
{
for (auto [seq] : extract<uint32_t>(results)) {
if (idx == 0) {
range.maxSequence = range.minSequence = seq;
}
else if (idx == 1)
{
} else if (idx == 1) {
range.maxSequence = seq;
}
@@ -343,15 +310,13 @@ public:
auto start = std::chrono::system_clock::now();
auto const res = executor_.read(yield, schema_->selectAllTransactionHashesInLedger, ledgerSequence);
if (not res)
{
if (not res) {
LOG(log_.error()) << "Could not fetch all transaction hashes: " << res.error();
return {};
}
auto const& result = res.value();
if (not result.hasRows())
{
if (not result.hasRows()) {
LOG(log_.error()) << "Could not fetch all transaction hashes - no rows; ledger = "
<< std::to_string(ledgerSequence);
return {};
@@ -377,8 +342,7 @@ public:
if (not res)
return std::nullopt;
if (auto const maybeRow = res->template get<uint32_t, ripple::AccountID, bool>(); maybeRow)
{
if (auto const maybeRow = res->template get<uint32_t, ripple::AccountID, bool>(); maybeRow) {
auto [seq, owner, isBurned] = *maybeRow;
auto result = std::make_optional<NFT>(tokenID, seq, owner, isBurned);
@@ -394,8 +358,7 @@ public:
// even though we are returning a blank URI, the NFT might have had
// one.
auto uriRes = executor_.read(yield, schema_->selectNFTURI, tokenID, ledgerSequence);
if (uriRes)
{
if (uriRes) {
if (auto const maybeUri = uriRes->template get<ripple::Blob>(); maybeUri)
result->uri = *maybeUri;
}
@@ -413,7 +376,8 @@ public:
std::uint32_t const limit,
bool const forward,
std::optional<TransactionsCursor> const& cursorIn,
boost::asio::yield_context yield) const override
boost::asio::yield_context yield
) const override
{
auto rng = fetchLedgerRange();
if (!rng)
@@ -427,14 +391,11 @@ public:
}();
auto cursor = cursorIn;
if (cursor)
{
if (cursor) {
statement.bindAt(1, cursor->asTuple());
LOG(log_.debug()) << "token_id = " << ripple::strHex(tokenID) << " tuple = " << cursor->ledgerSequence
<< cursor->transactionIndex;
}
else
{
} else {
auto const seq = forward ? rng->minSequence : rng->maxSequence;
auto const placeHolder = forward ? 0 : std::numeric_limits<std::uint32_t>::max();
@@ -447,8 +408,7 @@ public:
auto const res = executor_.read(yield, statement);
auto const& results = res.value();
if (not results.hasRows())
{
if (not results.hasRows()) {
LOG(log_.debug()) << "No rows returned";
return {};
}
@@ -457,11 +417,9 @@ public:
auto numRows = results.numRows();
LOG(log_.info()) << "num_rows = " << numRows;
for (auto [hash, data] : extract<ripple::uint256, std::tuple<uint32_t, uint32_t>>(results))
{
for (auto [hash, data] : extract<ripple::uint256, std::tuple<uint32_t, uint32_t>>(results)) {
hashes.push_back(hash);
if (--numRows == 0)
{
if (--numRows == 0) {
LOG(log_.debug()) << "Setting cursor";
cursor = data;
@@ -475,8 +433,7 @@ public:
auto const txns = fetchTransactions(hashes, yield);
LOG(log_.debug()) << "NFT Txns = " << txns.size();
if (txns.size() == limit)
{
if (txns.size() == limit) {
LOG(log_.debug()) << "Returning cursor";
return {txns, cursor};
}
@@ -489,20 +446,14 @@ public:
const override
{
LOG(log_.debug()) << "Fetching ledger object for seq " << sequence << ", key = " << ripple::to_string(key);
if (auto const res = executor_.read(yield, schema_->selectObject, key, sequence); res)
{
if (auto const result = res->template get<Blob>(); result)
{
if (auto const res = executor_.read(yield, schema_->selectObject, key, sequence); res) {
if (auto const result = res->template get<Blob>(); result) {
if (result->size())
return *result;
}
else
{
} else {
LOG(log_.debug()) << "Could not fetch ledger object - no rows";
}
}
else
{
} else {
LOG(log_.error()) << "Could not fetch ledger object: " << res.error();
}
@@ -512,18 +463,14 @@ public:
std::optional<TransactionAndMetadata>
fetchTransaction(ripple::uint256 const& hash, boost::asio::yield_context yield) const override
{
if (auto const res = executor_.read(yield, schema_->selectTransaction, hash); res)
{
if (auto const maybeValue = res->template get<Blob, Blob, uint32_t, uint32_t>(); maybeValue)
{
if (auto const res = executor_.read(yield, schema_->selectTransaction, hash); res) {
if (auto const maybeValue = res->template get<Blob, Blob, uint32_t, uint32_t>(); maybeValue) {
auto [transaction, meta, seq, date] = *maybeValue;
return std::make_optional<TransactionAndMetadata>(transaction, meta, seq, date);
}
LOG(log_.debug()) << "Could not fetch transaction - no rows";
}
else
{
} else {
LOG(log_.error()) << "Could not fetch transaction: " << res.error();
}
@@ -534,19 +481,15 @@ public:
doFetchSuccessorKey(ripple::uint256 key, std::uint32_t const ledgerSequence, boost::asio::yield_context yield)
const override
{
if (auto const res = executor_.read(yield, schema_->selectSuccessor, key, ledgerSequence); res)
{
if (auto const result = res->template get<ripple::uint256>(); result)
{
if (auto const res = executor_.read(yield, schema_->selectSuccessor, key, ledgerSequence); res) {
if (auto const result = res->template get<ripple::uint256>(); result) {
if (*result == lastKey)
return std::nullopt;
return *result;
}
LOG(log_.debug()) << "Could not fetch successor - no rows";
}
else
{
} else {
LOG(log_.error()) << "Could not fetch successor: " << res.error();
}
@@ -569,9 +512,11 @@ public:
auto const timeDiff = util::timed([this, yield, &results, &hashes, &statements]() {
// TODO: seems like a job for "hash IN (list of hashes)" instead?
std::transform(
std::cbegin(hashes), std::cend(hashes), std::back_inserter(statements), [this](auto const& hash) {
return schema_->selectTransaction.bind(hash);
});
std::cbegin(hashes),
std::cend(hashes),
std::back_inserter(statements),
[this](auto const& hash) { return schema_->selectTransaction.bind(hash); }
);
auto const entries = executor_.readEach(yield, statements);
std::transform(
@@ -583,7 +528,8 @@ public:
return *maybeRow;
return {};
});
}
);
});
assert(numHashes == results.size());
@@ -596,7 +542,8 @@ public:
doFetchLedgerObjects(
std::vector<ripple::uint256> const& keys,
std::uint32_t const sequence,
boost::asio::yield_context yield) const override
boost::asio::yield_context yield
) const override
{
if (keys.empty())
return {};
@@ -612,18 +559,24 @@ public:
// TODO: seems like a job for "key IN (list of keys)" instead?
std::transform(
std::cbegin(keys), std::cend(keys), std::back_inserter(statements), [this, &sequence](auto const& key) {
return schema_->selectObject.bind(key, sequence);
});
std::cbegin(keys),
std::cend(keys),
std::back_inserter(statements),
[this, &sequence](auto const& key) { return schema_->selectObject.bind(key, sequence); }
);
auto const entries = executor_.readEach(yield, statements);
std::transform(
std::cbegin(entries), std::cend(entries), std::back_inserter(results), [](auto const& res) -> Blob {
std::cbegin(entries),
std::cend(entries),
std::back_inserter(results),
[](auto const& res) -> Blob {
if (auto const maybeValue = res.template get<Blob>(); maybeValue)
return *maybeValue;
return {};
});
}
);
LOG(log_.trace()) << "Fetched " << numKeys << " objects";
return results;
@@ -634,15 +587,13 @@ public:
{
auto const [keys, timeDiff] = util::timed([this, &ledgerSequence, yield]() -> std::vector<ripple::uint256> {
auto const res = executor_.read(yield, schema_->selectDiff, ledgerSequence);
if (not res)
{
if (not res) {
LOG(log_.error()) << "Could not fetch ledger diff: " << res.error() << "; ledger = " << ledgerSequence;
return {};
}
auto const& results = res.value();
if (not results)
{
if (not results) {
LOG(log_.error()) << "Could not fetch ledger diff - no rows; ledger = " << ledgerSequence;
return {};
}
@@ -672,7 +623,8 @@ public:
std::back_inserter(results),
[](auto const& key, auto const& obj) {
return LedgerObject{key, obj};
});
}
);
return results;
}
@@ -705,8 +657,7 @@ public:
std::vector<Statement> statements;
statements.reserve(data.size() * 10); // assume 10 transactions avg
for (auto& record : data)
{
for (auto& record : data) {
std::transform(
std::begin(record.accounts),
std::end(record.accounts),
@@ -715,8 +666,10 @@ public:
return schema_->insertAccountTx.bind(
std::forward<decltype(account)>(account),
std::make_tuple(record.ledgerSequence, record.transactionIndex),
record.txHash);
});
record.txHash
);
}
);
}
executor_.write(std::move(statements));
@@ -730,7 +683,8 @@ public:
std::transform(std::cbegin(data), std::cend(data), std::back_inserter(statements), [this](auto const& record) {
return schema_->insertNFTTx.bind(
record.tokenID, std::make_tuple(record.ledgerSequence, record.transactionIndex), record.txHash);
record.tokenID, std::make_tuple(record.ledgerSequence, record.transactionIndex), record.txHash
);
});
executor_.write(std::move(statements));
@@ -742,13 +696,15 @@ public:
std::uint32_t const seq,
std::uint32_t const date,
std::string&& transaction,
std::string&& metadata) override
std::string&& metadata
) override
{
LOG(log_.trace()) << "Writing txn to cassandra";
executor_.write(schema_->insertLedgerTransaction, seq, hash);
executor_.write(
schema_->insertTransaction, std::move(hash), seq, date, std::move(transaction), std::move(metadata));
schema_->insertTransaction, std::move(hash), seq, date, std::move(transaction), std::move(metadata)
);
}
void
@@ -757,24 +713,25 @@ public:
std::vector<Statement> statements;
statements.reserve(data.size() * 3);
for (NFTsData const& record : data)
{
for (NFTsData const& record : data) {
statements.push_back(
schema_->insertNFT.bind(record.tokenID, record.ledgerSequence, record.owner, record.isBurned));
schema_->insertNFT.bind(record.tokenID, record.ledgerSequence, record.owner, record.isBurned)
);
// If `uri` is set (and it can be set to an empty uri), we know this
// is a net-new NFT. That is, this NFT has not been seen before by
// us _OR_ it is in the extreme edge case of a re-minted NFT ID with
// the same NFT ID as an already-burned token. In this case, we need
// to record the URI and link to the issuer_nf_tokens table.
if (record.uri)
{
if (record.uri) {
statements.push_back(schema_->insertIssuerNFT.bind(
ripple::nft::getIssuer(record.tokenID),
static_cast<uint32_t>(ripple::nft::getTaxon(record.tokenID)),
record.tokenID));
record.tokenID
));
statements.push_back(
schema_->insertNFTURI.bind(record.tokenID, record.ledgerSequence, record.uri.value()));
schema_->insertNFTURI.bind(record.tokenID, record.ledgerSequence, record.uri.value())
);
}
}
@@ -806,14 +763,12 @@ private:
{
auto const res = executor_.writeSync(statement);
auto maybeSuccess = res->template get<bool>();
if (not maybeSuccess)
{
if (not maybeSuccess) {
LOG(log_.error()) << "executeSyncUpdate - error getting result - no row";
return false;
}
if (not maybeSuccess.value())
{
if (not maybeSuccess.value()) {
LOG(log_.warn()) << "Update failed. Checking if DB state is what we expect";
// error may indicate that another writer wrote something.

17
src/data/DBHelpers.h
View File

@@ -33,8 +33,7 @@
/**
* @brief Struct used to keep track of what to write to account_transactions/account_tx tables.
*/
struct AccountTransactionsData
{
struct AccountTransactionsData {
boost::container::flat_set<ripple::AccountID> accounts;
std::uint32_t ledgerSequence{};
std::uint32_t transactionIndex{};
@@ -56,8 +55,7 @@ struct AccountTransactionsData
*
* Gets written to nf_token_transactions table and the like.
*/
struct NFTTransactionsData
{
struct NFTTransactionsData {
ripple::uint256 tokenID;
std::uint32_t ledgerSequence;
std::uint32_t transactionIndex;
@@ -74,8 +72,7 @@ struct NFTTransactionsData
*
* Gets written to nf_tokens table and the like.
*/
struct NFTsData
{
struct NFTsData {
ripple::uint256 tokenID;
std::uint32_t ledgerSequence;
@@ -107,7 +104,8 @@ struct NFTsData
ripple::uint256 const& tokenID,
ripple::AccountID const& owner,
ripple::Blob const& uri,
ripple::TxMeta const& meta)
ripple::TxMeta const& meta
)
: tokenID(tokenID), ledgerSequence(meta.getLgrSeq()), transactionIndex(meta.getIndex()), owner(owner), uri(uri)
{
}
@@ -133,7 +131,8 @@ struct NFTsData
ripple::uint256 const& tokenID,
std::uint32_t const ledgerSequence,
ripple::AccountID const& owner,
ripple::Blob const& uri)
ripple::Blob const& uri
)
: tokenID(tokenID), ledgerSequence(ledgerSequence), owner(owner), uri(uri)
{
}
@@ -252,7 +251,7 @@ getBookBase(T const& key)
inline std::string
uint256ToString(ripple::uint256 const& input)
{
return {reinterpret_cast<const char*>(input.data()), ripple::uint256::size()};
return {reinterpret_cast<char const*>(input.data()), ripple::uint256::size()};
}
/** @brief The ripple epoch start timestamp. Midnight on 1st January 2000. */

16
src/data/LedgerCache.cpp
View File

@@ -36,26 +36,20 @@ LedgerCache::update(std::vector<LedgerObject> const& objs, uint32_t seq, bool is
{
std::scoped_lock const lck{mtx_};
if (seq > latestSeq_)
{
if (seq > latestSeq_) {
assert(seq == latestSeq_ + 1 || latestSeq_ == 0);
latestSeq_ = seq;
}
for (auto const& obj : objs)
{
if (!obj.blob.empty())
{
for (auto const& obj : objs) {
if (!obj.blob.empty()) {
if (isBackground && deletes_.contains(obj.key))
continue;
auto& e = map_[obj.key];
if (seq > e.seq)
{
if (seq > e.seq) {
e = {seq, obj.blob};
}
}
else
{
} else {
map_.erase(obj.key);
if (!full_ && !isBackground)
deletes_.insert(obj.key);

6
src/data/LedgerCache.h
View File

@@ -33,10 +33,8 @@ namespace data {
/**
* @brief Cache for an entire ledger.
*/
class LedgerCache
{
struct CacheEntry
{
class LedgerCache {
struct CacheEntry {
uint32_t seq = 0;
Blob blob;
};

28
src/data/Types.h
View File

@@ -34,13 +34,12 @@ using Blob = std::vector<unsigned char>;
/**
* @brief Represents an object in the ledger.
*/
struct LedgerObject
{
struct LedgerObject {
ripple::uint256 key;
Blob blob;
bool
operator==(const LedgerObject& other) const
operator==(LedgerObject const& other) const
{
return key == other.key && blob == other.blob;
}
@@ -49,8 +48,7 @@ struct LedgerObject
/**
* @brief Represents a page of LedgerObjects.
*/
struct LedgerPage
{
struct LedgerPage {
std::vector<LedgerObject> objects;
std::optional<ripple::uint256> cursor;
};
@@ -58,8 +56,7 @@ struct LedgerPage
/**
* @brief Represents a page of book offer objects.
*/
struct BookOffersPage
{
struct BookOffersPage {
std::vector<LedgerObject> offers;
std::optional<ripple::uint256> cursor;
};
@@ -67,8 +64,7 @@ struct BookOffersPage
/**
* @brief Represents a transaction and its metadata bundled together.
*/
struct TransactionAndMetadata
{
struct TransactionAndMetadata {
Blob transaction;
Blob metadata;
std::uint32_t ledgerSequence = 0;
@@ -89,7 +85,7 @@ struct TransactionAndMetadata
}
bool
operator==(const TransactionAndMetadata& other) const
operator==(TransactionAndMetadata const& other) const
{
return transaction == other.transaction && metadata == other.metadata &&
ledgerSequence == other.ledgerSequence && date == other.date;
@@ -99,8 +95,7 @@ struct TransactionAndMetadata
/**
* @brief Represents a cursor into the transactions table.
*/
struct TransactionsCursor
{
struct TransactionsCursor {
std::uint32_t ledgerSequence = 0;
std::uint32_t transactionIndex = 0;
@@ -128,8 +123,7 @@ struct TransactionsCursor
/**
* @brief Represests a bundle of transactions with metadata and a cursor to the next page.
*/
struct TransactionsAndCursor
{
struct TransactionsAndCursor {
std::vector<TransactionAndMetadata> txns;
std::optional<TransactionsCursor> cursor;
};
@@ -137,8 +131,7 @@ struct TransactionsAndCursor
/**
* @brief Represents a NFToken.
*/
struct NFT
{
struct NFT {
ripple::uint256 tokenID;
std::uint32_t ledgerSequence{};
ripple::AccountID owner;
@@ -172,8 +165,7 @@ struct NFT
/**
* @brief Stores a range of sequences as a min and max pair.
*/
struct LedgerRange
{
struct LedgerRange {
std::uint32_t minSequence = 0;
std::uint32_t maxSequence = 0;
};

98
src/data/cassandra/Concepts.h
View File

@@ -34,57 +34,93 @@ namespace data::cassandra {
/**
* @brief The requirements of a settings provider.
*/
// clang-format off
template <typename T>
concept SomeSettingsProvider = requires(T a) {
{ a.getSettings() } -> std::same_as<Settings>;
{ a.getKeyspace() } -> std::same_as<std::string>;
{ a.getTablePrefix() } -> std::same_as<std::optional<std::string>>;
{ a.getReplicationFactor() } -> std::same_as<uint16_t>;
{ a.getTtl() } -> std::same_as<uint16_t>;
{
a.getSettings()
} -> std::same_as<Settings>;
{
a.getKeyspace()
} -> std::same_as<std::string>;
{
a.getTablePrefix()
} -> std::same_as<std::optional<std::string>>;
{
a.getReplicationFactor()
} -> std::same_as<uint16_t>;
{
a.getTtl()
} -> std::same_as<uint16_t>;
};
// clang-format on
/**
* @brief The requirements of an execution strategy.
*/
// clang-format off
template <typename T>
concept SomeExecutionStrategy = requires(
T a,
Settings settings,
Handle handle,
Statement statement,
T a,
Settings settings,
Handle handle,
Statement statement,
std::vector<Statement> statements,
PreparedStatement prepared,
boost::asio::yield_context token
) {
{ T(settings, handle) };
{ a.sync() } -> std::same_as<void>;
{ a.isTooBusy() } -> std::same_as<bool>;
{ a.writeSync(statement) } -> std::same_as<ResultOrError>;
{ a.writeSync(prepared) } -> std::same_as<ResultOrError>;
{ a.write(prepared) } -> std::same_as<void>;
{ a.write(std::move(statements)) } -> std::same_as<void>;
{ a.read(token, prepared) } -> std::same_as<ResultOrError>;
{ a.read(token, statement) } -> std::same_as<ResultOrError>;
{ a.read(token, statements) } -> std::same_as<ResultOrError>;
{ a.readEach(token, statements) } -> std::same_as<std::vector<Result>>;
{ a.stats() } -> std::same_as<boost::json::object>;
{
T(settings, handle)
};
{
a.sync()
} -> std::same_as<void>;
{
a.isTooBusy()
} -> std::same_as<bool>;
{
a.writeSync(statement)
} -> std::same_as<ResultOrError>;
{
a.writeSync(prepared)
} -> std::same_as<ResultOrError>;
{
a.write(prepared)
} -> std::same_as<void>;
{
a.write(std::move(statements))
} -> std::same_as<void>;
{
a.read(token, prepared)
} -> std::same_as<ResultOrError>;
{
a.read(token, statement)
} -> std::same_as<ResultOrError>;
{
a.read(token, statements)
} -> std::same_as<ResultOrError>;
{
a.readEach(token, statements)
} -> std::same_as<std::vector<Result>>;
{
a.stats()
} -> std::same_as<boost::json::object>;
};
// clang-format on
/**
* @brief The requirements of a retry policy.
*/
// clang-format off
template <typename T>
concept SomeRetryPolicy = requires(T a, boost::asio::io_context ioc, CassandraError err, uint32_t attempt) {
{ T(ioc) };
{ a.shouldRetry(err) } -> std::same_as<bool>;
{ a.retry([](){}) } -> std::same_as<void>;
{ a.calculateDelay(attempt) } -> std::same_as<std::chrono::milliseconds>;
{
T(ioc)
};
{
a.shouldRetry(err)
} -> std::same_as<bool>;
{
a.retry([]() {})
} -> std::same_as<void>;
{
a.calculateDelay(attempt)
} -> std::same_as<std::chrono::milliseconds>;
};
// clang-format on
} // namespace data::cassandra

9
src/data/cassandra/Error.h
View File

@@ -29,8 +29,7 @@ namespace data::cassandra {
/**
* @brief A simple container for both error message and error code.
*/
class CassandraError
{
class CassandraError {
std::string message_;
uint32_t code_{};
@@ -42,14 +41,16 @@ public:
template <typename T>
friend std::string
operator+(T const& lhs, CassandraError const& rhs) requires std::is_convertible_v<T, std::string>
operator+(T const& lhs, CassandraError const& rhs)
requires std::is_convertible_v<T, std::string>
{
return lhs + rhs.message();
}
template <typename T>
friend bool
operator==(T const& lhs, CassandraError const& rhs) requires std::is_convertible_v<T, std::string>
operator==(T const& lhs, CassandraError const& rhs)
requires std::is_convertible_v<T, std::string>
{
return lhs == rhs.message();
}

3
src/data/cassandra/Handle.cpp
View File

@@ -97,8 +97,7 @@ Handle::asyncExecuteEach(std::vector<Statement> const& statements) const
Handle::MaybeErrorType
Handle::executeEach(std::vector<Statement> const& statements) const
{
for (auto futures = asyncExecuteEach(statements); auto const& future : futures)
{
for (auto futures = asyncExecuteEach(statements); auto const& future : futures) {
if (auto rc = future.await(); not rc)
return rc;
}

3
src/data/cassandra/Handle.h
View File

@@ -42,8 +42,7 @@ namespace data::cassandra {
/**
* @brief Represents a handle to the cassandra database cluster
*/
class Handle
{
class Handle {
detail::Cluster cluster_;
detail::Session session_;

144
src/data/cassandra/Schema.h
View File

@@ -41,8 +41,7 @@ template <SomeSettingsProvider SettingsProviderType>
* @brief Manages the DB schema and provides access to prepared statements.
*/
template <SomeSettingsProvider SettingsProviderType>
class Schema
{
class Schema {
util::Logger log_{"Backend"};
std::reference_wrapper<SettingsProviderType const> settingsProvider_;
@@ -62,7 +61,8 @@ public:
AND durable_writes = true
)",
settingsProvider_.get().getKeyspace(),
settingsProvider_.get().getReplicationFactor());
settingsProvider_.get().getReplicationFactor()
);
}();
// =======================
@@ -85,7 +85,8 @@ public:
AND default_time_to_live = {}
)",
qualifiedTableName(settingsProvider_.get(), "objects"),
settingsProvider_.get().getTtl()));
settingsProvider_.get().getTtl()
));
statements.emplace_back(fmt::format(
R"(
@@ -100,7 +101,8 @@ public:
WITH default_time_to_live = {}
)",
qualifiedTableName(settingsProvider_.get(), "transactions"),
settingsProvider_.get().getTtl()));
settingsProvider_.get().getTtl()
));
statements.emplace_back(fmt::format(
R"(
@@ -113,7 +115,8 @@ public:
WITH default_time_to_live = {}
)",
qualifiedTableName(settingsProvider_.get(), "ledger_transactions"),
settingsProvider_.get().getTtl()));
settingsProvider_.get().getTtl()
));
statements.emplace_back(fmt::format(
R"(
@@ -127,7 +130,8 @@ public:
WITH default_time_to_live = {}
)",
qualifiedTableName(settingsProvider_.get(), "successor"),
settingsProvider_.get().getTtl()));
settingsProvider_.get().getTtl()
));
statements.emplace_back(fmt::format(
R"(
@@ -140,7 +144,8 @@ public:
WITH default_time_to_live = {}
)",
qualifiedTableName(settingsProvider_.get(), "diff"),
settingsProvider_.get().getTtl()));
settingsProvider_.get().getTtl()
));
statements.emplace_back(fmt::format(
R"(
@@ -155,7 +160,8 @@ public:
AND default_time_to_live = {}
)",
qualifiedTableName(settingsProvider_.get(), "account_tx"),
settingsProvider_.get().getTtl()));
settingsProvider_.get().getTtl()
));
statements.emplace_back(fmt::format(
R"(
@@ -167,7 +173,8 @@ public:
WITH default_time_to_live = {}
)",
qualifiedTableName(settingsProvider_.get(), "ledgers"),
settingsProvider_.get().getTtl()));
settingsProvider_.get().getTtl()
));
statements.emplace_back(fmt::format(
R"(
@@ -179,7 +186,8 @@ public:
WITH default_time_to_live = {}
)",
qualifiedTableName(settingsProvider_.get(), "ledger_hashes"),
settingsProvider_.get().getTtl()));
settingsProvider_.get().getTtl()
));
statements.emplace_back(fmt::format(
R"(
@@ -189,7 +197,8 @@ public:
sequence bigint
)
)",
qualifiedTableName(settingsProvider_.get(), "ledger_range")));
qualifiedTableName(settingsProvider_.get(), "ledger_range")
));
statements.emplace_back(fmt::format(
R"(
@@ -205,7 +214,8 @@ public:
AND default_time_to_live = {}
)",
qualifiedTableName(settingsProvider_.get(), "nf_tokens"),
settingsProvider_.get().getTtl()));
settingsProvider_.get().getTtl()
));
statements.emplace_back(fmt::format(
R"(
@@ -220,7 +230,8 @@ public:
AND default_time_to_live = {}
)",
qualifiedTableName(settingsProvider_.get(), "issuer_nf_tokens_v2"),
settingsProvider_.get().getTtl()));
settingsProvider_.get().getTtl()
));
statements.emplace_back(fmt::format(
R"(
@@ -235,7 +246,8 @@ public:
AND default_time_to_live = {}
)",
qualifiedTableName(settingsProvider_.get(), "nf_token_uris"),
settingsProvider_.get().getTtl()));
settingsProvider_.get().getTtl()
));
statements.emplace_back(fmt::format(
R"(
@@ -250,7 +262,8 @@ public:
AND default_time_to_live = {}
)",
qualifiedTableName(settingsProvider_.get(), "nf_token_transactions"),
settingsProvider_.get().getTtl()));
settingsProvider_.get().getTtl()
));
return statements;
}();
@@ -258,8 +271,7 @@ public:
/**
* @brief Prepared statements holder.
*/
class Statements
{
class Statements {
std::reference_wrapper<SettingsProviderType const> settingsProvider_;
std::reference_wrapper<Handle const> handle_;
@@ -280,7 +292,8 @@ public:
(key, sequence, object)
VALUES (?, ?, ?)
)",
qualifiedTableName(settingsProvider_.get(), "objects")));
qualifiedTableName(settingsProvider_.get(), "objects")
));
}();
PreparedStatement insertTransaction = [this]() {
@@ -290,7 +303,8 @@ public:
(hash, ledger_sequence, date, transaction, metadata)
VALUES (?, ?, ?, ?, ?)
)",
qualifiedTableName(settingsProvider_.get(), "transactions")));
qualifiedTableName(settingsProvider_.get(), "transactions")
));
}();
PreparedStatement insertLedgerTransaction = [this]() {
@@ -300,7 +314,8 @@ public:
(ledger_sequence, hash)
VALUES (?, ?)
)",
qualifiedTableName(settingsProvider_.get(), "ledger_transactions")));
qualifiedTableName(settingsProvider_.get(), "ledger_transactions")
));
}();
PreparedStatement insertSuccessor = [this]() {
@@ -310,7 +325,8 @@ public:
(key, seq, next)
VALUES (?, ?, ?)
)",
qualifiedTableName(settingsProvider_.get(), "successor")));
qualifiedTableName(settingsProvider_.get(), "successor")
));
}();
PreparedStatement insertDiff = [this]() {
@@ -320,7 +336,8 @@ public:
(seq, key)
VALUES (?, ?)
)",
qualifiedTableName(settingsProvider_.get(), "diff")));
qualifiedTableName(settingsProvider_.get(), "diff")
));
}();
PreparedStatement insertAccountTx = [this]() {
@@ -330,7 +347,8 @@ public:
(account, seq_idx, hash)
VALUES (?, ?, ?)
)",
qualifiedTableName(settingsProvider_.get(), "account_tx")));
qualifiedTableName(settingsProvider_.get(), "account_tx")
));
}();
PreparedStatement insertNFT = [this]() {
@@ -340,7 +358,8 @@ public:
(token_id, sequence, owner, is_burned)
VALUES (?, ?, ?, ?)
)",
qualifiedTableName(settingsProvider_.get(), "nf_tokens")));
qualifiedTableName(settingsProvider_.get(), "nf_tokens")
));
}();
PreparedStatement insertIssuerNFT = [this]() {
@@ -350,7 +369,8 @@ public:
(issuer, taxon, token_id)
VALUES (?, ?, ?)
)",
qualifiedTableName(settingsProvider_.get(), "issuer_nf_tokens_v2")));
qualifiedTableName(settingsProvider_.get(), "issuer_nf_tokens_v2")
));
}();
PreparedStatement insertNFTURI = [this]() {
@@ -360,7 +380,8 @@ public:
(token_id, sequence, uri)
VALUES (?, ?, ?)
)",
qualifiedTableName(settingsProvider_.get(), "nf_token_uris")));
qualifiedTableName(settingsProvider_.get(), "nf_token_uris")
));
}();
PreparedStatement insertNFTTx = [this]() {
@@ -370,7 +391,8 @@ public:
(token_id, seq_idx, hash)
VALUES (?, ?, ?)
)",
qualifiedTableName(settingsProvider_.get(), "nf_token_transactions")));
qualifiedTableName(settingsProvider_.get(), "nf_token_transactions")
));
}();
PreparedStatement insertLedgerHeader = [this]() {
@@ -380,7 +402,8 @@ public:
(sequence, header)
VALUES (?, ?)
)",
qualifiedTableName(settingsProvider_.get(), "ledgers")));
qualifiedTableName(settingsProvider_.get(), "ledgers")
));
}();
PreparedStatement insertLedgerHash = [this]() {
@@ -390,7 +413,8 @@ public:
(hash, sequence)
VALUES (?, ?)
)",
qualifiedTableName(settingsProvider_.get(), "ledger_hashes")));
qualifiedTableName(settingsProvider_.get(), "ledger_hashes")
));
}();
//
@@ -405,7 +429,8 @@ public:
WHERE is_latest = ?
IF sequence IN (?, null)
)",
qualifiedTableName(settingsProvider_.get(), "ledger_range")));
qualifiedTableName(settingsProvider_.get(), "ledger_range")
));
}();
PreparedStatement deleteLedgerRange = [this]() {
@@ -415,7 +440,8 @@ public:
SET sequence = ?
WHERE is_latest = false
)",
qualifiedTableName(settingsProvider_.get(), "ledger_range")));
qualifiedTableName(settingsProvider_.get(), "ledger_range")
));
}();
//
@@ -432,7 +458,8 @@ public:
ORDER BY seq DESC
LIMIT 1
)",
qualifiedTableName(settingsProvider_.get(), "successor")));
qualifiedTableName(settingsProvider_.get(), "successor")
));
}();
PreparedStatement selectDiff = [this]() {
@@ -442,7 +469,8 @@ public:
FROM {}
WHERE seq = ?
)",
qualifiedTableName(settingsProvider_.get(), "diff")));
qualifiedTableName(settingsProvider_.get(), "diff")
));
}();
PreparedStatement selectObject = [this]() {
@@ -455,7 +483,8 @@ public:
ORDER BY sequence DESC
LIMIT 1
)",
qualifiedTableName(settingsProvider_.get(), "objects")));
qualifiedTableName(settingsProvider_.get(), "objects")
));
}();
PreparedStatement selectTransaction = [this]() {
@@ -465,7 +494,8 @@ public:
FROM {}
WHERE hash = ?
)",
qualifiedTableName(settingsProvider_.get(), "transactions")));
qualifiedTableName(settingsProvider_.get(), "transactions")
));
}();
PreparedStatement selectAllTransactionHashesInLedger = [this]() {
@@ -475,7 +505,8 @@ public:
FROM {}
WHERE ledger_sequence = ?
)",
qualifiedTableName(settingsProvider_.get(), "ledger_transactions")));
qualifiedTableName(settingsProvider_.get(), "ledger_transactions")
));
}();
PreparedStatement selectLedgerPageKeys = [this]() {
@@ -489,7 +520,8 @@ public:
LIMIT ?
ALLOW FILTERING
)",
qualifiedTableName(settingsProvider_.get(), "objects")));
qualifiedTableName(settingsProvider_.get(), "objects")
));
}();
PreparedStatement selectLedgerPage = [this]() {
@@ -503,7 +535,8 @@ public:
LIMIT ?
ALLOW FILTERING
)",
qualifiedTableName(settingsProvider_.get(), "objects")));
qualifiedTableName(settingsProvider_.get(), "objects")
));
}();
PreparedStatement getToken = [this]() {
@@ -514,7 +547,8 @@ public:
WHERE key = ?
LIMIT 1
)",
qualifiedTableName(settingsProvider_.get(), "objects")));
qualifiedTableName(settingsProvider_.get(), "objects")
));
}();
PreparedStatement selectAccountTx = [this]() {
@@ -526,7 +560,8 @@ public:
AND seq_idx < ?
LIMIT ?
)",
qualifiedTableName(settingsProvider_.get(), "account_tx")));
qualifiedTableName(settingsProvider_.get(), "account_tx")
));
}();
PreparedStatement selectAccountTxForward = [this]() {
@@ -539,7 +574,8 @@ public:
ORDER BY seq_idx ASC
LIMIT ?
)",
qualifiedTableName(settingsProvider_.get(), "account_tx")));
qualifiedTableName(settingsProvider_.get(), "account_tx")
));
}();
PreparedStatement selectNFT = [this]() {
@@ -552,7 +588,8 @@ public:
ORDER BY sequence DESC
LIMIT 1
)",
qualifiedTableName(settingsProvider_.get(), "nf_tokens")));
qualifiedTableName(settingsProvider_.get(), "nf_tokens")
));
}();
PreparedStatement selectNFTURI = [this]() {
@@ -565,7 +602,8 @@ public:
ORDER BY sequence DESC
LIMIT 1
)",
qualifiedTableName(settingsProvider_.get(), "nf_token_uris")));
qualifiedTableName(settingsProvider_.get(), "nf_token_uris")
));
}();
PreparedStatement selectNFTTx = [this]() {
@@ -578,7 +616,8 @@ public:
ORDER BY seq_idx DESC
LIMIT ?
)",
qualifiedTableName(settingsProvider_.get(), "nf_token_transactions")));
qualifiedTableName(settingsProvider_.get(), "nf_token_transactions")
));
}();
PreparedStatement selectNFTTxForward = [this]() {
@@ -591,7 +630,8 @@ public:
ORDER BY seq_idx ASC
LIMIT ?
)",
qualifiedTableName(settingsProvider_.get(), "nf_token_transactions")));
qualifiedTableName(settingsProvider_.get(), "nf_token_transactions")
));
}();
PreparedStatement selectLedgerByHash = [this]() {
@@ -602,7 +642,8 @@ public:
WHERE hash = ?
LIMIT 1
)",
qualifiedTableName(settingsProvider_.get(), "ledger_hashes")));
qualifiedTableName(settingsProvider_.get(), "ledger_hashes")
));
}();
PreparedStatement selectLedgerBySeq = [this]() {
@@ -612,7 +653,8 @@ public:
FROM {}
WHERE sequence = ?
)",
qualifiedTableName(settingsProvider_.get(), "ledgers")));
qualifiedTableName(settingsProvider_.get(), "ledgers")
));
}();
PreparedStatement selectLatestLedger = [this]() {
@@ -622,7 +664,8 @@ public:
FROM {}
WHERE is_latest = true
)",
qualifiedTableName(settingsProvider_.get(), "ledger_range")));
qualifiedTableName(settingsProvider_.get(), "ledger_range")
));
}();
PreparedStatement selectLedgerRange = [this]() {
@@ -631,7 +674,8 @@ public:
SELECT sequence
FROM {}
)",
qualifiedTableName(settingsProvider_.get(), "ledger_range")));
qualifiedTableName(settingsProvider_.get(), "ledger_range")
));
}();
};

23
src/data/cassandra/SettingsProvider.cpp
View File

@@ -35,11 +35,8 @@ namespace detail {
inline Settings::ContactPoints
tag_invoke(boost::json::value_to_tag<Settings::ContactPoints>, boost::json::value const& value)
{
if (not value.is_object())
{
throw std::runtime_error(
"Feed entire Cassandra section to parse "
"Settings::ContactPoints instead");
if (not value.is_object()) {
throw std::runtime_error("Feed entire Cassandra section to parse Settings::ContactPoints instead");
}
util::Config const obj{value};
@@ -79,18 +76,15 @@ SettingsProvider::getSettings() const
std::optional<std::string>
SettingsProvider::parseOptionalCertificate() const
{
if (auto const certPath = config_.maybeValue<std::string>("certfile"); certPath)
{
if (auto const certPath = config_.maybeValue<std::string>("certfile"); certPath) {
auto const path = std::filesystem::path(*certPath);
std::ifstream fileStream(path.string(), std::ios::in);
if (!fileStream)
{
if (!fileStream) {
throw std::system_error(errno, std::generic_category(), "Opening certificate " + path.string());
}
std::string contents(std::istreambuf_iterator<char>{fileStream}, std::istreambuf_iterator<char>{});
if (fileStream.bad())
{
if (fileStream.bad()) {
throw std::system_error(errno, std::generic_category(), "Reading certificate " + path.string());
}
@@ -104,12 +98,9 @@ Settings
SettingsProvider::parseSettings() const
{
auto settings = Settings::defaultSettings();
if (auto const bundle = config_.maybeValue<Settings::SecureConnectionBundle>("secure_connect_bundle"); bundle)
{
if (auto const bundle = config_.maybeValue<Settings::SecureConnectionBundle>("secure_connect_bundle"); bundle) {
settings.connectionInfo = *bundle;
}
else
{
} else {
settings.connectionInfo =
config_.valueOrThrow<Settings::ContactPoints>("Missing contact_points in Cassandra config");
}

3
src/data/cassandra/SettingsProvider.h
View File

@@ -30,8 +30,7 @@ namespace data::cassandra {
/**
* @brief Provides settings for @ref BasicCassandraBackend.
*/
class SettingsProvider
{
class SettingsProvider {
util::Config config_;
std::string keyspace_;

3
src/data/cassandra/Types.h
View File

@@ -52,8 +52,7 @@ using Batch = detail::Batch;
* because clio uses bigint (int64) everywhere except for when one need
* to specify LIMIT, which needs an int32 :-/
*/
struct Limit
{
struct Limit {
int32_t limit;
};

26
src/data/cassandra/impl/AsyncExecutor.h
View File

@@ -48,8 +48,7 @@ template <
typename StatementType,
typename HandleType = Handle,
SomeRetryPolicy RetryPolicyType = ExponentialBackoffRetryPolicy>
class AsyncExecutor : public std::enable_shared_from_this<AsyncExecutor<StatementType, HandleType, RetryPolicyType>>
{
class AsyncExecutor : public std::enable_shared_from_this<AsyncExecutor<StatementType, HandleType, RetryPolicyType>> {
using FutureWithCallbackType = typename HandleType::FutureWithCallbackType;
using CallbackType = std::function<void(typename HandleType::ResultOrErrorType)>;
using RetryCallbackType = std::function<void()>;
@@ -77,13 +76,13 @@ public:
RetryCallbackType&& onRetry)
{
// this is a helper that allows us to use std::make_shared below
struct EnableMakeShared : public AsyncExecutor<StatementType, HandleType, RetryPolicyType>
{
struct EnableMakeShared : public AsyncExecutor<StatementType, HandleType, RetryPolicyType> {
EnableMakeShared(
boost::asio::io_context& ioc,
StatementType&& data,
CallbackType&& onComplete,
RetryCallbackType&& onRetry)
RetryCallbackType&& onRetry
)
: AsyncExecutor(ioc, std::move(data), std::move(onComplete), std::move(onRetry))
{
}
@@ -98,7 +97,8 @@ private:
boost::asio::io_context& ioc,
StatementType&& data,
CallbackType&& onComplete,
RetryCallbackType&& onRetry)
RetryCallbackType&& onRetry
)
: data_{std::move(data)}, retryPolicy_{ioc}, onComplete_{std::move(onComplete)}, onRetry_{std::move(onRetry)}
{
}
@@ -110,19 +110,13 @@ private:
// lifetime is extended by capturing self ptr
auto handler = [this, &handle, self](auto&& res) mutable {
if (res)
{
if (res) {
onComplete_(std::forward<decltype(res)>(res));
}
else
{
if (retryPolicy_.shouldRetry(res.error()))
{
} else {
if (retryPolicy_.shouldRetry(res.error())) {
onRetry_();
retryPolicy_.retry([self, &handle]() { self->execute(handle); });
}
else
{
} else {
onComplete_(std::forward<decltype(res)>(res)); // report error
}
}

6
src/data/cassandra/impl/Batch.cpp
View File

@@ -37,8 +37,7 @@ Batch::Batch(std::vector<Statement> const& statements)
{
cass_batch_set_is_idempotent(*this, cass_true);
for (auto const& statement : statements)
{
for (auto const& statement : statements) {
if (auto const res = add(statement); not res)
throw std::runtime_error("Failed to add statement to batch: " + res.error());
}
@@ -47,8 +46,7 @@ Batch::Batch(std::vector<Statement> const& statements)
MaybeError
Batch::add(Statement const& statement)
{
if (auto const rc = cass_batch_add_statement(*this, statement); rc != CASS_OK)
{
if (auto const rc = cass_batch_add_statement(*this, statement); rc != CASS_OK) {
return Error{CassandraError{cass_error_desc(rc), rc}};
}
return {};

3
src/data/cassandra/impl/Batch.h
View File

@@ -26,8 +26,7 @@
namespace data::cassandra::detail {
struct Batch : public ManagedObject<CassBatch>
{
struct Batch : public ManagedObject<CassBatch> {
Batch(std::vector<Statement> const& statements);
MaybeError

37
src/data/cassandra/impl/Cluster.cpp
View File

@@ -31,8 +31,7 @@ namespace {
constexpr auto clusterDeleter = [](CassCluster* ptr) { cass_cluster_free(ptr); };
template <class... Ts>
struct overloadSet : Ts...
{
struct overloadSet : Ts... {
using Ts::operator()...;
};
@@ -48,16 +47,15 @@ Cluster::Cluster(Settings const& settings) : ManagedObject{cass_cluster_new(), c
using std::to_string;
cass_cluster_set_token_aware_routing(*this, cass_true);
if (auto const rc = cass_cluster_set_protocol_version(*this, CASS_PROTOCOL_VERSION_V4); rc != CASS_OK)
{
throw std::runtime_error(
fmt::format("Error setting cassandra protocol version to v4: {}", cass_error_desc(rc)));
if (auto const rc = cass_cluster_set_protocol_version(*this, CASS_PROTOCOL_VERSION_V4); rc != CASS_OK) {
throw std::runtime_error(fmt::format("Error setting cassandra protocol version to v4: {}", cass_error_desc(rc))
);
}
if (auto const rc = cass_cluster_set_num_threads_io(*this, settings.threads); rc != CASS_OK)
{
if (auto const rc = cass_cluster_set_num_threads_io(*this, settings.threads); rc != CASS_OK) {
throw std::runtime_error(
fmt::format("Error setting cassandra io threads to {}: {}", settings.threads, cass_error_desc(rc)));
fmt::format("Error setting cassandra io threads to {}: {}", settings.threads, cass_error_desc(rc))
);
}
cass_log_set_level(settings.enableLog ? CASS_LOG_TRACE : CASS_LOG_DISABLED);
@@ -65,15 +63,13 @@ Cluster::Cluster(Settings const& settings) : ManagedObject{cass_cluster_new(), c
cass_cluster_set_request_timeout(*this, settings.requestTimeout.count());
if (auto const rc = cass_cluster_set_core_connections_per_host(*this, settings.coreConnectionsPerHost);
rc != CASS_OK)
{
rc != CASS_OK) {
throw std::runtime_error(fmt::format("Could not set core connections per host: {}", cass_error_desc(rc)));
}
auto const queueSize =
settings.queueSizeIO.value_or(settings.maxWriteRequestsOutstanding + settings.maxReadRequestsOutstanding);
if (auto const rc = cass_cluster_set_queue_size_io(*this, queueSize); rc != CASS_OK)
{
if (auto const rc = cass_cluster_set_queue_size_io(*this, queueSize); rc != CASS_OK) {
throw std::runtime_error(fmt::format("Could not set queue size for IO per host: {}", cass_error_desc(rc)));
}
@@ -93,7 +89,8 @@ Cluster::setupConnection(Settings const& settings)
overloadSet{
[this](Settings::ContactPoints const& points) { setupContactPoints(points); },
[this](Settings::SecureConnectionBundle const& bundle) { setupSecureBundle(bundle); }},
settings.connectionInfo);
settings.connectionInfo
);
}
void
@@ -101,10 +98,10 @@ Cluster::setupContactPoints(Settings::ContactPoints const& points)
{
using std::to_string;
auto throwErrorIfNeeded = [](CassError rc, std::string const& label, std::string const& value) {
if (rc != CASS_OK)
{
if (rc != CASS_OK) {
throw std::runtime_error(
fmt::format("Cassandra: Error setting {} [{}]: {}", label, value, cass_error_desc(rc)));
fmt::format("Cassandra: Error setting {} [{}]: {}", label, value, cass_error_desc(rc))
);
}
};
@@ -114,8 +111,7 @@ Cluster::setupContactPoints(Settings::ContactPoints const& points)
throwErrorIfNeeded(rc, "contact_points", points.contactPoints);
}
if (points.port)
{
if (points.port) {
auto const rc = cass_cluster_set_port(*this, points.port.value());
throwErrorIfNeeded(rc, "port", to_string(points.port.value()));
}
@@ -125,8 +121,7 @@ void
Cluster::setupSecureBundle(Settings::SecureConnectionBundle const& bundle)
{
LOG(log_.debug()) << "Attempt connection using secure bundle";
if (auto const rc = cass_cluster_set_cloud_secure_connection_bundle(*this, bundle.bundle.data()); rc != CASS_OK)
{
if (auto const rc = cass_cluster_set_cloud_secure_connection_bundle(*this, bundle.bundle.data()); rc != CASS_OK) {
throw std::runtime_error("Failed to connect using secure connection bundle " + bundle.bundle);
}
}

12
src/data/cassandra/impl/Cluster.h
View File

@@ -38,16 +38,14 @@ namespace data::cassandra::detail {
/**
* @brief Bundles all cassandra settings in one place.
*/
struct Settings
{
struct Settings {
static constexpr std::size_t DEFAULT_CONNECTION_TIMEOUT = 10000;
static constexpr uint32_t DEFAULT_MAX_WRITE_REQUESTS_OUTSTANDING = 10'000;
static constexpr uint32_t DEFAULT_MAX_READ_REQUESTS_OUTSTANDING = 100'000;
/**
* @brief Represents the configuration of contact points for cassandra.
*/
struct ContactPoints
{
struct ContactPoints {
std::string contactPoints = "127.0.0.1"; // defaults to localhost
std::optional<uint16_t> port = {};
};
@@ -55,8 +53,7 @@ struct Settings
/**
* @brief Represents the configuration of a secure connection bundle.
*/
struct SecureConnectionBundle
{
struct SecureConnectionBundle {
std::string bundle; // no meaningful default
};
@@ -117,8 +114,7 @@ struct Settings
}
};
class Cluster : public ManagedObject<CassCluster>
{
class Cluster : public ManagedObject<CassCluster> {
util::Logger log_{"Backend"};
public:

85
src/data/cassandra/impl/ExecutionStrategy.h
View File

@@ -49,8 +49,7 @@ namespace data::cassandra::detail {
* This is ok for now because we are hopefully going to be getting rid of it entirely later on.
*/
template <typename HandleType = Handle, SomeBackendCounters BackendCountersType = BackendCounters>
class DefaultExecutionStrategy
{
class DefaultExecutionStrategy {
util::Logger log_{"Backend"};
std::uint32_t maxWriteRequestsOutstanding_;
@@ -89,7 +88,8 @@ public:
DefaultExecutionStrategy(
Settings const& settings,
HandleType const& handle,
typename BackendCountersType::PtrType counters = BackendCountersType::make())
typename BackendCountersType::PtrType counters = BackendCountersType::make()
)
: maxWriteRequestsOutstanding_{settings.maxWriteRequestsOutstanding}
, maxReadRequestsOutstanding_{settings.maxReadRequestsOutstanding}
, work_{ioc_}
@@ -141,11 +141,9 @@ public:
writeSync(StatementType const& statement)
{
counters_->registerWriteSync();
while (true)
{
while (true) {
auto res = handle_.get().execute(statement);
if (res)
{
if (res) {
return res;
}
@@ -194,7 +192,8 @@ public:
counters_->registerWriteFinished();
},
[this]() { counters_->registerWriteRetry(); });
[this]() { counters_->registerWriteRetry(); }
);
}
/**
@@ -223,7 +222,8 @@ public:
decrementOutstandingRequestCount();
counters_->registerWriteFinished();
},
[this]() { counters_->registerWriteRetry(); });
[this]() { counters_->registerWriteRetry(); }
);
}
/**
@@ -262,8 +262,7 @@ public:
counters_->registerReadStarted(numStatements);
// todo: perhaps use policy instead
while (true)
{
while (true) {
numReadRequestsOutstanding_ += numStatements;
auto init = [this, &statements, &future]<typename Self>(Self& self) {
@@ -272,27 +271,25 @@ public:
future.emplace(handle_.get().asyncExecute(statements, [sself](auto&& res) mutable {
boost::asio::post(
boost::asio::get_associated_executor(*sself),
[sself, res = std::forward<decltype(res)>(res)]() mutable { sself->complete(std::move(res)); });
[sself, res = std::forward<decltype(res)>(res)]() mutable { sself->complete(std::move(res)); }
);
}));
};
auto res = boost::asio::async_compose<CompletionTokenType, void(ResultOrErrorType)>(
init, token, boost::asio::get_associated_executor(token));
init, token, boost::asio::get_associated_executor(token)
);
numReadRequestsOutstanding_ -= numStatements;
if (res)
{
if (res) {
counters_->registerReadFinished(numStatements);
return res;
}
LOG(log_.error()) << "Failed batch read in coroutine: " << res.error();
try
{
try {
throwErrorIfNeeded(res.error());
}
catch (...)
{
} catch (...) {
counters_->registerReadError(numStatements);
throw;
}
@@ -317,8 +314,7 @@ public:
counters_->registerReadStarted();
// todo: perhaps use policy instead
while (true)
{
while (true) {
++numReadRequestsOutstanding_;
auto init = [this, &statement, &future]<typename Self>(Self& self) {
auto sself = std::make_shared<Self>(std::move(self));
@@ -326,27 +322,25 @@ public:
future.emplace(handle_.get().asyncExecute(statement, [sself](auto&& res) mutable {
boost::asio::post(
boost::asio::get_associated_executor(*sself),
[sself, res = std::forward<decltype(res)>(res)]() mutable { sself->complete(std::move(res)); });
[sself, res = std::forward<decltype(res)>(res)]() mutable { sself->complete(std::move(res)); }
);
}));
};
auto res = boost::asio::async_compose<CompletionTokenType, void(ResultOrErrorType)>(
init, token, boost::asio::get_associated_executor(token));
init, token, boost::asio::get_associated_executor(token)
);
--numReadRequestsOutstanding_;
if (res)
{
if (res) {
counters_->registerReadFinished();
return res;
}
LOG(log_.error()) << "Failed read in coroutine: " << res.error();
try
{
try {
throwErrorIfNeeded(res.error());
}
catch (...)
{
} catch (...) {
counters_->registerReadError();
throw;
}
@@ -383,10 +377,10 @@ public:
++errorsCount;
// when all async operations complete unblock the result
if (--numOutstanding == 0)
{
boost::asio::post(
boost::asio::get_associated_executor(*sself), [sself]() mutable { sself->complete(); });
if (--numOutstanding == 0) {
boost::asio::post(boost::asio::get_associated_executor(*sself), [sself]() mutable {
sself->complete();
});
}
};
@@ -396,15 +390,16 @@ public:
std::back_inserter(futures),
[this, &executionHandler](auto const& statement) {
return handle_.get().asyncExecute(statement, executionHandler);
});
}
);
};
boost::asio::async_compose<CompletionTokenType, void()>(
init, token, boost::asio::get_associated_executor(token));
init, token, boost::asio::get_associated_executor(token)
);
numReadRequestsOutstanding_ -= statements.size();
if (errorsCount > 0)
{
if (errorsCount > 0) {
assert(errorsCount <= statements.size());
counters_->registerReadError(errorsCount);
counters_->registerReadFinished(statements.size() - errorsCount);
@@ -424,7 +419,8 @@ public:
auto entry = future.get();
auto&& res = entry.value();
return std::move(res);
});
}
);
assert(futures.size() == statements.size());
assert(results.size() == statements.size());
@@ -446,8 +442,7 @@ private:
{
{
std::unique_lock<std::mutex> lck(throttleMutex_);
if (!canAddWriteRequest())
{
if (!canAddWriteRequest()) {
LOG(log_.trace()) << "Max outstanding requests reached. "
<< "Waiting for other requests to finish";
throttleCv_.wait(lck, [this]() { return canAddWriteRequest(); });
@@ -460,8 +455,7 @@ private:
decrementOutstandingRequestCount()
{
// sanity check
if (numWriteRequestsOutstanding_ == 0)
{
if (numWriteRequestsOutstanding_ == 0) {
assert(false);
throw std::runtime_error("decrementing num outstanding below 0");
}
@@ -472,8 +466,7 @@ private:
std::lock_guard const lck(throttleMutex_);
throttleCv_.notify_one();
}
if (cur == 0)
{
if (cur == 0) {
// mutex lock required to prevent race condition around spurious
// wakeup
std::lock_guard const lck(syncMutex_);

13
src/data/cassandra/impl/Future.cpp
View File

@@ -37,8 +37,7 @@ namespace data::cassandra::detail {
MaybeError
Future::await() const
{
if (auto const rc = cass_future_error_code(*this); rc)
{
if (auto const rc = cass_future_error_code(*this); rc) {
auto errMsg = [this](std::string const& label) {
char const* message = nullptr;
std::size_t len = 0;
@@ -53,8 +52,7 @@ Future::await() const
ResultOrError
Future::get() const
{
if (auto const rc = cass_future_error_code(*this); rc)
{
if (auto const rc = cass_future_error_code(*this); rc) {
auto const errMsg = [this](std::string const& label) {
char const* message = nullptr;
std::size_t len = 0;
@@ -75,8 +73,7 @@ invokeHelper(CassFuture* ptr, void* cbPtr)
// stackoverflow.com/questions/77004137/boost-asio-async-compose-gets-stuck-under-load
auto* cb = static_cast<FutureWithCallback::FnType*>(cbPtr);
auto local = std::make_unique<FutureWithCallback::FnType>(std::move(*cb));
if (auto const rc = cass_future_error_code(ptr); rc)
{
if (auto const rc = cass_future_error_code(ptr); rc) {
auto const errMsg = [&ptr](std::string const& label) {
char const* message = nullptr;
std::size_t len = 0;
@@ -84,9 +81,7 @@ invokeHelper(CassFuture* ptr, void* cbPtr)
return label + ": " + std::string{message, len};
}("invokeHelper");
(*local)(Error{CassandraError{errMsg, rc}});
}
else
{
} else {
(*local)(Result{cass_future_get_result(ptr)});
}
}

6
src/data/cassandra/impl/Future.h
View File

@@ -26,8 +26,7 @@
namespace data::cassandra::detail {
struct Future : public ManagedObject<CassFuture>
{
struct Future : public ManagedObject<CassFuture> {
/* implicit */ Future(CassFuture* ptr);
MaybeError
@@ -40,8 +39,7 @@ struct Future : public ManagedObject<CassFuture>
void
invokeHelper(CassFuture* ptr, void* cbPtr);
class FutureWithCallback : public Future
{
class FutureWithCallback : public Future {
public:
using FnType = std::function<void(ResultOrError)>;
using FnPtrType = std::unique_ptr<FnType>;

3
src/data/cassandra/impl/ManagedObject.h
View File

@@ -24,8 +24,7 @@
namespace data::cassandra::detail {
template <typename Managed>
class ManagedObject
{
class ManagedObject {
protected:
std::unique_ptr<Managed, void (*)(Managed*)> ptr_;

52
src/data/cassandra/impl/Result.h
View File

@@ -44,8 +44,7 @@ extractColumn(CassRow const* row, std::size_t idx)
Type output;
auto throwErrorIfNeeded = [](CassError rc, std::string_view label) {
if (rc != CASS_OK)
{
if (rc != CASS_OK) {
auto const tag = '[' + std::string{label} + ']';
throw std::logic_error(tag + ": " + cass_error_desc(rc));
}
@@ -55,60 +54,46 @@ extractColumn(CassRow const* row, std::size_t idx)
using UintTupleType = std::tuple<uint32_t, uint32_t>;
using UCharVectorType = std::vector<unsigned char>;
if constexpr (std::is_same_v<DecayedType, ripple::uint256>)
{
if constexpr (std::is_same_v<DecayedType, ripple::uint256>) {
cass_byte_t const* buf = nullptr;
std::size_t bufSize = 0;
auto const rc = cass_value_get_bytes(cass_row_get_column(row, idx), &buf, &bufSize);
throwErrorIfNeeded(rc, "Extract ripple::uint256");
output = ripple::uint256::fromVoid(buf);
}
else if constexpr (std::is_same_v<DecayedType, ripple::AccountID>)
{
} else if constexpr (std::is_same_v<DecayedType, ripple::AccountID>) {
cass_byte_t const* buf = nullptr;
std::size_t bufSize = 0;
auto const rc = cass_value_get_bytes(cass_row_get_column(row, idx), &buf, &bufSize);
throwErrorIfNeeded(rc, "Extract ripple::AccountID");
output = ripple::AccountID::fromVoid(buf);
}
else if constexpr (std::is_same_v<DecayedType, UCharVectorType>)
{
} else if constexpr (std::is_same_v<DecayedType, UCharVectorType>) {
cass_byte_t const* buf = nullptr;
std::size_t bufSize = 0;
auto const rc = cass_value_get_bytes(cass_row_get_column(row, idx), &buf, &bufSize);
throwErrorIfNeeded(rc, "Extract vector<unsigned char>");
output = UCharVectorType{buf, buf + bufSize};
}
else if constexpr (std::is_same_v<DecayedType, UintTupleType>)
{
} else if constexpr (std::is_same_v<DecayedType, UintTupleType>) {
auto const* tuple = cass_row_get_column(row, idx);
output = TupleIterator::fromTuple(tuple).extract<uint32_t, uint32_t>();
}
else if constexpr (std::is_convertible_v<DecayedType, std::string>)
{
} else if constexpr (std::is_convertible_v<DecayedType, std::string>) {
char const* value = nullptr;
std::size_t len = 0;
auto const rc = cass_value_get_string(cass_row_get_column(row, idx), &value, &len);
throwErrorIfNeeded(rc, "Extract string");
output = std::string{value, len};
}
else if constexpr (std::is_same_v<DecayedType, bool>)
{
} else if constexpr (std::is_same_v<DecayedType, bool>) {
cass_bool_t flag = cass_bool_t::cass_false;
auto const rc = cass_value_get_bool(cass_row_get_column(row, idx), &flag);
throwErrorIfNeeded(rc, "Extract bool");
output = flag != cass_bool_t::cass_false;
}
// clio only uses bigint (int64_t) so we convert any incoming type
else if constexpr (std::is_convertible_v<DecayedType, int64_t>)
{
else if constexpr (std::is_convertible_v<DecayedType, int64_t>) {
int64_t out = 0;
auto const rc = cass_value_get_int64(cass_row_get_column(row, idx), &out);
throwErrorIfNeeded(rc, "Extract int64");
output = static_cast<DecayedType>(out);
}
else
{
} else {
// type not supported for extraction
static_assert(unsupported_v<DecayedType>);
}
@@ -116,8 +101,7 @@ extractColumn(CassRow const* row, std::size_t idx)
return output;
}
struct Result : public ManagedObject<CassResult const>
{
struct Result : public ManagedObject<CassResult const> {
/* implicit */ Result(CassResult const* ptr);
[[nodiscard]] std::size_t
@@ -128,7 +112,8 @@ struct Result : public ManagedObject<CassResult const>
template <typename... RowTypes>
std::optional<std::tuple<RowTypes...>>
get() const requires(std::tuple_size<std::tuple<RowTypes...>>{} > 1)
get() const
requires(std::tuple_size<std::tuple<RowTypes...>>{} > 1)
{
// row managed internally by cassandra driver, hence no ManagedObject.
auto const* row = cass_result_first_row(*this);
@@ -153,8 +138,7 @@ struct Result : public ManagedObject<CassResult const>
}
};
class ResultIterator : public ManagedObject<CassIterator>
{
class ResultIterator : public ManagedObject<CassIterator> {
bool hasMore_ = false;
public:
@@ -185,17 +169,13 @@ public:
};
template <typename... Types>
class ResultExtractor
{
class ResultExtractor {
std::reference_wrapper<Result const> ref_;
public:
struct Sentinel
{
};
struct Sentinel {};
struct Iterator
{
struct Iterator {
using iterator_category = std::input_iterator_tag;
using difference_type = std::size_t; // rows count
using value_type = std::tuple<Types...>;

5
src/data/cassandra/impl/RetryPolicy.h
View File

@@ -35,8 +35,7 @@ namespace data::cassandra::detail {
/**
* @brief A retry policy that employs exponential backoff
*/
class ExponentialBackoffRetryPolicy
{
class ExponentialBackoffRetryPolicy {
util::Logger log_{"Backend"};
boost::asio::steady_timer timer_;
@@ -75,7 +74,7 @@ public:
retry(Fn&& fn)
{
timer_.expires_after(calculateDelay(attempt_++));
timer_.async_wait([fn = std::forward<Fn>(fn)]([[maybe_unused]] const auto& err) {
timer_.async_wait([fn = std::forward<Fn>(fn)]([[maybe_unused]] auto const& err) {
// todo: deal with cancellation (thru err)
fn();
});

3
src/data/cassandra/impl/Session.h
View File

@@ -25,8 +25,7 @@
namespace data::cassandra::detail {
class Session : public ManagedObject<CassSession>
{
class Session : public ManagedObject<CassSession> {
static constexpr auto deleter = [](CassSession* ptr) { cass_session_free(ptr); };
public:

3
src/data/cassandra/impl/SslContext.cpp
View File

@@ -28,8 +28,7 @@ namespace data::cassandra::detail {
SslContext::SslContext(std::string const& certificate) : ManagedObject{cass_ssl_new(), contextDeleter}
{
cass_ssl_set_verify_flags(*this, CASS_SSL_VERIFY_NONE);
if (auto const rc = cass_ssl_add_trusted_cert(*this, certificate.c_str()); rc != CASS_OK)
{
if (auto const rc = cass_ssl_add_trusted_cert(*this, certificate.c_str()); rc != CASS_OK) {
throw std::runtime_error(std::string{"Error setting Cassandra SSL Context: "} + cass_error_desc(rc));
}
}

3
src/data/cassandra/impl/SslContext.h
View File

@@ -27,8 +27,7 @@
namespace data::cassandra::detail {
struct SslContext : public ManagedObject<CassSsl>
{
struct SslContext : public ManagedObject<CassSsl> {
explicit SslContext(std::string const& certificate);
};

40
src/data/cassandra/impl/Statement.h
View File

@@ -35,8 +35,7 @@
namespace data::cassandra::detail {
class Statement : public ManagedObject<CassStatement>
{
class Statement : public ManagedObject<CassStatement> {
static constexpr auto deleter = [](CassStatement* ptr) { cass_statement_free(ptr); };
template <typename>
@@ -101,50 +100,34 @@ public:
using UCharVectorType = std::vector<unsigned char>;
using UintTupleType = std::tuple<uint32_t, uint32_t>;
if constexpr (std::is_same_v<DecayedType, ripple::uint256>)
{
if constexpr (std::is_same_v<DecayedType, ripple::uint256>) {
auto const rc = bindBytes(value.data(), value.size());
throwErrorIfNeeded(rc, "Bind ripple::uint256");
}
else if constexpr (std::is_same_v<DecayedType, ripple::AccountID>)
{
} else if constexpr (std::is_same_v<DecayedType, ripple::AccountID>) {
auto const rc = bindBytes(value.data(), value.size());
throwErrorIfNeeded(rc, "Bind ripple::AccountID");
}
else if constexpr (std::is_same_v<DecayedType, UCharVectorType>)
{
} else if constexpr (std::is_same_v<DecayedType, UCharVectorType>) {
auto const rc = bindBytes(value.data(), value.size());
throwErrorIfNeeded(rc, "Bind vector<unsigned char>");
}
else if constexpr (std::is_convertible_v<DecayedType, std::string>)
{
} else if constexpr (std::is_convertible_v<DecayedType, std::string>) {
// reinterpret_cast is needed here :'(
auto const rc = bindBytes(reinterpret_cast<unsigned char const*>(value.data()), value.size());
throwErrorIfNeeded(rc, "Bind string (as bytes)");
}
else if constexpr (std::is_same_v<DecayedType, UintTupleType>)
{
} else if constexpr (std::is_same_v<DecayedType, UintTupleType>) {
auto const rc = cass_statement_bind_tuple(*this, idx, Tuple{std::forward<Type>(value)});
throwErrorIfNeeded(rc, "Bind tuple<uint32, uint32>");
}
else if constexpr (std::is_same_v<DecayedType, bool>)
{
} else if constexpr (std::is_same_v<DecayedType, bool>) {
auto const rc = cass_statement_bind_bool(*this, idx, value ? cass_true : cass_false);
throwErrorIfNeeded(rc, "Bind bool");
}
else if constexpr (std::is_same_v<DecayedType, Limit>)
{
} else if constexpr (std::is_same_v<DecayedType, Limit>) {
auto const rc = cass_statement_bind_int32(*this, idx, value.limit);
throwErrorIfNeeded(rc, "Bind limit (int32)");
}
// clio only uses bigint (int64_t) so we convert any incoming type
else if constexpr (std::is_convertible_v<DecayedType, int64_t>)
{
else if constexpr (std::is_convertible_v<DecayedType, int64_t>) {
auto const rc = cass_statement_bind_int64(*this, idx, value);
throwErrorIfNeeded(rc, "Bind int64");
}
else
{
} else {
// type not supported for binding
static_assert(unsupported_v<DecayedType>);
}
@@ -156,8 +139,7 @@ public:
*
* This is used to produce Statement objects that can be executed.
*/
class PreparedStatement : public ManagedObject<CassPrepared const>
{
class PreparedStatement : public ManagedObject<CassPrepared const> {
static constexpr auto deleter = [](CassPrepared const* ptr) { cass_prepared_free(ptr); };
public:

29
src/data/cassandra/impl/Tuple.h
View File

@@ -30,8 +30,7 @@
namespace data::cassandra::detail {
class Tuple : public ManagedObject<CassTuple>
{
class Tuple : public ManagedObject<CassTuple> {
static constexpr auto deleter = [](CassTuple* ptr) { cass_tuple_free(ptr); };
template <typename>
@@ -61,8 +60,7 @@ public:
{
using std::to_string;
auto throwErrorIfNeeded = [idx](CassError rc, std::string_view label) {
if (rc != CASS_OK)
{
if (rc != CASS_OK) {
auto const tag = '[' + std::string{label} + ']';
throw std::logic_error(tag + " at idx " + to_string(idx) + ": " + cass_error_desc(rc));
}
@@ -70,27 +68,22 @@ public:
using DecayedType = std::decay_t<Type>;
if constexpr (std::is_same_v<DecayedType, bool>)
{
if constexpr (std::is_same_v<DecayedType, bool>) {
auto const rc = cass_tuple_set_bool(*this, idx, value ? cass_true : cass_false);
throwErrorIfNeeded(rc, "Bind bool");
}
// clio only uses bigint (int64_t) so we convert any incoming type
else if constexpr (std::is_convertible_v<DecayedType, int64_t>)
{
else if constexpr (std::is_convertible_v<DecayedType, int64_t>) {
auto const rc = cass_tuple_set_int64(*this, idx, value);
throwErrorIfNeeded(rc, "Bind int64");
}
else
{
} else {
// type not supported for binding
static_assert(unsupported_v<DecayedType>);
}
}
};
class TupleIterator : public ManagedObject<CassIterator>
{
class TupleIterator : public ManagedObject<CassIterator> {
template <typename>
static constexpr bool unsupported_v = false;
@@ -119,8 +112,7 @@ private:
throw std::logic_error("Could not extract next value from tuple iterator");
auto throwErrorIfNeeded = [](CassError rc, std::string_view label) {
if (rc != CASS_OK)
{
if (rc != CASS_OK) {
auto const tag = '[' + std::string{label} + ']';
throw std::logic_error(tag + ": " + cass_error_desc(rc));
}
@@ -129,15 +121,12 @@ private:
using DecayedType = std::decay_t<Type>;
// clio only uses bigint (int64_t) so we convert any incoming type
if constexpr (std::is_convertible_v<DecayedType, int64_t>)
{
if constexpr (std::is_convertible_v<DecayedType, int64_t>) {
int64_t out = 0;
auto const rc = cass_value_get_int64(cass_iterator_get_value(*this), &out);
throwErrorIfNeeded(rc, "Extract int64 from tuple");
output = static_cast<DecayedType>(out);
}
else
{
} else {
// type not supported for extraction
static_assert(unsupported_v<DecayedType>);
}

16
src/etl/ETLHelpers.h
View File

@@ -36,8 +36,7 @@ namespace etl {
* Any later calls to methods of this datastructure will not wait. Once the datastructure is stopped, the datastructure
* remains stopped for the rest of its lifetime.
*/
class NetworkValidatedLedgers
{
class NetworkValidatedLedgers {
// max sequence validated by network
std::optional<uint32_t> max_;
@@ -95,12 +94,9 @@ public:
{
std::unique_lock lck(m_);
auto pred = [sequence, this]() -> bool { return (max_ && sequence <= *max_); };
if (maxWaitMs)
{
if (maxWaitMs) {
cv_.wait_for(lck, std::chrono::milliseconds(*maxWaitMs));
}
else
{
} else {
cv_.wait(lck, pred);
}
return pred();
@@ -115,8 +111,7 @@ public:
* added or removed from the queue. These waits are blocking calls.
*/
template <class T>
class ThreadSafeQueue
{
class ThreadSafeQueue {
std::queue<T> queue_;
mutable std::mutex m_;
@@ -221,8 +216,7 @@ getMarkers(size_t numMarkers)
std::vector<ripple::uint256> markers;
markers.reserve(numMarkers);
ripple::uint256 base{0};
for (size_t i = 0; i < numMarkers; ++i)
{
for (size_t i = 0; i < numMarkers; ++i) {
markers.push_back(base);
base.data()[0] += incr;
}

100
src/etl/ETLService.cpp
View File

@@ -36,8 +36,7 @@ ETLService::runETLPipeline(uint32_t startSequence, uint32_t numExtractors)
state_.isWriting = true;
auto rng = backend_->hardFetchLedgerRangeNoThrow();
if (!rng || rng->maxSequence < startSequence - 1)
{
if (!rng || rng->maxSequence < startSequence - 1) {
assert(false);
throw std::runtime_error("runETLPipeline: parent ledger is null");
}
@@ -46,10 +45,10 @@ ETLService::runETLPipeline(uint32_t startSequence, uint32_t numExtractors)
auto extractors = std::vector<std::unique_ptr<ExtractorType>>{};
auto pipe = DataPipeType{numExtractors, startSequence};
for (auto i = 0u; i < numExtractors; ++i)
{
for (auto i = 0u; i < numExtractors; ++i) {
extractors.push_back(std::make_unique<ExtractorType>(
pipe, networkValidatedLedgers_, ledgerFetcher_, startSequence + i, finishSequence_, state_));
pipe, networkValidatedLedgers_, ledgerFetcher_, startSequence + i, finishSequence_, state_
));
}
auto transformer =
@@ -85,54 +84,40 @@ void
ETLService::monitor()
{
auto rng = backend_->hardFetchLedgerRangeNoThrow();
if (!rng)
{
if (!rng) {
LOG(log_.info()) << "Database is empty. Will download a ledger from the network.";
std::optional<ripple::LedgerHeader> ledger;
try
{
if (startSequence_)
{
try {
if (startSequence_) {
LOG(log_.info()) << "ledger sequence specified in config. "
<< "Will begin ETL process starting with ledger " << *startSequence_;
ledger = ledgerLoader_.loadInitialLedger(*startSequence_);
}
else
{
} else {
LOG(log_.info()) << "Waiting for next ledger to be validated by network...";
std::optional<uint32_t> mostRecentValidated = networkValidatedLedgers_->getMostRecent();
if (mostRecentValidated)
{
if (mostRecentValidated) {
LOG(log_.info()) << "Ledger " << *mostRecentValidated << " has been validated. Downloading...";
ledger = ledgerLoader_.loadInitialLedger(*mostRecentValidated);
}
else
{
LOG(log_.info()) << "The wait for the next validated ledger has been aborted. Exiting monitor loop";
} else {
LOG(log_.info()) << "The wait for the next validated ledger has been aborted. "
"Exiting monitor loop";
return;
}
}
}
catch (std::runtime_error const& e)
{
} catch (std::runtime_error const& e) {
LOG(log_.fatal()) << "Failed to load initial ledger: " << e.what();
return amendmentBlockHandler_.onAmendmentBlock();
}
if (ledger)
{
if (ledger) {
rng = backend_->hardFetchLedgerRangeNoThrow();
}
else
{
} else {
LOG(log_.error()) << "Failed to load initial ledger. Exiting monitor loop";
return;
}
}
else
{
} else {
if (startSequence_)
LOG(log_.warn()) << "start sequence specified but db is already populated";
@@ -146,8 +131,7 @@ ETLService::monitor()
LOG(log_.debug()) << "Database is populated. "
<< "Starting monitor loop. sequence = " << nextSequence;
while (true)
{
while (true) {
nextSequence = publishNextSequence(nextSequence);
}
}
@@ -155,13 +139,10 @@ ETLService::monitor()
uint32_t
ETLService::publishNextSequence(uint32_t nextSequence)
{
if (auto rng = backend_->hardFetchLedgerRangeNoThrow(); rng && rng->maxSequence >= nextSequence)
{
if (auto rng = backend_->hardFetchLedgerRangeNoThrow(); rng && rng->maxSequence >= nextSequence) {
ledgerPublisher_.publish(nextSequence, {});
++nextSequence;
}
else if (networkValidatedLedgers_->waitUntilValidatedByNetwork(nextSequence, util::MILLISECONDS_PER_SECOND))
{
} else if (networkValidatedLedgers_->waitUntilValidatedByNetwork(nextSequence, util::MILLISECONDS_PER_SECOND)) {
LOG(log_.info()) << "Ledger with sequence = " << nextSequence << " has been validated by the network. "
<< "Attempting to find in database and publish";
@@ -174,8 +155,7 @@ ETLService::publishNextSequence(uint32_t nextSequence)
constexpr size_t timeoutSeconds = 10;
bool const success = ledgerPublisher_.publish(nextSequence, timeoutSeconds);
if (!success)
{
if (!success) {
LOG(log_.warn()) << "Failed to publish ledger with sequence = " << nextSequence << " . Beginning ETL";
// returns the most recent sequence published empty optional if no sequence was published
@@ -185,9 +165,7 @@ ETLService::publishNextSequence(uint32_t nextSequence)
// if no ledger was published, don't increment nextSequence
if (lastPublished)
nextSequence = *lastPublished + 1;
}
else
{
} else {
++nextSequence;
}
}
@@ -199,13 +177,11 @@ ETLService::monitorReadOnly()
{
LOG(log_.debug()) << "Starting reporting in strict read only mode";
const auto latestSequenceOpt = [this]() -> std::optional<uint32_t> {
auto const latestSequenceOpt = [this]() -> std::optional<uint32_t> {
auto rng = backend_->hardFetchLedgerRangeNoThrow();
if (!rng)
{
if (auto net = networkValidatedLedgers_->getMostRecent())
{
if (!rng) {
if (auto net = networkValidatedLedgers_->getMostRecent()) {
return *net;
}
return std::nullopt;
@@ -214,8 +190,7 @@ ETLService::monitorReadOnly()
return rng->maxSequence;
}();
if (!latestSequenceOpt.has_value())
{
if (!latestSequenceOpt.has_value()) {
return;
}
@@ -224,17 +199,14 @@ ETLService::monitorReadOnly()
cacheLoader_.load(latestSequence);
latestSequence++;
while (true)
{
if (auto rng = backend_->hardFetchLedgerRangeNoThrow(); rng && rng->maxSequence >= latestSequence)
{
while (true) {
if (auto rng = backend_->hardFetchLedgerRangeNoThrow(); rng && rng->maxSequence >= latestSequence) {
ledgerPublisher_.publish(latestSequence, {});
latestSequence = latestSequence + 1;
}
else
{
// if we can't, wait until it's validated by the network, or 1 second passes, whichever occurs first.
// Even if we don't hear from rippled, if ledgers are being written to the db, we publish them.
} else {
// if we can't, wait until it's validated by the network, or 1 second passes, whichever occurs
// first. Even if we don't hear from rippled, if ledgers are being written to the db, we publish
// them.
networkValidatedLedgers_->waitUntilValidatedByNetwork(latestSequence, util::MILLISECONDS_PER_SECOND);
}
}
@@ -255,12 +227,9 @@ ETLService::doWork()
worker_ = std::thread([this]() {
beast::setCurrentThreadName("ETLService worker");
if (state_.isReadOnly)
{
if (state_.isReadOnly) {
monitorReadOnly();
}
else
{
} else {
monitor();
}
});
@@ -272,7 +241,8 @@ ETLService::ETLService(
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManagerType> subscriptions,
std::shared_ptr<LoadBalancerType> balancer,
std::shared_ptr<NetworkValidatedLedgersType> ledgers)
std::shared_ptr<NetworkValidatedLedgersType> ledgers
)
: backend_(backend)
, loadBalancer_(balancer)
, networkValidatedLedgers_(std::move(ledgers))

9
src/etl/ETLService.h
View File

@@ -66,8 +66,7 @@ namespace etl {
* the others will fall back to monitoring/publishing. In this sense, this class dynamically transitions from monitoring
* to writing and from writing to monitoring, based on the activity of other processes running on different machines.
*/
class ETLService
{
class ETLService {
// TODO: make these template parameters in ETLService
using SubscriptionManagerType = feed::SubscriptionManager;
using LoadBalancerType = LoadBalancer;
@@ -122,7 +121,8 @@ public:
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManagerType> subscriptions,
std::shared_ptr<LoadBalancerType> balancer,
std::shared_ptr<NetworkValidatedLedgersType> ledgers);
std::shared_ptr<NetworkValidatedLedgersType> ledgers
);
/**
* @brief A factory function to spawn new ETLService instances.
@@ -143,7 +143,8 @@ public:
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManagerType> subscriptions,
std::shared_ptr<LoadBalancerType> balancer,
std::shared_ptr<NetworkValidatedLedgersType> ledgers)
std::shared_ptr<NetworkValidatedLedgersType> ledgers
)
{
auto etl = std::make_shared<ETLService>(config, ioc, backend, subscriptions, balancer, ledgers);
etl->run();

6
src/etl/ETLState.cpp
View File

@@ -43,10 +43,8 @@ tag_invoke(boost::json::value_to_tag<ETLState>, boost::json::value const& jv)
ETLState state;
auto const& jsonObject = jv.as_object();
if (!jsonObject.contains(JS(error)))
{
if (jsonObject.contains(JS(result)) && jsonObject.at(JS(result)).as_object().contains(JS(info)))
{
if (!jsonObject.contains(JS(error))) {
if (jsonObject.contains(JS(result)) && jsonObject.at(JS(result)).as_object().contains(JS(info))) {
auto const rippledInfo = jsonObject.at(JS(result)).as_object().at(JS(info)).as_object();
if (rippledInfo.contains(JS(network_id)))
state.networkID.emplace(boost::json::value_to<int64_t>(rippledInfo.at(JS(network_id))));

3
src/etl/ETLState.h
View File

@@ -31,8 +31,7 @@ class Source;
/**
* @brief This class is responsible for fetching and storing the state of the ETL information, such as the network id
*/
struct ETLState
{
struct ETLState {
std::optional<uint32_t> networkID;
/**

72
src/etl/LoadBalancer.cpp
View File

@@ -47,7 +47,8 @@ LoadBalancer::make_Source(
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManager> subscriptions,
std::shared_ptr<NetworkValidatedLedgers> validatedLedgers,
LoadBalancer& balancer)
LoadBalancer& balancer
)
{
auto src = std::make_unique<ProbingSource>(config, ioc, backend, subscriptions, validatedLedgers, balancer);
src->run();
@@ -61,7 +62,8 @@ LoadBalancer::make_LoadBalancer(
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManager> subscriptions,
std::shared_ptr<NetworkValidatedLedgers> validatedLedgers)
std::shared_ptr<NetworkValidatedLedgers> validatedLedgers
)
{
return std::make_shared<LoadBalancer>(config, ioc, backend, subscriptions, validatedLedgers);
}
@@ -71,34 +73,29 @@ LoadBalancer::LoadBalancer(
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManager> subscriptions,
std::shared_ptr<NetworkValidatedLedgers> validatedLedgers)
std::shared_ptr<NetworkValidatedLedgers> validatedLedgers
)
{
static constexpr std::uint32_t MAX_DOWNLOAD = 256;
if (auto value = config.maybeValue<uint32_t>("num_markers"); value)
{
if (auto value = config.maybeValue<uint32_t>("num_markers"); value) {
downloadRanges_ = std::clamp(*value, 1u, MAX_DOWNLOAD);
}
else if (backend->fetchLedgerRange())
{
} else if (backend->fetchLedgerRange()) {
downloadRanges_ = 4;
}
for (auto const& entry : config.array("etl_sources"))
{
for (auto const& entry : config.array("etl_sources")) {
std::unique_ptr<Source> source = make_Source(entry, ioc, backend, subscriptions, validatedLedgers, *this);
// checking etl node validity
auto const state = ETLState::fetchETLStateFromSource(*source);
if (!state.networkID)
{
if (!state.networkID) {
LOG(log_.error()) << "Failed to fetch ETL state from source = " << source->toString()
<< " Please check the configuration and network";
throw std::logic_error("ETL node not available");
}
if (etlState_ && etlState_->networkID != state.networkID)
{
if (etlState_ && etlState_->networkID != state.networkID) {
LOG(log_.error()) << "ETL sources must be on the same network. "
<< "Source network id = " << *(state.networkID)
<< " does not match others network id = " << *(etlState_->networkID);
@@ -109,8 +106,7 @@ LoadBalancer::LoadBalancer(
LOG(log_.info()) << "Added etl source - " << sources_.back()->toString();
}
if (sources_.empty())
{
if (sources_.empty()) {
LOG(log_.error()) << "No ETL sources configured. Please check the configuration";
throw std::logic_error("No ETL sources configured");
}
@@ -129,19 +125,17 @@ LoadBalancer::loadInitialLedger(uint32_t sequence, bool cacheOnly)
[this, &response, &sequence, cacheOnly](auto& source) {
auto [data, res] = source->loadInitialLedger(sequence, downloadRanges_, cacheOnly);
if (!res)
{
if (!res) {
LOG(log_.error()) << "Failed to download initial ledger."
<< " Sequence = " << sequence << " source = " << source->toString();
}
else
{
} else {
response = std::move(data);
}
return res;
},
sequence);
sequence
);
return {std::move(response), success};
}
@@ -153,8 +147,7 @@ LoadBalancer::fetchLedger(uint32_t ledgerSequence, bool getObjects, bool getObje
[&response, ledgerSequence, getObjects, getObjectNeighbors, log = log_](auto& source) {
auto [status, data] = source->fetchLedger(ledgerSequence, getObjects, getObjectNeighbors);
response = std::move(data);
if (status.ok() && response.validated())
{
if (status.ok() && response.validated()) {
LOG(log.info()) << "Successfully fetched ledger = " << ledgerSequence
<< " from source = " << source->toString();
return true;
@@ -165,9 +158,9 @@ LoadBalancer::fetchLedger(uint32_t ledgerSequence, bool getObjects, bool getObje
<< ", source = " << source->toString();
return false;
},
ledgerSequence);
if (success)
{
ledgerSequence
);
if (success) {
return response;
}
return {};
@@ -177,14 +170,14 @@ std::optional<boost::json::object>
LoadBalancer::forwardToRippled(
boost::json::object const& request,
std::optional<std::string> const& clientIp,
boost::asio::yield_context yield) const
boost::asio::yield_context yield
) const
{
srand(static_cast<unsigned>(time(0)));
auto sourceIdx = rand() % sources_.size();
auto numAttempts = 0u;
while (numAttempts < sources_.size())
{
while (numAttempts < sources_.size()) {
if (auto res = sources_[sourceIdx]->forwardToRippled(request, clientIp, yield))
return res;
@@ -198,8 +191,7 @@ LoadBalancer::forwardToRippled(
bool
LoadBalancer::shouldPropagateTxnStream(Source* in) const
{
for (auto& src : sources_)
{
for (auto& src : sources_) {
assert(src);
// We pick the first Source encountered that is connected
@@ -229,8 +221,7 @@ LoadBalancer::execute(Func f, uint32_t ledgerSequence)
auto sourceIdx = rand() % sources_.size();
auto numAttempts = 0;
while (true)
{
while (true) {
auto& source = sources_[sourceIdx];
LOG(log_.debug()) << "Attempting to execute func. ledger sequence = " << ledgerSequence
@@ -239,11 +230,9 @@ LoadBalancer::execute(Func f, uint32_t ledgerSequence)
/* Sometimes rippled has ledger but doesn't actually know. However,
but this does NOT happen in the normal case and is safe to remove
This || true is only needed when loading full history standalone */
if (source->hasLedger(ledgerSequence))
{
if (source->hasLedger(ledgerSequence)) {
bool const res = f(source);
if (res)
{
if (res) {
LOG(log_.debug()) << "Successfully executed func at source = " << source->toString()
<< " - ledger sequence = " << ledgerSequence;
break;
@@ -251,16 +240,13 @@ LoadBalancer::execute(Func f, uint32_t ledgerSequence)
LOG(log_.warn()) << "Failed to execute func at source = " << source->toString()
<< " - ledger sequence = " << ledgerSequence;
}
else
{
} else {
LOG(log_.warn()) << "Ledger not present at source = " << source->toString()
<< " - ledger sequence = " << ledgerSequence;
}
sourceIdx = (sourceIdx + 1) % sources_.size();
numAttempts++;
if (numAttempts % sources_.size() == 0)
{
if (numAttempts % sources_.size() == 0) {
LOG(log_.info()) << "Ledger sequence " << ledgerSequence
<< " is not yet available from any configured sources. "
<< "Sleeping and trying again";

15
src/etl/LoadBalancer.h
View File

@@ -48,8 +48,7 @@ namespace etl {
* which ledgers have been validated by the network, and the range of ledgers each etl source has). This class also
* allows requests for ledger data to be load balanced across all possible ETL sources.
*/
class LoadBalancer
{
class LoadBalancer {
public:
using RawLedgerObjectType = org::xrpl::rpc::v1::RawLedgerObject;
using GetLedgerResponseType = org::xrpl::rpc::v1::GetLedgerResponse;
@@ -79,7 +78,8 @@ public:
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManager> subscriptions,
std::shared_ptr<NetworkValidatedLedgers> validatedLedgers);
std::shared_ptr<NetworkValidatedLedgers> validatedLedgers
);
/**
* @brief A factory function for the load balancer.
@@ -96,7 +96,8 @@ public:
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManager> subscriptions,
std::shared_ptr<NetworkValidatedLedgers> validatedLedgers);
std::shared_ptr<NetworkValidatedLedgers> validatedLedgers
);
/**
* @brief A factory function for the ETL source.
@@ -115,7 +116,8 @@ public:
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManager> subscriptions,
std::shared_ptr<NetworkValidatedLedgers> validatedLedgers,
LoadBalancer& balancer);
LoadBalancer& balancer
);
~LoadBalancer();
@@ -174,7 +176,8 @@ public:
forwardToRippled(
boost::json::object const& request,
std::optional<std::string> const& clientIp,
boost::asio::yield_context yield) const;
boost::asio::yield_context yield
) const;
/**
* @brief Return state of ETL nodes.

79
src/etl/NFTHelpers.cpp
View File

@@ -45,27 +45,26 @@ getNFTokenMintData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
// that were changed.
std::optional<ripple::AccountID> owner;
for (ripple::STObject const& node : txMeta.getNodes())
{
for (ripple::STObject const& node : txMeta.getNodes()) {
if (node.getFieldU16(ripple::sfLedgerEntryType) != ripple::ltNFTOKEN_PAGE)
continue;
if (!owner)
owner = ripple::AccountID::fromVoid(node.getFieldH256(ripple::sfLedgerIndex).data());
if (node.getFName() == ripple::sfCreatedNode)
{
if (node.getFName() == ripple::sfCreatedNode) {
ripple::STArray const& toAddNFTs =
node.peekAtField(ripple::sfNewFields).downcast<ripple::STObject>().getFieldArray(ripple::sfNFTokens);
std::transform(
toAddNFTs.begin(), toAddNFTs.end(), std::back_inserter(finalIDs), [](ripple::STObject const& nft) {
return nft.getFieldH256(ripple::sfNFTokenID);
});
toAddNFTs.begin(),
toAddNFTs.end(),
std::back_inserter(finalIDs),
[](ripple::STObject const& nft) { return nft.getFieldH256(ripple::sfNFTokenID); }
);
}
// Else it's modified, as there should never be a deleted NFToken page
// as a result of a mint.
else
{
else {
// When a mint results in splitting an existing page,
// it results in a created page and a modified node. Sometimes,
// the created node needs to be linked to a third page, resulting
@@ -82,9 +81,11 @@ getNFTokenMintData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
ripple::STArray const& toAddNFTs = previousFields.getFieldArray(ripple::sfNFTokens);
std::transform(
toAddNFTs.begin(), toAddNFTs.end(), std::back_inserter(prevIDs), [](ripple::STObject const& nft) {
return nft.getFieldH256(ripple::sfNFTokenID);
});
toAddNFTs.begin(),
toAddNFTs.end(),
std::back_inserter(prevIDs),
[](ripple::STObject const& nft) { return nft.getFieldH256(ripple::sfNFTokenID); }
);
ripple::STArray const& toAddFinalNFTs =
node.peekAtField(ripple::sfFinalFields).downcast<ripple::STObject>().getFieldArray(ripple::sfNFTokens);
@@ -92,7 +93,8 @@ getNFTokenMintData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
toAddFinalNFTs.begin(),
toAddFinalNFTs.end(),
std::back_inserter(finalIDs),
[](ripple::STObject const& nft) { return nft.getFieldH256(ripple::sfNFTokenID); });
[](ripple::STObject const& nft) { return nft.getFieldH256(ripple::sfNFTokenID); }
);
}
}
@@ -105,10 +107,9 @@ getNFTokenMintData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
// There should always be a difference so the returned finalIDs
// iterator should never be end(). But better safe than sorry.
if (finalIDs.size() != prevIDs.size() + 1 || diff.first == finalIDs.end() || !owner)
{
throw std::runtime_error(
fmt::format(" - unexpected NFTokenMint data in tx {}", strHex(sttx.getTransactionID())));
if (finalIDs.size() != prevIDs.size() + 1 || diff.first == finalIDs.end() || !owner) {
throw std::runtime_error(fmt::format(" - unexpected NFTokenMint data in tx {}", strHex(sttx.getTransactionID()))
);
}
return {
@@ -125,8 +126,7 @@ getNFTokenBurnData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
// Determine who owned the token when it was burned by finding an
// NFTokenPage that was deleted or modified that contains this
// tokenID.
for (ripple::STObject const& node : txMeta.getNodes())
{
for (ripple::STObject const& node : txMeta.getNodes()) {
if (node.getFieldU16(ripple::sfLedgerEntryType) != ripple::ltNFTOKEN_PAGE ||
node.getFName() == ripple::sfCreatedNode)
continue;
@@ -141,15 +141,12 @@ getNFTokenBurnData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
// need to look in the FinalFields.
std::optional<ripple::STArray> prevNFTs;
if (node.isFieldPresent(ripple::sfPreviousFields))
{
if (node.isFieldPresent(ripple::sfPreviousFields)) {
ripple::STObject const& previousFields =
node.peekAtField(ripple::sfPreviousFields).downcast<ripple::STObject>();
if (previousFields.isFieldPresent(ripple::sfNFTokens))
prevNFTs = previousFields.getFieldArray(ripple::sfNFTokens);
}
else if (!prevNFTs && node.getFName() == ripple::sfDeletedNode)
{
} else if (!prevNFTs && node.getFName() == ripple::sfDeletedNode) {
prevNFTs =
node.peekAtField(ripple::sfFinalFields).downcast<ripple::STObject>().getFieldArray(ripple::sfNFTokens);
}
@@ -161,15 +158,13 @@ getNFTokenBurnData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
std::find_if(prevNFTs->begin(), prevNFTs->end(), [&tokenID](ripple::STObject const& candidate) {
return candidate.getFieldH256(ripple::sfNFTokenID) == tokenID;
});
if (nft != prevNFTs->end())
{
if (nft != prevNFTs->end()) {
return std::make_pair(
txs,
NFTsData(
tokenID,
ripple::AccountID::fromVoid(node.getFieldH256(ripple::sfLedgerIndex).data()),
txMeta,
true));
tokenID, ripple::AccountID::fromVoid(node.getFieldH256(ripple::sfLedgerIndex).data()), txMeta, true
)
);
}
}
@@ -184,14 +179,12 @@ getNFTokenAcceptOfferData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx
// If we have the buy offer from this tx, we can determine the owner
// more easily by just looking at the owner of the accepted NFTokenOffer
// object.
if (sttx.isFieldPresent(ripple::sfNFTokenBuyOffer))
{
if (sttx.isFieldPresent(ripple::sfNFTokenBuyOffer)) {
auto const affectedBuyOffer =
std::find_if(txMeta.getNodes().begin(), txMeta.getNodes().end(), [&sttx](ripple::STObject const& node) {
return node.getFieldH256(ripple::sfLedgerIndex) == sttx.getFieldH256(ripple::sfNFTokenBuyOffer);
});
if (affectedBuyOffer == txMeta.getNodes().end())
{
if (affectedBuyOffer == txMeta.getNodes().end()) {
std::stringstream msg;
msg << " - unexpected NFTokenAcceptOffer data in tx " << sttx.getTransactionID();
throw std::runtime_error(msg.str());
@@ -213,8 +206,7 @@ getNFTokenAcceptOfferData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx
std::find_if(txMeta.getNodes().begin(), txMeta.getNodes().end(), [&sttx](ripple::STObject const& node) {
return node.getFieldH256(ripple::sfLedgerIndex) == sttx.getFieldH256(ripple::sfNFTokenSellOffer);
});
if (affectedSellOffer == txMeta.getNodes().end())
{
if (affectedSellOffer == txMeta.getNodes().end()) {
std::stringstream msg;
msg << " - unexpected NFTokenAcceptOffer data in tx " << sttx.getTransactionID();
throw std::runtime_error(msg.str());
@@ -228,8 +220,7 @@ getNFTokenAcceptOfferData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx
.downcast<ripple::STObject>()
.getAccountID(ripple::sfOwner);
for (ripple::STObject const& node : txMeta.getNodes())
{
for (ripple::STObject const& node : txMeta.getNodes()) {
if (node.getFieldU16(ripple::sfLedgerEntryType) != ripple::ltNFTOKEN_PAGE ||
node.getFName() == ripple::sfDeletedNode)
continue;
@@ -240,8 +231,7 @@ getNFTokenAcceptOfferData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx
continue;
ripple::STArray const& nfts = [&node] {
if (node.getFName() == ripple::sfCreatedNode)
{
if (node.getFName() == ripple::sfCreatedNode) {
return node.peekAtField(ripple::sfNewFields)
.downcast<ripple::STObject>()
.getFieldArray(ripple::sfNFTokens);
@@ -254,8 +244,7 @@ getNFTokenAcceptOfferData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx
auto const nft = std::find_if(nfts.begin(), nfts.end(), [&tokenID](ripple::STObject const& candidate) {
return candidate.getFieldH256(ripple::sfNFTokenID) == tokenID;
});
if (nft != nfts.end())
{
if (nft != nfts.end()) {
return {
{NFTTransactionsData(tokenID, txMeta, sttx.getTransactionID())},
NFTsData(tokenID, nodeOwner, txMeta, false)};
@@ -275,8 +264,7 @@ std::pair<std::vector<NFTTransactionsData>, std::optional<NFTsData>>
getNFTokenCancelOfferData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
{
std::vector<NFTTransactionsData> txs;
for (ripple::STObject const& node : txMeta.getNodes())
{
for (ripple::STObject const& node : txMeta.getNodes()) {
if (node.getFieldU16(ripple::sfLedgerEntryType) != ripple::ltNFTOKEN_OFFER)
continue;
@@ -312,8 +300,7 @@ getNFTDataFromTx(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
if (txMeta.getResultTER() != ripple::tesSUCCESS)
return {{}, {}};
switch (sttx.getTxnType())
{
switch (sttx.getTxnType()) {
case ripple::TxType::ttNFTOKEN_MINT:
return getNFTokenMintData(txMeta, sttx);

24
src/etl/ProbingSource.cpp
View File

@@ -28,7 +28,8 @@ ProbingSource::ProbingSource(
std::shared_ptr<feed::SubscriptionManager> subscriptions,
std::shared_ptr<NetworkValidatedLedgers> nwvl,
LoadBalancer& balancer,
boost::asio::ssl::context sslCtx)
boost::asio::ssl::context sslCtx
)
: sslCtx_{std::move(sslCtx)}
, sslSrc_{make_shared<
SslSource>(config, ioc, std::ref(sslCtx_), backend, subscriptions, nwvl, balancer, make_SSLHooks())}
@@ -74,8 +75,7 @@ ProbingSource::hasLedger(uint32_t sequence) const
boost::json::object
ProbingSource::toJson() const
{
if (!currentSrc_)
{
if (!currentSrc_) {
boost::json::object sourcesJson = {
{"ws", plainSrc_->toJson()},
{"wss", sslSrc_->toJson()},
@@ -124,7 +124,8 @@ std::optional<boost::json::object>
ProbingSource::forwardToRippled(
boost::json::object const& request,
std::optional<std::string> const& clientIp,
boost::asio::yield_context yield) const
boost::asio::yield_context yield
) const
{
if (!currentSrc_) // Source may connect to rippled before the connection built to check the validity
{
@@ -140,7 +141,8 @@ std::optional<boost::json::object>
ProbingSource::requestFromRippled(
boost::json::object const& request,
std::optional<std::string> const& clientIp,
boost::asio::yield_context yield) const
boost::asio::yield_context yield
) const
{
if (!currentSrc_)
return {};
@@ -156,8 +158,7 @@ ProbingSource::make_SSLHooks() noexcept
if (currentSrc_)
return SourceHooks::Action::STOP;
if (!ec)
{
if (!ec) {
plainSrc_->pause();
currentSrc_ = sslSrc_;
LOG(log_.info()) << "Selected WSS as the main source: " << currentSrc_->toString();
@@ -167,8 +168,7 @@ ProbingSource::make_SSLHooks() noexcept
// onDisconnected
[this](auto /* ec */) {
std::lock_guard const lck(mtx_);
if (currentSrc_)
{
if (currentSrc_) {
currentSrc_ = nullptr;
plainSrc_->resume();
}
@@ -185,8 +185,7 @@ ProbingSource::make_PlainHooks() noexcept
if (currentSrc_)
return SourceHooks::Action::STOP;
if (!ec)
{
if (!ec) {
sslSrc_->pause();
currentSrc_ = plainSrc_;
LOG(log_.info()) << "Selected Plain WS as the main source: " << currentSrc_->toString();
@@ -196,8 +195,7 @@ ProbingSource::make_PlainHooks() noexcept
// onDisconnected
[this](auto /* ec */) {
std::lock_guard const lck(mtx_);
if (currentSrc_)
{
if (currentSrc_) {
currentSrc_ = nullptr;
sslSrc_->resume();
}

12
src/etl/ProbingSource.h
View File

@@ -39,8 +39,7 @@ namespace etl {
* First to connect pauses the other and the probing is considered done at this point.
* If however the connected source loses connection the probing is kickstarted again.
*/
class ProbingSource : public Source
{
class ProbingSource : public Source {
public:
// TODO: inject when unit tests will be written for ProbingSource
using GetLedgerResponseType = org::xrpl::rpc::v1::GetLedgerResponse;
@@ -73,7 +72,8 @@ public:
std::shared_ptr<feed::SubscriptionManager> subscriptions,
std::shared_ptr<NetworkValidatedLedgers> nwvl,
LoadBalancer& balancer,
boost::asio::ssl::context sslCtx = boost::asio::ssl::context{boost::asio::ssl::context::tlsv12});
boost::asio::ssl::context sslCtx = boost::asio::ssl::context{boost::asio::ssl::context::tlsv12}
);
~ProbingSource() override = default;
@@ -108,7 +108,8 @@ public:
forwardToRippled(
boost::json::object const& request,
std::optional<std::string> const& clientIp,
boost::asio::yield_context yield) const override;
boost::asio::yield_context yield
) const override;
boost::uuids::uuid
token() const override;
@@ -118,7 +119,8 @@ private:
requestFromRippled(
boost::json::object const& request,
std::optional<std::string> const& clientIp,
boost::asio::yield_context yield) const override;
boost::asio::yield_context yield
) const override;
SourceHooks
make_SSLHooks() noexcept;

68
src/etl/Source.cpp
View File

@@ -50,27 +50,22 @@ PlainSource::close(bool startAgain)
if (closing_)
return;
if (derived().ws().is_open())
{
if (derived().ws().is_open()) {
// onStop() also calls close(). If the async_close is called twice,
// an assertion fails. Using closing_ makes sure async_close is only
// called once
closing_ = true;
derived().ws().async_close(boost::beast::websocket::close_code::normal, [this, startAgain](auto ec) {
if (ec)
{
if (ec) {
LOG(log_.error()) << "async_close: error code = " << ec << " - " << toString();
}
closing_ = false;
if (startAgain)
{
if (startAgain) {
ws_ = std::make_unique<StreamType>(strand_);
run();
}
});
}
else if (startAgain)
{
} else if (startAgain) {
ws_ = std::make_unique<StreamType>(strand_);
run();
}
@@ -85,26 +80,21 @@ SslSource::close(bool startAgain)
if (closing_)
return;
if (derived().ws().is_open())
{
// onStop() also calls close(). If the async_close is called twice, an assertion fails. Using closing_ makes
// sure async_close is only called once
if (derived().ws().is_open()) {
// onStop() also calls close(). If the async_close is called twice, an assertion fails. Using closing_
// makes sure async_close is only called once
closing_ = true;
derived().ws().async_close(boost::beast::websocket::close_code::normal, [this, startAgain](auto ec) {
if (ec)
{
if (ec) {
LOG(log_.error()) << "async_close: error code = " << ec << " - " << toString();
}
closing_ = false;
if (startAgain)
{
if (startAgain) {
ws_ = std::make_unique<StreamType>(strand_, *sslCtx_);
run();
}
});
}
else if (startAgain)
{
} else if (startAgain) {
ws_ = std::make_unique<StreamType>(strand_, *sslCtx_);
run();
}
@@ -114,15 +104,13 @@ SslSource::close(bool startAgain)
void
PlainSource::onConnect(
boost::beast::error_code ec,
boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint)
boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint
)
{
if (ec)
{
if (ec) {
// start over
reconnect(ec);
}
else
{
} else {
connected_ = true;
numFailures_ = 0;
@@ -134,7 +122,8 @@ PlainSource::onConnect(
boost::beast::websocket::stream_base::decorator([](boost::beast::websocket::request_type& req) {
req.set(boost::beast::http::field::user_agent, "clio-client");
req.set("X-User", "clio-client");
}));
})
);
// Update the host_ string. This will provide the value of the
// Host HTTP header during the WebSocket handshake.
@@ -147,13 +136,10 @@ PlainSource::onConnect(
void
SslSource::onConnect(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint)
{
if (ec)
{
if (ec) {
// start over
reconnect(ec);
}
else
{
} else {
connected_ = true;
numFailures_ = 0;
@@ -165,28 +151,28 @@ SslSource::onConnect(boost::beast::error_code ec, boost::asio::ip::tcp::resolver
boost::beast::websocket::stream_base::decorator([](boost::beast::websocket::request_type& req) {
req.set(boost::beast::http::field::user_agent, "clio-client");
req.set("X-User", "clio-client");
}));
})
);
// Update the host_ string. This will provide the value of the
// Host HTTP header during the WebSocket handshake.
// See https://tools.ietf.org/html/rfc7230#section-5.4
auto host = ip_ + ':' + std::to_string(endpoint.port());
ws().next_layer().async_handshake(
boost::asio::ssl::stream_base::client, [this, endpoint](auto ec) { onSslHandshake(ec, endpoint); });
ws().next_layer().async_handshake(boost::asio::ssl::stream_base::client, [this, endpoint](auto ec) {
onSslHandshake(ec, endpoint);
});
}
}
void
SslSource::onSslHandshake(
boost::beast::error_code ec,
boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint)
boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint
)
{
if (ec)
{
if (ec) {
reconnect(ec);
}
else
{
} else {
auto host = ip_ + ':' + std::to_string(endpoint.port());
ws().async_handshake(host, "/", [this](auto ec) { onHandshake(ec); });
}

204
src/etl/Source.h
View File

@@ -60,8 +60,7 @@ class ProbingSource;
* Note: Since sources below are implemented via CRTP, it sort of makes no sense to have a virtual base class.
* We should consider using a vector of ProbingSources instead of vector of unique ptrs to this virtual base.
*/
class Source
{
class Source {
public:
/** @return true if source is connected; false otherwise */
virtual bool
@@ -133,7 +132,8 @@ public:
forwardToRippled(
boost::json::object const& request,
std::optional<std::string> const& forwardToRippledclientIp,
boost::asio::yield_context yield) const = 0;
boost::asio::yield_context yield
) const = 0;
/**
* @return A token that uniquely identifies this source instance.
@@ -166,14 +166,14 @@ private:
requestFromRippled(
boost::json::object const& request,
std::optional<std::string> const& clientIp,
boost::asio::yield_context yield) const = 0;
boost::asio::yield_context yield
) const = 0;
};
/**
* @brief Hooks for source events such as connects and disconnects.
*/
struct SourceHooks
{
struct SourceHooks {
enum class Action { STOP, PROCEED };
std::function<Action(boost::beast::error_code)> onConnected;
@@ -186,8 +186,7 @@ struct SourceHooks
* @tparam Derived The derived class for CRTP
*/
template <class Derived>
class SourceImpl : public Source
{
class SourceImpl : public Source {
std::string wsPort_;
std::string grpcPort_;
@@ -248,7 +247,8 @@ public:
std::shared_ptr<feed::SubscriptionManager> subscriptions,
std::shared_ptr<NetworkValidatedLedgers> validatedLedgers,
LoadBalancer& balancer,
SourceHooks hooks)
SourceHooks hooks
)
: networkValidatedLedgers_(std::move(validatedLedgers))
, backend_(std::move(backend))
, subscriptions_(std::move(subscriptions))
@@ -265,22 +265,19 @@ public:
ip_ = config.valueOr<std::string>("ip", {});
wsPort_ = config.valueOr<std::string>("ws_port", {});
if (auto value = config.maybeValue<std::string>("grpc_port"); value)
{
if (auto value = config.maybeValue<std::string>("grpc_port"); value) {
grpcPort_ = *value;
try
{
try {
boost::asio::ip::tcp::endpoint const endpoint{boost::asio::ip::make_address(ip_), std::stoi(grpcPort_)};
std::stringstream ss;
ss << endpoint;
grpc::ChannelArguments chArgs;
chArgs.SetMaxReceiveMessageSize(-1);
stub_ = org::xrpl::rpc::v1::XRPLedgerAPIService::NewStub(
grpc::CreateCustomChannel(ss.str(), grpc::InsecureChannelCredentials(), chArgs));
grpc::CreateCustomChannel(ss.str(), grpc::InsecureChannelCredentials(), chArgs)
);
LOG(log_.debug()) << "Made stub for remote = " << toString();
}
catch (std::exception const& e)
{
} catch (std::exception const& e) {
LOG(log_.debug()) << "Exception while creating stub = " << e.what() << " . Remote = " << toString();
}
}
@@ -307,7 +304,8 @@ public:
requestFromRippled(
boost::json::object const& request,
std::optional<std::string> const& clientIp,
boost::asio::yield_context yield) const override
boost::asio::yield_context yield
) const override
{
LOG(log_.trace()) << "Attempting to forward request to tx. Request = " << boost::json::serialize(request);
@@ -319,8 +317,7 @@ public:
namespace net = boost::asio;
using tcp = boost::asio::ip::tcp;
try
{
try {
auto executor = boost::asio::get_associated_executor(yield);
beast::error_code ec;
tcp::resolver resolver{executor};
@@ -336,8 +333,8 @@ public:
if (ec)
return {};
// if client ip is know, change the User-Agent of the handshake and to tell rippled to charge the client IP
// for RPC resources. See "secure_gateway" in
// if client ip is know, change the User-Agent of the handshake and to tell rippled to charge the client
// IP for RPC resources. See "secure_gateway" in
// https://github.com/ripple/rippled/blob/develop/cfg/rippled-example.cfg
// TODO: user-agent can be clio-[version]
@@ -364,8 +361,7 @@ public:
auto end = begin + buffer.data().size();
auto parsed = boost::json::parse(std::string(begin, end));
if (!parsed.is_object())
{
if (!parsed.is_object()) {
LOG(log_.error()) << "Error parsing response: " << std::string{begin, end};
return {};
}
@@ -374,9 +370,7 @@ public:
response["forwarded"] = true;
return response;
}
catch (std::exception const& e)
{
} catch (std::exception const& e) {
LOG(log_.error()) << "Encountered exception : " << e.what();
return {};
}
@@ -386,14 +380,11 @@ public:
hasLedger(uint32_t sequence) const override
{
std::lock_guard const lck(mtx_);
for (auto& pair : validatedLedgers_)
{
if (sequence >= pair.first && sequence <= pair.second)
{
for (auto& pair : validatedLedgers_) {
if (sequence >= pair.first && sequence <= pair.second) {
return true;
}
if (sequence < pair.first)
{
if (sequence < pair.first) {
// validatedLedgers_ is a sorted list of disjoint ranges
// if the sequence comes before this range, the sequence will
// come before all subsequent ranges
@@ -423,11 +414,10 @@ public:
grpc::Status const status = stub_->GetLedger(&context, request, &response);
if (status.ok() && !response.is_unlimited())
{
log_.warn()
<< "is_unlimited is false. Make sure secure_gateway is set correctly on the ETL source. source = "
<< toString() << "; status = " << status.error_message();
if (status.ok() && !response.is_unlimited()) {
log_.warn(
) << "is_unlimited is false. Make sure secure_gateway is set correctly on the ETL source. source = "
<< toString() << "; status = " << status.error_message();
}
return {status, std::move(response)};
@@ -452,11 +442,11 @@ public:
res["grpc_port"] = grpcPort_;
auto last = getLastMsgTime();
if (last.time_since_epoch().count() != 0)
{
if (last.time_since_epoch().count() != 0) {
res["last_msg_age_seconds"] = std::to_string(
std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now() - getLastMsgTime())
.count());
.count()
);
}
return res;
@@ -474,8 +464,7 @@ public:
std::vector<etl::detail::AsyncCallData> calls;
auto markers = getMarkers(numMarkers);
for (size_t i = 0; i < markers.size(); ++i)
{
for (size_t i = 0; i < markers.size(); ++i) {
std::optional<ripple::uint256> nextMarker;
if (i + 1 < markers.size())
@@ -495,13 +484,11 @@ public:
size_t progress = incr;
std::vector<std::string> edgeKeys;
while (numFinished < calls.size() && cq.Next(&tag, &ok))
{
while (numFinished < calls.size() && cq.Next(&tag, &ok)) {
assert(tag);
auto ptr = static_cast<etl::detail::AsyncCallData*>(tag);
if (!ok)
{
if (!ok) {
LOG(log_.error()) << "loadInitialLedger - ok is false";
return {{}, false}; // handle cancelled
}
@@ -509,8 +496,7 @@ public:
LOG(log_.trace()) << "Marker prefix = " << ptr->getMarkerPrefix();
auto result = ptr->process(stub_, cq, *backend_, abort, cacheOnly);
if (result != etl::detail::AsyncCallData::CallStatus::MORE)
{
if (result != etl::detail::AsyncCallData::CallStatus::MORE) {
++numFinished;
LOG(log_.debug()) << "Finished a marker. "
<< "Current number of finished = " << numFinished;
@@ -524,8 +510,7 @@ public:
if (result == etl::detail::AsyncCallData::CallStatus::ERRORED)
abort = true;
if (backend_->cache().size() > progress)
{
if (backend_->cache().size() > progress) {
LOG(log_.info()) << "Downloaded " << backend_->cache().size() << " records from rippled";
progress += incr;
}
@@ -539,10 +524,10 @@ public:
forwardToRippled(
boost::json::object const& request,
std::optional<std::string> const& clientIp,
boost::asio::yield_context yield) const override
boost::asio::yield_context yield
) const override
{
if (auto resp = forwardCache_.get(request); resp)
{
if (auto resp = forwardCache_.get(request); resp) {
LOG(log_.debug()) << "request hit forwardCache";
return resp;
}
@@ -573,13 +558,10 @@ public:
void
onResolve(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type results)
{
if (ec)
{
if (ec) {
// try again
reconnect(ec);
}
else
{
} else {
static constexpr std::size_t LOWEST_LAYER_TIMEOUT_SECONDS = 30;
boost::beast::get_lowest_layer(derived().ws())
.expires_after(std::chrono::seconds(LOWEST_LAYER_TIMEOUT_SECONDS));
@@ -600,13 +582,10 @@ public:
if (auto action = hooks_.onConnected(ec); action == SourceHooks::Action::STOP)
return;
if (ec)
{
if (ec) {
// start over
reconnect(ec);
}
else
{
} else {
boost::json::object const jv{
{"command", "subscribe"},
{"streams", {"ledger", "manifests", "validations", "transactions_proposed"}},
@@ -617,10 +596,11 @@ public:
derived().ws().set_option(
boost::beast::websocket::stream_base::decorator([](boost::beast::websocket::request_type& req) {
req.set(
boost::beast::http::field::user_agent,
std::string(BOOST_BEAST_VERSION_STRING) + " clio-client");
boost::beast::http::field::user_agent, std::string(BOOST_BEAST_VERSION_STRING) + " clio-client"
);
req.set("X-User", "coro-client");
}));
})
);
// Send subscription message
derived().ws().async_write(boost::asio::buffer(s), [this](auto ec, size_t size) { onWrite(ec, size); });
@@ -636,12 +616,9 @@ public:
void
onWrite(boost::beast::error_code ec, [[maybe_unused]] size_t size)
{
if (ec)
{
if (ec) {
reconnect(ec);
}
else
{
} else {
derived().ws().async_read(readBuffer_, [this](auto ec, size_t size) { onRead(ec, size); });
}
}
@@ -655,12 +632,9 @@ public:
void
onRead(boost::beast::error_code ec, size_t size)
{
if (ec)
{
if (ec) {
reconnect(ec);
}
else
{
} else {
handleMessage(size);
derived().ws().async_read(readBuffer_, [this](auto ec, size_t size) { onRead(ec, size); });
}
@@ -677,8 +651,7 @@ public:
{
setLastMsgTime();
try
{
try {
auto const msg = boost::beast::buffers_to_string(readBuffer_.data());
readBuffer_.consume(size);
@@ -686,65 +659,48 @@ public:
auto const response = raw.as_object();
uint32_t ledgerIndex = 0;
if (response.contains("result"))
{
if (response.contains("result")) {
auto const& result = response.at("result").as_object();
if (result.contains("ledger_index"))
ledgerIndex = result.at("ledger_index").as_int64();
if (result.contains("validated_ledgers"))
{
if (result.contains("validated_ledgers")) {
auto const& validatedLedgers = result.at("validated_ledgers").as_string();
setValidatedRange({validatedLedgers.data(), validatedLedgers.size()});
}
LOG(log_.info()) << "Received a message on ledger "
<< " subscription stream. Message : " << response << " - " << toString();
}
else if (response.contains("type") && response.at("type") == "ledgerClosed")
{
} else if (response.contains("type") && response.at("type") == "ledgerClosed") {
LOG(log_.info()) << "Received a message on ledger "
<< " subscription stream. Message : " << response << " - " << toString();
if (response.contains("ledger_index"))
{
if (response.contains("ledger_index")) {
ledgerIndex = response.at("ledger_index").as_int64();
}
if (response.contains("validated_ledgers"))
{
if (response.contains("validated_ledgers")) {
auto const& validatedLedgers = response.at("validated_ledgers").as_string();
setValidatedRange({validatedLedgers.data(), validatedLedgers.size()});
}
}
else
{
if (balancer_.shouldPropagateTxnStream(this))
{
if (response.contains("transaction"))
{
} else {
if (balancer_.shouldPropagateTxnStream(this)) {
if (response.contains("transaction")) {
forwardCache_.freshen();
subscriptions_->forwardProposedTransaction(response);
}
else if (response.contains("type") && response.at("type") == "validationReceived")
{
} else if (response.contains("type") && response.at("type") == "validationReceived") {
subscriptions_->forwardValidation(response);
}
else if (response.contains("type") && response.at("type") == "manifestReceived")
{
} else if (response.contains("type") && response.at("type") == "manifestReceived") {
subscriptions_->forwardManifest(response);
}
}
}
if (ledgerIndex != 0)
{
if (ledgerIndex != 0) {
LOG(log_.trace()) << "Pushing ledger sequence = " << ledgerIndex << " - " << toString();
networkValidatedLedgers_->push(ledgerIndex);
}
return true;
}
catch (std::exception const& e)
{
} catch (std::exception const& e) {
LOG(log_.error()) << "Exception in handleMessage : " << e.what();
return false;
}
@@ -780,8 +736,7 @@ protected:
// when the timer is cancelled. connection_refused will occur repeatedly
std::string err = ec.message();
// if we cannot connect to the transaction processing process
if (ec.category() == boost::asio::error::get_ssl_category())
{
if (ec.category() == boost::asio::error::get_ssl_category()) {
err = std::string(" (") + boost::lexical_cast<std::string>(ERR_GET_LIB(ec.value())) + "," +
boost::lexical_cast<std::string>(ERR_GET_REASON(ec.value())) + ") ";
@@ -793,12 +748,9 @@ protected:
LOG(log_.error()) << err;
}
if (ec != boost::asio::error::operation_aborted && ec != boost::asio::error::connection_refused)
{
if (ec != boost::asio::error::operation_aborted && ec != boost::asio::error::connection_refused) {
LOG(log_.error()) << "error code = " << ec << " - " << toString();
}
else
{
} else {
LOG(log_.warn()) << "error code = " << ec << " - " << toString();
}
@@ -833,19 +785,15 @@ private:
std::vector<std::pair<uint32_t, uint32_t>> pairs;
std::vector<std::string> ranges;
boost::split(ranges, range, boost::is_any_of(","));
for (auto& pair : ranges)
{
for (auto& pair : ranges) {
std::vector<std::string> minAndMax;
boost::split(minAndMax, pair, boost::is_any_of("-"));
if (minAndMax.size() == 1)
{
if (minAndMax.size() == 1) {
uint32_t const sequence = std::stoll(minAndMax[0]);
pairs.emplace_back(sequence, sequence);
}
else
{
} else {
assert(minAndMax.size() == 2);
uint32_t const min = std::stoll(minAndMax[0]);
uint32_t const max = std::stoll(minAndMax[1]);
@@ -871,8 +819,7 @@ private:
/**
* @brief Implementation of a source that uses a regular, non-secure websocket connection.
*/
class PlainSource : public SourceImpl<PlainSource>
{
class PlainSource : public SourceImpl<PlainSource> {
using StreamType = boost::beast::websocket::stream<boost::beast::tcp_stream>;
std::unique_ptr<StreamType> ws_;
@@ -895,7 +842,8 @@ public:
std::shared_ptr<feed::SubscriptionManager> subscriptions,
std::shared_ptr<NetworkValidatedLedgers> validatedLedgers,
LoadBalancer& balancer,
SourceHooks hooks)
SourceHooks hooks
)
: SourceImpl(config, ioc, backend, subscriptions, validatedLedgers, balancer, std::move(hooks))
, ws_(std::make_unique<StreamType>(strand_))
{
@@ -929,8 +877,7 @@ public:
/**
* @brief Implementation of a source that uses a secure websocket connection.
*/
class SslSource : public SourceImpl<SslSource>
{
class SslSource : public SourceImpl<SslSource> {
using StreamType = boost::beast::websocket::stream<boost::beast::ssl_stream<boost::beast::tcp_stream>>;
std::optional<std::reference_wrapper<boost::asio::ssl::context>> sslCtx_;
std::unique_ptr<StreamType> ws_;
@@ -956,7 +903,8 @@ public:
std::shared_ptr<feed::SubscriptionManager> subscriptions,
std::shared_ptr<NetworkValidatedLedgers> validatedLedgers,
LoadBalancer& balancer,
SourceHooks hooks)
SourceHooks hooks
)
: SourceImpl(config, ioc, backend, subscriptions, validatedLedgers, balancer, std::move(hooks))
, sslCtx_(sslCtx)
, ws_(std::make_unique<StreamType>(strand_, *sslCtx_))

3
src/etl/SystemState.h
View File

@@ -26,8 +26,7 @@ namespace etl {
/**
* @brief Represents the state of the ETL subsystem.
*/
struct SystemState
{
struct SystemState {
/**
* @brief Whether the process is in strict read-only mode.
*

14
src/etl/impl/AmendmentBlock.h
View File

@@ -30,21 +30,18 @@
namespace etl::detail {
struct AmendmentBlockAction
{
struct AmendmentBlockAction {
void
operator()()
{
static util::Logger const log{"ETL"};
LOG(log.fatal())
<< "Can't process new ledgers: The current ETL source is not compatible with the version of the "
"libxrpl Clio is currently using. Please upgrade Clio to a newer version.";
LOG(log.fatal()) << "Can't process new ledgers: The current ETL source is not compatible with the version of "
<< "the libxrpl Clio is currently using. Please upgrade Clio to a newer version.";
}
};
template <typename ActionCallableType = AmendmentBlockAction>
class AmendmentBlockHandler
{
class AmendmentBlockHandler {
std::reference_wrapper<boost::asio::io_context> ctx_;
std::reference_wrapper<SystemState> state_;
boost::asio::steady_timer timer_;
@@ -58,7 +55,8 @@ public:
boost::asio::io_context& ioc,
SystemState& state,
DurationType interval = DurationType{1},
ActionCallableType&& action = ActionCallableType())
ActionCallableType&& action = ActionCallableType()
)
: ctx_{std::ref(ioc)}
, state_{std::ref(state)}
, timer_{ioc}

47
src/etl/impl/AsyncData.h
View File

@@ -28,8 +28,7 @@
namespace etl::detail {
class AsyncCallData
{
class AsyncCallData {
util::Logger log_{"ETL"};
std::unique_ptr<org::xrpl::rpc::v1::GetLedgerDataResponse> cur_;
@@ -47,8 +46,7 @@ public:
AsyncCallData(uint32_t seq, ripple::uint256 const& marker, std::optional<ripple::uint256> const& nextMarker)
{
request_.mutable_ledger()->set_sequence(seq);
if (marker.isNonZero())
{
if (marker.isNonZero()) {
request_.set_marker(marker.data(), ripple::uint256::size());
}
request_.set_user("ETL");
@@ -77,25 +75,23 @@ public:
grpc::CompletionQueue& cq,
BackendInterface& backend,
bool abort,
bool cacheOnly = false)
bool cacheOnly = false
)
{
LOG(log_.trace()) << "Processing response. "
<< "Marker prefix = " << getMarkerPrefix();
if (abort)
{
if (abort) {
LOG(log_.error()) << "AsyncCallData aborted";
return CallStatus::ERRORED;
}
if (!status_.ok())
{
if (!status_.ok()) {
LOG(log_.error()) << "AsyncCallData status_ not ok: "
<< " code = " << status_.error_code() << " message = " << status_.error_message();
return CallStatus::ERRORED;
}
if (!next_->is_unlimited())
{
LOG(log_.warn()) << "AsyncCallData is_unlimited is false. Make sure "
"secure_gateway is set correctly at the ETL source";
if (!next_->is_unlimited()) {
LOG(log_.warn()) << "AsyncCallData is_unlimited is false. "
<< "Make sure secure_gateway is set correctly at the ETL source";
}
std::swap(cur_, next_);
@@ -112,8 +108,7 @@ public:
more = false;
// if we are not done, make the next async call
if (more)
{
if (more) {
request_.set_marker(cur_->marker());
call(stub, cq);
}
@@ -124,25 +119,23 @@ public:
std::vector<data::LedgerObject> cacheUpdates;
cacheUpdates.reserve(numObjects);
for (int i = 0; i < numObjects; ++i)
{
for (int i = 0; i < numObjects; ++i) {
auto& obj = *(cur_->mutable_ledger_objects()->mutable_objects(i));
if (!more && nextPrefix_ != 0x00)
{
if (!more && nextPrefix_ != 0x00) {
if (static_cast<unsigned char>(obj.key()[0]) >= nextPrefix_)
continue;
}
cacheUpdates.push_back(
{*ripple::uint256::fromVoidChecked(obj.key()),
{obj.mutable_data()->begin(), obj.mutable_data()->end()}});
if (!cacheOnly)
{
{*ripple::uint256::fromVoidChecked(obj.key()), {obj.mutable_data()->begin(), obj.mutable_data()->end()}}
);
if (!cacheOnly) {
if (!lastKey_.empty())
backend.writeSuccessor(std::move(lastKey_), request_.ledger().sequence(), std::string{obj.key()});
lastKey_ = obj.key();
backend.writeNFTs(getNFTDataFromObj(request_.ledger().sequence(), obj.key(), obj.data()));
backend.writeLedgerObject(
std::move(*obj.mutable_key()), request_.ledger().sequence(), std::move(*obj.mutable_data()));
std::move(*obj.mutable_key()), request_.ledger().sequence(), std::move(*obj.mutable_data())
);
}
}
backend.cache().update(cacheUpdates, request_.ledger().sequence(), cacheOnly);
@@ -157,7 +150,8 @@ public:
context_ = std::make_unique<grpc::ClientContext>();
std::unique_ptr<grpc::ClientAsyncResponseReader<org::xrpl::rpc::v1::GetLedgerDataResponse>> rpc(
stub->PrepareAsyncGetLedgerData(context_.get(), request_, &cq));
stub->PrepareAsyncGetLedgerData(context_.get(), request_, &cq)
);
rpc->StartCall();
@@ -167,8 +161,7 @@ public:
std::string
getMarkerPrefix()
{
if (next_->marker().empty())
{
if (next_->marker().empty()) {
return "";
}
return ripple::strHex(std::string{next_->marker().data()[0]});

115
src/etl/impl/CacheLoader.h
View File

@@ -40,8 +40,7 @@ namespace etl::detail {
* @brief Cache loading interface
*/
template <typename CacheType>
class CacheLoader
{
class CacheLoader {
static constexpr size_t DEFAULT_NUM_CACHE_DIFFS = 32;
static constexpr size_t DEFAULT_NUM_CACHE_MARKERS = 48;
static constexpr size_t DEFAULT_CACHE_PAGE_FETCH_SIZE = 512;
@@ -64,8 +63,7 @@ class CacheLoader
// number of ledger objects to fetch concurrently per marker during cache download
size_t cachePageFetchSize_ = DEFAULT_CACHE_PAGE_FETCH_SIZE;
struct ClioPeer
{
struct ClioPeer {
std::string ip;
int port{};
};
@@ -80,14 +78,13 @@ public:
util::Config const& config,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> const& backend,
CacheType& ledgerCache)
CacheType& ledgerCache
)
: ioContext_{std::ref(ioc)}, backend_{backend}, cache_{ledgerCache}
{
if (config.contains("cache"))
{
if (config.contains("cache")) {
auto const cache = config.section("cache");
if (auto entry = cache.maybeValue<std::string>("load"); entry)
{
if (auto entry = cache.maybeValue<std::string>("load"); entry) {
if (boost::iequals(*entry, "sync"))
cacheLoadStyle_ = LoadStyle::SYNC;
if (boost::iequals(*entry, "async"))
@@ -100,10 +97,8 @@ public:
numCacheMarkers_ = cache.valueOr<size_t>("num_markers", numCacheMarkers_);
cachePageFetchSize_ = cache.valueOr<size_t>("page_fetch_size", cachePageFetchSize_);
if (auto peers = cache.maybeArray("peers"); peers)
{
for (auto const& peer : *peers)
{
if (auto peers = cache.maybeArray("peers"); peers) {
for (auto const& peer : *peers) {
auto ip = peer.value<std::string>("ip");
auto port = peer.value<uint32_t>("port");
@@ -133,24 +128,20 @@ public:
void
load(uint32_t seq)
{
if (cacheLoadStyle_ == LoadStyle::NOT_AT_ALL)
{
if (cacheLoadStyle_ == LoadStyle::NOT_AT_ALL) {
cache_.get().setDisabled();
LOG(log_.warn()) << "Cache is disabled. Not loading";
return;
}
if (cache_.get().isFull())
{
if (cache_.get().isFull()) {
assert(false);
return;
}
if (!clioPeers_.empty())
{
if (!clioPeers_.empty()) {
boost::asio::spawn(ioContext_.get(), [this, seq](boost::asio::yield_context yield) {
for (auto const& peer : clioPeers_)
{
for (auto const& peer : clioPeers_) {
// returns true on success
if (loadCacheFromClioPeer(seq, peer.ip, std::to_string(peer.port), yield))
return;
@@ -166,8 +157,7 @@ public:
// If loading synchronously, poll cache until full
static constexpr size_t SLEEP_TIME_SECONDS = 10;
while (cacheLoadStyle_ == LoadStyle::SYNC && not cache_.get().isFull())
{
while (cacheLoadStyle_ == LoadStyle::SYNC && not cache_.get().isFull()) {
LOG(log_.debug()) << "Cache not full. Cache size = " << cache_.get().size() << ". Sleeping ...";
std::this_thread::sleep_for(std::chrono::seconds(SLEEP_TIME_SECONDS));
if (cache_.get().isFull())
@@ -187,15 +177,15 @@ private:
uint32_t ledgerIndex,
std::string const& ip,
std::string const& port,
boost::asio::yield_context yield)
boost::asio::yield_context yield
)
{
LOG(log_.info()) << "Loading cache from peer. ip = " << ip << " . port = " << port;
namespace beast = boost::beast; // from <boost/beast.hpp>
namespace websocket = beast::websocket; // from
namespace net = boost::asio; // from
using tcp = boost::asio::ip::tcp; // from
try
{
try {
beast::error_code ec;
// These objects perform our I/O
tcp::resolver resolver{ioContext_.get()};
@@ -239,20 +229,17 @@ private:
bool started = false;
size_t numAttempts = 0;
do
{
do {
// Send the message
ws->async_write(net::buffer(boost::json::serialize(getRequest(marker))), yield[ec]);
if (ec)
{
if (ec) {
LOG(log_.error()) << "error writing = " << ec.message();
return false;
}
beast::flat_buffer buffer;
ws->async_read(buffer, yield[ec]);
if (ec)
{
if (ec) {
LOG(log_.error()) << "error reading = " << ec.message();
return false;
}
@@ -260,27 +247,22 @@ private:
auto raw = beast::buffers_to_string(buffer.data());
auto parsed = boost::json::parse(raw);
if (!parsed.is_object())
{
if (!parsed.is_object()) {
LOG(log_.error()) << "Error parsing response: " << raw;
return false;
}
LOG(log_.trace()) << "Successfully parsed response " << parsed;
if (auto const& response = parsed.as_object(); response.contains("error"))
{
if (auto const& response = parsed.as_object(); response.contains("error")) {
LOG(log_.error()) << "Response contains error: " << response;
auto const& err = response.at("error");
if (err.is_string() && err.as_string() == "lgrNotFound")
{
if (err.is_string() && err.as_string() == "lgrNotFound") {
static constexpr size_t MAX_ATTEMPTS = 5;
++numAttempts;
if (numAttempts >= MAX_ATTEMPTS)
{
LOG(log_.error()) << " ledger not found at peer after 5 attempts. "
"peer = "
<< ip << " ledger = " << ledgerIndex
<< ". Check your config and the health of the peer";
if (numAttempts >= MAX_ATTEMPTS) {
LOG(log_.error())
<< " ledger not found at peer after 5 attempts. peer = " << ip
<< " ledger = " << ledgerIndex << ". Check your config and the health of the peer";
return false;
}
LOG(log_.warn()) << "Ledger not found. ledger = " << ledgerIndex
@@ -293,17 +275,13 @@ private:
started = true;
auto const& response = parsed.as_object()["result"].as_object();
if (!response.contains("cache_full") || !response.at("cache_full").as_bool())
{
if (!response.contains("cache_full") || !response.at("cache_full").as_bool()) {
LOG(log_.error()) << "cache not full for clio node. ip = " << ip;
return false;
}
if (response.contains("marker"))
{
if (response.contains("marker")) {
marker = response.at("marker");
}
else
{
} else {
marker = {};
}
@@ -311,14 +289,12 @@ private:
std::vector<data::LedgerObject> objects;
objects.reserve(state.size());
for (auto const& ledgerObject : state)
{
for (auto const& ledgerObject : state) {
auto const& obj = ledgerObject.as_object();
data::LedgerObject stateObject = {};
if (!stateObject.key.parseHex(obj.at("index").as_string().c_str()))
{
if (!stateObject.key.parseHex(obj.at("index").as_string().c_str())) {
LOG(log_.error()) << "failed to parse object id";
return false;
}
@@ -335,9 +311,7 @@ private:
cache_.get().setFull();
return true;
}
catch (std::exception const& e)
{
} catch (std::exception const& e) {
LOG(log_.error()) << "Encountered exception : " << e.what() << " - ip = " << ip;
return false;
}
@@ -351,8 +325,7 @@ private:
auto append = [](auto&& a, auto&& b) { a.insert(std::end(a), std::begin(b), std::end(b)); };
for (size_t i = 0; i < numCacheDiffs_; ++i)
{
for (size_t i = 0; i < numCacheDiffs_; ++i) {
append(diff, data::synchronousAndRetryOnTimeout([&](auto yield) {
return backend_->fetchLedgerDiff(seq - i, yield);
}));
@@ -365,16 +338,14 @@ private:
diff.erase(std::unique(diff.begin(), diff.end(), [](auto a, auto b) { return a.key == b.key; }), diff.end());
cursors.emplace_back();
for (auto const& obj : diff)
{
for (auto const& obj : diff) {
if (!obj.blob.empty())
cursors.emplace_back(obj.key);
}
cursors.emplace_back();
std::stringstream cursorStr;
for (auto const& c : cursors)
{
for (auto const& c : cursors) {
if (c)
cursorStr << ripple::strHex(*c) << ", ";
}
@@ -387,8 +358,7 @@ private:
auto markers = std::make_shared<std::atomic_int>(0);
auto numRemaining = std::make_shared<std::atomic_int>(cursors.size() - 1);
for (size_t i = 0; i < cursors.size() - 1; ++i)
{
for (size_t i = 0; i < cursors.size() - 1; ++i) {
auto const start = cursors.at(i);
auto const end = cursors.at(i + 1);
@@ -403,8 +373,7 @@ private:
cursor.has_value() ? ripple::strHex(cursor.value()) : ripple::strHex(data::firstKey);
LOG(log_.debug()) << "Starting a cursor: " << cursorStr << " markers = " << *markers;
while (not stopping_)
{
while (not stopping_) {
auto res = data::retryOnTimeout([this, seq, &cursor, yield]() {
return backend_->fetchLedgerPage(cursor, seq, cachePageFetchSize_, false, yield);
});
@@ -424,21 +393,19 @@ private:
--(*markers);
markers->notify_one();
if (--(*numRemaining) == 0)
{
if (--(*numRemaining) == 0) {
auto endTime = std::chrono::system_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::seconds>(endTime - startTime);
LOG(log_.info()) << "Finished loading cache. cache size = " << cache_.get().size()
<< ". Took " << duration.count() << " seconds";
cache_.get().setFull();
}
else
{
} else {
LOG(log_.info()) << "Finished a cursor. num remaining = " << *numRemaining
<< " start = " << cursorStr << " markers = " << *markers;
}
});
}
);
}
}};
}

3
src/etl/impl/ExtractionDataPipe.h
View File

@@ -31,8 +31,7 @@ namespace etl::detail {
* @brief A collection of thread safe async queues used by Extractor and Transformer to communicate
*/
template <typename RawDataType>
class ExtractionDataPipe
{
class ExtractionDataPipe {
public:
using DataType = std::optional<RawDataType>;
using QueueType = ThreadSafeQueue<DataType>; // TODO: probably should use boost::lockfree::queue instead?

23
src/etl/impl/Extractor.h
View File

@@ -36,8 +36,7 @@ namespace etl::detail {
* @brief Extractor thread that is fetching GRPC data and enqueue it on the DataPipeType
*/
template <typename DataPipeType, typename NetworkValidatedLedgersType, typename LedgerFetcherType>
class Extractor
{
class Extractor {
util::Logger log_{"ETL"};
std::reference_wrapper<DataPipeType> pipe_;
@@ -56,7 +55,8 @@ public:
LedgerFetcherType& ledgerFetcher,
uint32_t startSequence,
std::optional<uint32_t> finishSequence,
SystemState const& state)
SystemState const& state
)
: pipe_(std::ref(pipe))
, networkValidatedLedgers_{std::move(networkValidatedLedgers)}
, ledgerFetcher_{std::ref(ledgerFetcher)}
@@ -89,16 +89,17 @@ private:
double totalTime = 0.0;
auto currentSequence = startSequence_;
while (!shouldFinish(currentSequence) && networkValidatedLedgers_->waitUntilValidatedByNetwork(currentSequence))
{
auto [fetchResponse, time] = ::util::timed<std::chrono::duration<double>>(
[this, currentSequence]() { return ledgerFetcher_.get().fetchDataAndDiff(currentSequence); });
while (!shouldFinish(currentSequence) && networkValidatedLedgers_->waitUntilValidatedByNetwork(currentSequence)
) {
auto [fetchResponse, time] = ::util::timed<std::chrono::duration<double>>([this, currentSequence]() {
return ledgerFetcher_.get().fetchDataAndDiff(currentSequence);
});
totalTime += time;
// if the fetch is unsuccessful, stop. fetchLedger only returns false if the server is shutting down, or if
// the ledger was found in the database (which means another process already wrote the ledger that this
// process was trying to extract; this is a form of a write conflict).
// Otherwise, fetchDataAndDiff will keep trying to fetch the specified ledger until successful.
// if the fetch is unsuccessful, stop. fetchLedger only returns false if the server is shutting down, or
// if the ledger was found in the database (which means another process already wrote the ledger that
// this process was trying to extract; this is a form of a write conflict). Otherwise, fetchDataAndDiff
// will keep trying to fetch the specified ledger until successful.
if (!fetchResponse)
break;

16
src/etl/impl/ForwardCache.cpp
View File

@@ -33,10 +33,10 @@ ForwardCache::freshen()
auto numOutstanding = std::make_shared<std::atomic_uint>(latestForwarded_.size());
for (auto const& cacheEntry : latestForwarded_)
{
for (auto const& cacheEntry : latestForwarded_) {
boost::asio::spawn(
strand_, [this, numOutstanding, command = cacheEntry.first](boost::asio::yield_context yield) {
strand_,
[this, numOutstanding, command = cacheEntry.first](boost::asio::yield_context yield) {
boost::json::object const request = {{"command", command}};
auto resp = source_.requestFromRippled(request, std::nullopt, yield);
@@ -47,7 +47,8 @@ ForwardCache::freshen()
std::scoped_lock const lk(mtx_);
latestForwarded_[command] = resp;
}
});
}
);
}
}
@@ -63,12 +64,9 @@ std::optional<boost::json::object>
ForwardCache::get(boost::json::object const& request) const
{
std::optional<std::string> command = {};
if (request.contains("command") && !request.contains("method") && request.at("command").is_string())
{
if (request.contains("command") && !request.contains("method") && request.at("command").is_string()) {
command = request.at("command").as_string().c_str();
}
else if (request.contains("method") && !request.contains("command") && request.at("method").is_string())
{
} else if (request.contains("method") && !request.contains("command") && request.at("method").is_string()) {
command = request.at("method").as_string().c_str();
}

9
src/etl/impl/ForwardCache.h
View File

@@ -40,8 +40,7 @@ namespace etl::detail {
/**
* @brief Cache for rippled responses
*/
class ForwardCache
{
class ForwardCache {
using ResponseType = std::optional<boost::json::object>;
static constexpr std::uint32_t DEFAULT_DURATION = 10;
@@ -60,15 +59,13 @@ public:
ForwardCache(util::Config const& config, boost::asio::io_context& ioc, Source const& source)
: strand_(boost::asio::make_strand(ioc)), source_(source)
{
if (config.contains("cache"))
{
if (config.contains("cache")) {
auto commands = config.arrayOrThrow("cache", "Source cache must be array");
if (config.contains("cache_duration"))
duration_ = config.valueOrThrow<uint32_t>("cache_duration", "Source cache_duration must be a number");
for (auto const& command : commands)
{
for (auto const& command : commands) {
auto key = command.valueOrThrow<std::string>("Source forward command must be array of strings");
latestForwarded_[key] = {};
}

6
src/etl/impl/LedgerFetcher.h
View File

@@ -35,8 +35,7 @@ namespace etl::detail {
* @brief GRPC Ledger data fetcher
*/
template <typename LoadBalancerType>
class LedgerFetcher
{
class LedgerFetcher {
public:
using OptionalGetLedgerResponseType = typename LoadBalancerType::OptionalGetLedgerResponseType;
@@ -91,7 +90,8 @@ public:
LOG(log_.debug()) << "Attempting to fetch ledger with sequence = " << sequence;
auto response = loadBalancer_->fetchLedger(
sequence, true, !backend_->cache().isFull() || backend_->cache().latestLedgerSequence() >= sequence);
sequence, true, !backend_->cache().isFull() || backend_->cache().latestLedgerSequence() >= sequence
);
if (response)
LOG(log_.trace()) << "GetLedger reply = " << response->DebugString();

51
src/etl/impl/LedgerLoader.h
View File

@@ -35,8 +35,7 @@
/**
* @brief Account transactions, NFT transactions and NFT data bundled togeher.
*/
struct FormattedTransactionsData
{
struct FormattedTransactionsData {
std::vector<AccountTransactionsData> accountTxData;
std::vector<NFTTransactionsData> nfTokenTxData;
std::vector<NFTsData> nfTokensData;
@@ -48,8 +47,7 @@ namespace etl::detail {
* @brief Loads ledger data into the DB
*/
template <typename LoadBalancerType, typename LedgerFetcherType>
class LedgerLoader
{
class LedgerLoader {
public:
using GetLedgerResponseType = typename LoadBalancerType::GetLedgerResponseType;
using OptionalGetLedgerResponseType = typename LoadBalancerType::OptionalGetLedgerResponseType;
@@ -71,7 +69,8 @@ public:
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<LoadBalancerType> balancer,
LedgerFetcherType& fetcher,
SystemState const& state)
SystemState const& state
)
: backend_{std::move(backend)}
, loadBalancer_{std::move(balancer)}
, fetcher_{std::ref(fetcher)}
@@ -95,8 +94,7 @@ public:
{
FormattedTransactionsData result;
for (auto& txn : *(data.mutable_transactions_list()->mutable_transactions()))
{
for (auto& txn : *(data.mutable_transactions_list()->mutable_transactions())) {
std::string* raw = txn.mutable_transaction_blob();
ripple::SerialIter it{raw->data(), raw->size()};
@@ -113,13 +111,14 @@ public:
result.accountTxData.emplace_back(txMeta, sttx.getTransactionID());
static constexpr std::size_t KEY_SIZE = 32;
std::string keyStr{reinterpret_cast<const char*>(sttx.getTransactionID().data()), KEY_SIZE};
std::string keyStr{reinterpret_cast<char const*>(sttx.getTransactionID().data()), KEY_SIZE};
backend_->writeTransaction(
std::move(keyStr),
ledger.seq,
ledger.closeTime.time_since_epoch().count(),
std::move(*raw),
std::move(*txn.mutable_metadata_blob()));
std::move(*txn.mutable_metadata_blob())
);
}
// Remove all but the last NFTsData for each id. unique removes all but the first of a group, so we want to
@@ -130,9 +129,10 @@ public:
// Now we can unique the NFTs by tokenID.
auto last = std::unique(
result.nfTokensData.begin(), result.nfTokensData.end(), [](NFTsData const& a, NFTsData const& b) {
return a.tokenID == b.tokenID;
});
result.nfTokensData.begin(),
result.nfTokensData.end(),
[](NFTsData const& a, NFTsData const& b) { return a.tokenID == b.tokenID; }
);
result.nfTokensData.erase(last, result.nfTokensData.end());
return result;
@@ -151,8 +151,7 @@ public:
{
// check that database is actually empty
auto rng = backend_->hardFetchLedgerRangeNoThrow();
if (rng)
{
if (rng) {
LOG(log_.fatal()) << "Database is not empty";
assert(false);
return {};
@@ -186,15 +185,13 @@ public:
// into the queue
auto [edgeKeys, success] = loadBalancer_->loadInitialLedger(sequence);
if (success)
{
if (success) {
size_t numWrites = 0;
backend_->cache().setFull();
auto seconds =
::util::timed<std::chrono::seconds>([this, edgeKeys = &edgeKeys, sequence, &numWrites]() {
for (auto& key : *edgeKeys)
{
for (auto& key : *edgeKeys) {
LOG(log_.debug()) << "Writing edge key = " << ripple::strHex(key);
auto succ =
backend_->cache().getSuccessor(*ripple::uint256::fromVoidChecked(key), sequence);
@@ -203,27 +200,24 @@ public:
}
ripple::uint256 prev = data::firstKey;
while (auto cur = backend_->cache().getSuccessor(prev, sequence))
{
while (auto cur = backend_->cache().getSuccessor(prev, sequence)) {
assert(cur);
if (prev == data::firstKey)
backend_->writeSuccessor(uint256ToString(prev), sequence, uint256ToString(cur->key));
if (isBookDir(cur->key, cur->blob))
{
if (isBookDir(cur->key, cur->blob)) {
auto base = getBookBase(cur->key);
// make sure the base is not an actual object
if (!backend_->cache().get(cur->key, sequence))
{
if (!backend_->cache().get(cur->key, sequence)) {
auto succ = backend_->cache().getSuccessor(base, sequence);
assert(succ);
if (succ->key == cur->key)
{
if (succ->key == cur->key) {
LOG(log_.debug()) << "Writing book successor = " << ripple::strHex(base)
<< " - " << ripple::strHex(cur->key);
backend_->writeSuccessor(
uint256ToString(base), sequence, uint256ToString(cur->key));
uint256ToString(base), sequence, uint256ToString(cur->key)
);
}
}
@@ -246,8 +240,7 @@ public:
LOG(log_.debug()) << "Loaded initial ledger";
if (not state_.get().isStopping)
{
if (not state_.get().isStopping) {
backend_->writeAccountTransactions(std::move(insertTxResult.accountTxData));
backend_->writeNFTs(std::move(insertTxResult.nfTokensData));
backend_->writeNFTTransactions(std::move(insertTxResult.nfTokenTxData));

49
src/etl/impl/LedgerPublisher.h
View File

@@ -44,8 +44,7 @@ namespace etl::detail {
* strand is used to ensure ledgers are published in order.
*/
template <typename SubscriptionManagerType, typename CacheType>
class LedgerPublisher
{
class LedgerPublisher {
util::Logger log_{"ETL"};
boost::asio::strand<boost::asio::io_context::executor_type> publishStrand_;
@@ -73,7 +72,8 @@ public:
std::shared_ptr<BackendInterface> backend,
CacheType& cache,
std::shared_ptr<SubscriptionManagerType> subscriptions,
SystemState const& state)
SystemState const& state
)
: publishStrand_{boost::asio::make_strand(ioc)}
, backend_{std::move(backend)}
, cache_{cache}
@@ -95,20 +95,15 @@ public:
{
LOG(log_.info()) << "Attempting to publish ledger = " << ledgerSequence;
size_t numAttempts = 0;
while (not state_.get().isStopping)
{
while (not state_.get().isStopping) {
auto range = backend_->hardFetchLedgerRangeNoThrow();
if (!range || range->maxSequence < ledgerSequence)
{
if (!range || range->maxSequence < ledgerSequence) {
++numAttempts;
LOG(log_.debug()) << "Trying to publish. Could not find "
"ledger with sequence = "
<< ledgerSequence;
LOG(log_.debug()) << "Trying to publish. Could not find ledger with sequence = " << ledgerSequence;
// We try maxAttempts times to publish the ledger, waiting one second in between each attempt.
if (maxAttempts && numAttempts >= maxAttempts)
{
if (maxAttempts && numAttempts >= maxAttempts) {
LOG(log_.debug()) << "Failed to publish ledger after " << numAttempts << " attempts.";
return false;
}
@@ -116,8 +111,9 @@ public:
continue;
}
auto lgr = data::synchronousAndRetryOnTimeout(
[&](auto yield) { return backend_->fetchLedgerBySequence(ledgerSequence, yield); });
auto lgr = data::synchronousAndRetryOnTimeout([&](auto yield) {
return backend_->fetchLedgerBySequence(ledgerSequence, yield);
});
assert(lgr);
publish(*lgr);
@@ -140,12 +136,12 @@ public:
boost::asio::post(publishStrand_, [this, lgrInfo = lgrInfo]() {
LOG(log_.info()) << "Publishing ledger " << std::to_string(lgrInfo.seq);
if (!state_.get().isWriting)
{
if (!state_.get().isWriting) {
LOG(log_.info()) << "Updating cache";
std::vector<data::LedgerObject> const diff = data::synchronousAndRetryOnTimeout(
[&](auto yield) { return backend_->fetchLedgerDiff(lgrInfo.seq, yield); });
std::vector<data::LedgerObject> const diff = data::synchronousAndRetryOnTimeout([&](auto yield) {
return backend_->fetchLedgerDiff(lgrInfo.seq, yield);
});
cache_.get().update(diff, lgrInfo.seq);
backend_->updateRange(lgrInfo.seq);
@@ -157,14 +153,16 @@ public:
// if the ledger closed over MAX_LEDGER_AGE_SECONDS ago, assume we are still catching up and don't publish
// TODO: this probably should be a strategy
static constexpr std::uint32_t MAX_LEDGER_AGE_SECONDS = 600;
if (age < MAX_LEDGER_AGE_SECONDS)
{
std::optional<ripple::Fees> fees = data::synchronousAndRetryOnTimeout(
[&](auto yield) { return backend_->fetchFees(lgrInfo.seq, yield); });
if (age < MAX_LEDGER_AGE_SECONDS) {
std::optional<ripple::Fees> fees = data::synchronousAndRetryOnTimeout([&](auto yield) {
return backend_->fetchFees(lgrInfo.seq, yield);
});
assert(fees);
std::vector<data::TransactionAndMetadata> transactions = data::synchronousAndRetryOnTimeout(
[&](auto yield) { return backend_->fetchAllTransactionsInLedger(lgrInfo.seq, yield); });
std::vector<data::TransactionAndMetadata> transactions =
data::synchronousAndRetryOnTimeout([&](auto yield) {
return backend_->fetchAllTransactionsInLedger(lgrInfo.seq, yield);
});
auto const ledgerRange = backend_->fetchLedgerRange();
assert(ledgerRange);
@@ -191,8 +189,7 @@ public:
setLastPublishTime();
LOG(log_.info()) << "Published ledger " << std::to_string(lgrInfo.seq);
}
else
} else
LOG(log_.info()) << "Skipping publishing ledger " << std::to_string(lgrInfo.seq);
});

90
src/etl/impl/Transformer.h
View File

@@ -54,8 +54,7 @@ template <
typename LedgerLoaderType,
typename LedgerPublisherType,
typename AmendmentBlockHandlerType>
class Transformer
{
class Transformer {
using GetLedgerResponseType = typename LedgerLoaderType::GetLedgerResponseType;
using RawLedgerObjectType = typename LedgerLoaderType::RawLedgerObjectType;
@@ -86,7 +85,8 @@ public:
LedgerPublisherType& publisher,
AmendmentBlockHandlerType& amendmentBlockHandler,
uint32_t startSequence,
SystemState& state)
SystemState& state
)
: pipe_{std::ref(pipe)}
, backend_{std::move(backend)}
, loader_{std::ref(loader)}
@@ -124,8 +124,7 @@ private:
beast::setCurrentThreadName("ETLService transform");
uint32_t currentSequence = startSequence_;
while (not hasWriteConflict())
{
while (not hasWriteConflict()) {
auto fetchResponse = pipe_.get().popNext(currentSequence);
++currentSequence;
@@ -140,8 +139,7 @@ private:
auto const start = std::chrono::system_clock::now();
auto [lgrInfo, success] = buildNextLedger(*fetchResponse);
if (success)
{
if (success) {
auto const numTxns = fetchResponse->transactions_list().transactions_size();
auto const numObjects = fetchResponse->ledger_objects().objects_size();
auto const end = std::chrono::system_clock::now();
@@ -155,9 +153,7 @@ private:
// success is false if the ledger was already written
publisher_.get().publish(lgrInfo);
}
else
{
} else {
LOG(log_.error()) << "Error writing ledger. " << util::toString(lgrInfo);
}
@@ -184,17 +180,14 @@ private:
writeSuccessors(lgrInfo, rawData);
std::optional<FormattedTransactionsData> insertTxResultOp;
try
{
try {
updateCache(lgrInfo, rawData);
LOG(log_.debug()) << "Inserted/modified/deleted all objects. Number of objects = "
<< rawData.ledger_objects().objects_size();
insertTxResultOp.emplace(loader_.get().insertTransactions(lgrInfo, rawData));
}
catch (std::runtime_error const& e)
{
} catch (std::runtime_error const& e) {
LOG(log_.fatal()) << "Failed to build next ledger: " << e.what();
amendmentBlockHandler_.get().onAmendmentBlock();
@@ -233,16 +226,14 @@ private:
std::set<ripple::uint256> bookSuccessorsToCalculate;
std::set<ripple::uint256> modified;
for (auto& obj : *(rawData.mutable_ledger_objects()->mutable_objects()))
{
for (auto& obj : *(rawData.mutable_ledger_objects()->mutable_objects())) {
auto key = ripple::uint256::fromVoidChecked(obj.key());
assert(key);
cacheUpdates.push_back({*key, {obj.mutable_data()->begin(), obj.mutable_data()->end()}});
LOG(log_.debug()) << "key = " << ripple::strHex(*key) << " - mod type = " << obj.mod_type();
if (obj.mod_type() != RawLedgerObjectType::MODIFIED && !rawData.object_neighbors_included())
{
if (obj.mod_type() != RawLedgerObjectType::MODIFIED && !rawData.object_neighbors_included()) {
LOG(log_.debug()) << "object neighbors not included. using cache";
if (!backend_->cache().isFull() || backend_->cache().latestLedgerSequence() != lgrInfo.seq - 1)
@@ -252,28 +243,24 @@ private:
auto checkBookBase = false;
auto const isDeleted = (blob->size() == 0);
if (isDeleted)
{
if (isDeleted) {
auto const old = backend_->cache().get(*key, lgrInfo.seq - 1);
assert(old);
checkBookBase = isBookDir(*key, *old);
}
else
{
} else {
checkBookBase = isBookDir(*key, *blob);
}
if (checkBookBase)
{
if (checkBookBase) {
LOG(log_.debug()) << "Is book dir. Key = " << ripple::strHex(*key);
auto const bookBase = getBookBase(*key);
auto const oldFirstDir = backend_->cache().getSuccessor(bookBase, lgrInfo.seq - 1);
assert(oldFirstDir);
// We deleted the first directory, or we added a directory prior to the old first directory
if ((isDeleted && key == oldFirstDir->key) || (!isDeleted && key < oldFirstDir->key))
{
// We deleted the first directory, or we added a directory prior to the old first
// directory
if ((isDeleted && key == oldFirstDir->key) || (!isDeleted && key < oldFirstDir->key)) {
LOG(log_.debug())
<< "Need to recalculate book base successor. base = " << ripple::strHex(bookBase)
<< " - key = " << ripple::strHex(*key) << " - isDeleted = " << isDeleted
@@ -292,14 +279,12 @@ private:
backend_->cache().update(cacheUpdates, lgrInfo.seq);
// rippled didn't send successor information, so use our cache
if (!rawData.object_neighbors_included())
{
if (!rawData.object_neighbors_included()) {
LOG(log_.debug()) << "object neighbors not included. using cache";
if (!backend_->cache().isFull() || backend_->cache().latestLedgerSequence() != lgrInfo.seq)
throw std::logic_error("Cache is not full, but object neighbors were not included");
for (auto const& obj : cacheUpdates)
{
for (auto const& obj : cacheUpdates) {
if (modified.contains(obj.key))
continue;
@@ -311,15 +296,12 @@ private:
if (!ub)
ub = {data::lastKey, {}};
if (obj.blob.empty())
{
if (obj.blob.empty()) {
LOG(log_.debug()) << "writing successor for deleted object " << ripple::strHex(obj.key) << " - "
<< ripple::strHex(lb->key) << " - " << ripple::strHex(ub->key);
backend_->writeSuccessor(uint256ToString(lb->key), lgrInfo.seq, uint256ToString(ub->key));
}
else
{
} else {
backend_->writeSuccessor(uint256ToString(lb->key), lgrInfo.seq, uint256ToString(obj.key));
backend_->writeSuccessor(uint256ToString(obj.key), lgrInfo.seq, uint256ToString(ub->key));
@@ -328,18 +310,14 @@ private:
}
}
for (auto const& base : bookSuccessorsToCalculate)
{
for (auto const& base : bookSuccessorsToCalculate) {
auto succ = backend_->cache().getSuccessor(base, lgrInfo.seq);
if (succ)
{
if (succ) {
backend_->writeSuccessor(uint256ToString(base), lgrInfo.seq, uint256ToString(succ->key));
LOG(log_.debug()) << "Updating book successor " << ripple::strHex(base) << " - "
<< ripple::strHex(succ->key);
}
else
{
} else {
backend_->writeSuccessor(uint256ToString(base), lgrInfo.seq, uint256ToString(data::lastKey));
LOG(log_.debug()) << "Updating book successor " << ripple::strHex(base) << " - "
@@ -359,12 +337,10 @@ private:
writeSuccessors(ripple::LedgerHeader const& lgrInfo, GetLedgerResponseType& rawData)
{
// Write successor info, if included from rippled
if (rawData.object_neighbors_included())
{
if (rawData.object_neighbors_included()) {
LOG(log_.debug()) << "object neighbors included";
for (auto& obj : *(rawData.mutable_book_successors()))
{
for (auto& obj : *(rawData.mutable_book_successors())) {
auto firstBook = std::move(*obj.mutable_first_book());
if (!firstBook.size())
firstBook = uint256ToString(data::lastKey);
@@ -374,10 +350,8 @@ private:
backend_->writeSuccessor(std::move(*obj.mutable_book_base()), lgrInfo.seq, std::move(firstBook));
}
for (auto& obj : *(rawData.mutable_ledger_objects()->mutable_objects()))
{
if (obj.mod_type() != RawLedgerObjectType::MODIFIED)
{
for (auto& obj : *(rawData.mutable_ledger_objects()->mutable_objects())) {
if (obj.mod_type() != RawLedgerObjectType::MODIFIED) {
std::string* predPtr = obj.mutable_predecessor();
if (predPtr->empty())
*predPtr = uint256ToString(data::firstKey);
@@ -385,23 +359,19 @@ private:
if (succPtr->empty())
*succPtr = uint256ToString(data::lastKey);
if (obj.mod_type() == RawLedgerObjectType::DELETED)
{
if (obj.mod_type() == RawLedgerObjectType::DELETED) {
LOG(log_.debug()) << "Modifying successors for deleted object " << ripple::strHex(obj.key())
<< " - " << ripple::strHex(*predPtr) << " - " << ripple::strHex(*succPtr);
backend_->writeSuccessor(std::move(*predPtr), lgrInfo.seq, std::move(*succPtr));
}
else
{
} else {
LOG(log_.debug()) << "adding successor for new object " << ripple::strHex(obj.key()) << " - "
<< ripple::strHex(*predPtr) << " - " << ripple::strHex(*succPtr);
backend_->writeSuccessor(std::move(*predPtr), lgrInfo.seq, std::string{obj.key()});
backend_->writeSuccessor(std::string{obj.key()}, lgrInfo.seq, std::move(*succPtr));
}
}
else
} else
LOG(log_.debug()) << "object modified " << ripple::strHex(obj.key());
}
}

64
src/feed/SubscriptionManager.cpp
View File

@@ -52,7 +52,8 @@ getLedgerPubMessage(
ripple::LedgerHeader const& lgrInfo,
ripple::Fees const& fees,
std::string const& ledgerRange,
std::uint32_t txnCount)
std::uint32_t txnCount
)
{
boost::json::object pubMsg;
@@ -127,8 +128,9 @@ SubscriptionManager::unsubAccount(ripple::AccountID const& account, SessionPtrTy
void
SubscriptionManager::subBook(ripple::Book const& book, SessionPtrType session)
{
subscribeHelper(
session, book, bookSubscribers_, [this, book](SessionPtrType session) { unsubBook(book, session); });
subscribeHelper(session, book, bookSubscribers_, [this, book](SessionPtrType session) {
unsubBook(book, session);
});
}
void
@@ -154,10 +156,12 @@ SubscriptionManager::pubLedger(
ripple::LedgerHeader const& lgrInfo,
ripple::Fees const& fees,
std::string const& ledgerRange,
std::uint32_t txnCount)
std::uint32_t txnCount
)
{
auto message = std::make_shared<std::string>(
boost::json::serialize(getLedgerPubMessage(lgrInfo, fees, ledgerRange, txnCount)));
auto message =
std::make_shared<std::string>(boost::json::serialize(getLedgerPubMessage(lgrInfo, fees, ledgerRange, txnCount))
);
ledgerSubscribers_.publish(message);
}
@@ -184,12 +188,10 @@ SubscriptionManager::pubTransaction(data::TransactionAndMetadata const& blobs, r
ripple::transResultInfo(meta->getResultTER(), token, human);
pubObj["engine_result"] = token;
pubObj["engine_result_message"] = human;
if (tx->getTxnType() == ripple::ttOFFER_CREATE)
{
if (tx->getTxnType() == ripple::ttOFFER_CREATE) {
auto account = tx->getAccountID(ripple::sfAccount);
auto amount = tx->getFieldAmount(ripple::sfTakerGets);
if (account != amount.issue().account)
{
if (account != amount.issue().account) {
ripple::STAmount ownerFunds;
auto fetchFundsSynchronous = [&]() {
data::synchronous([&](boost::asio::yield_context yield) {
@@ -213,40 +215,30 @@ SubscriptionManager::pubTransaction(data::TransactionAndMetadata const& blobs, r
std::unordered_set<ripple::Book> alreadySent;
for (auto const& node : meta->getNodes())
{
if (node.getFieldU16(ripple::sfLedgerEntryType) == ripple::ltOFFER)
{
for (auto const& node : meta->getNodes()) {
if (node.getFieldU16(ripple::sfLedgerEntryType) == ripple::ltOFFER) {
ripple::SField const* field = nullptr;
// We need a field that contains the TakerGets and TakerPays
// parameters.
if (node.getFName() == ripple::sfModifiedNode)
{
if (node.getFName() == ripple::sfModifiedNode) {
field = &ripple::sfPreviousFields;
}
else if (node.getFName() == ripple::sfCreatedNode)
{
} else if (node.getFName() == ripple::sfCreatedNode) {
field = &ripple::sfNewFields;
}
else if (node.getFName() == ripple::sfDeletedNode)
{
} else if (node.getFName() == ripple::sfDeletedNode) {
field = &ripple::sfFinalFields;
}
if (field != nullptr)
{
auto data = dynamic_cast<const ripple::STObject*>(node.peekAtPField(*field));
if (field != nullptr) {
auto data = dynamic_cast<ripple::STObject const*>(node.peekAtPField(*field));
if ((data != nullptr) && data->isFieldPresent(ripple::sfTakerPays) &&
data->isFieldPresent(ripple::sfTakerGets))
{
data->isFieldPresent(ripple::sfTakerGets)) {
// determine the OrderBook
ripple::Book const book{
data->getFieldAmount(ripple::sfTakerGets).issue(),
data->getFieldAmount(ripple::sfTakerPays).issue()};
if (alreadySent.find(book) == alreadySent.end())
{
if (alreadySent.find(book) == alreadySent.end()) {
bookSubscribers_.publish(pubMsg, book);
alreadySent.insert(book);
}
@@ -259,7 +251,8 @@ SubscriptionManager::pubTransaction(data::TransactionAndMetadata const& blobs, r
void
SubscriptionManager::pubBookChanges(
ripple::LedgerHeader const& lgrInfo,
std::vector<data::TransactionAndMetadata> const& transactions)
std::vector<data::TransactionAndMetadata> const& transactions
)
{
auto const json = rpc::computeBookChanges(lgrInfo, transactions);
auto const bookChangesMsg = std::make_shared<std::string>(boost::json::serialize(json));
@@ -334,8 +327,9 @@ SubscriptionManager::unsubProposedAccount(ripple::AccountID const& account, Sess
void
SubscriptionManager::subProposedTransactions(SessionPtrType session)
{
subscribeHelper(
session, txProposedSubscribers_, [this](SessionPtrType session) { unsubProposedTransactions(session); });
subscribeHelper(session, txProposedSubscribers_, [this](SessionPtrType session) {
unsubProposedTransactions(session);
});
}
void
@@ -360,7 +354,8 @@ SubscriptionManager::subscribeHelper(
SessionPtrType const& session,
Key const& k,
SubscriptionMap<Key>& subs,
CleanupFunction&& func)
CleanupFunction&& func
)
{
if (subs.hasSession(session, k))
return;
@@ -376,8 +371,7 @@ SubscriptionManager::cleanup(SessionPtrType session)
if (!cleanupFuncs_.contains(session))
return;
for (auto const& f : cleanupFuncs_[session])
{
for (auto const& f : cleanupFuncs_[session]) {
f(session);
}

31
src/feed/SubscriptionManager.h
View File

@@ -46,16 +46,12 @@ template <class T>
inline void
sendToSubscribers(std::shared_ptr<std::string> const& message, T& subscribers, std::atomic_uint64_t& counter)
{
for (auto it = subscribers.begin(); it != subscribers.end();)
{
for (auto it = subscribers.begin(); it != subscribers.end();) {
auto& session = *it;
if (session->dead())
{
if (session->dead()) {
it = subscribers.erase(it);
--counter;
}
else
{
} else {
session->send(message);
++it;
}
@@ -73,8 +69,7 @@ template <class T>
inline void
addSession(SessionPtrType session, T& subscribers, std::atomic_uint64_t& counter)
{
if (!subscribers.contains(session))
{
if (!subscribers.contains(session)) {
subscribers.insert(session);
++counter;
}
@@ -91,8 +86,7 @@ template <class T>
inline void
removeSession(SessionPtrType session, T& subscribers, std::atomic_uint64_t& counter)
{
if (subscribers.contains(session))
{
if (subscribers.contains(session)) {
subscribers.erase(session);
--counter;
}
@@ -101,8 +95,7 @@ removeSession(SessionPtrType session, T& subscribers, std::atomic_uint64_t& coun
/**
* @brief Represents a subscription stream.
*/
class Subscription
{
class Subscription {
boost::asio::strand<boost::asio::io_context::executor_type> strand_;
std::unordered_set<SessionPtrType> subscribers_ = {};
std::atomic_uint64_t subCount_ = 0;
@@ -179,8 +172,7 @@ public:
* @brief Represents a collection of subscriptions where each stream is mapped to a key.
*/
template <class Key>
class SubscriptionMap
{
class SubscriptionMap {
using SubscribersType = std::set<SessionPtrType>;
boost::asio::strand<boost::asio::io_context::executor_type> strand_;
@@ -234,8 +226,7 @@ public:
--subCount_;
subscribers_[key].erase(session);
if (subscribers_[key].size() == 0)
{
if (subscribers_[key].size() == 0) {
subscribers_.erase(key);
}
});
@@ -287,8 +278,7 @@ public:
/**
* @brief Manages subscriptions.
*/
class SubscriptionManager
{
class SubscriptionManager {
util::Logger log_{"Subscriptions"};
std::vector<std::thread> workers_;
@@ -385,7 +375,8 @@ public:
ripple::LedgerHeader const& lgrInfo,
ripple::Fees const& fees,
std::string const& ledgerRange,
std::uint32_t txnCount);
std::uint32_t txnCount
);
/**
* @brief Publish to the book changes stream.

34
src/main/Main.cpp
View File

@@ -18,12 +18,6 @@
//==============================================================================
#include <grpc/impl/codegen/port_platform.h>
#ifdef GRPC_TSAN_ENABLED
#undef GRPC_TSAN_ENABLED
#endif
#ifdef GRPC_ASAN_ENABLED
#undef GRPC_ASAN_ENABLED
#endif
#include <data/BackendFactory.h>
#include <etl/ETLService.h>
@@ -79,14 +73,12 @@ parseCli(int argc, char* argv[])
po::store(po::command_line_parser(argc, argv).options(description).positional(positional).run(), parsed);
po::notify(parsed);
if (parsed.count("version") != 0u)
{
if (parsed.count("version") != 0u) {
std::cout << Build::getClioFullVersionString() << '\n';
std::exit(EXIT_SUCCESS);
}
if (parsed.count("help") != 0u)
{
if (parsed.count("help") != 0u) {
std::cout << "Clio server " << Build::getClioFullVersionString() << "\n\n" << description;
std::exit(EXIT_SUCCESS);
}
@@ -153,12 +145,10 @@ start(io_context& ioc, std::uint32_t numThreads)
int
main(int argc, char* argv[])
try
{
try {
auto const configPath = parseCli(argc, argv);
auto const config = ConfigReader::open(configPath);
if (!config)
{
if (!config) {
std::cerr << "Couldnt parse config '" << configPath << "'." << std::endl;
return EXIT_FAILURE;
}
@@ -167,8 +157,7 @@ try
LOG(LogService::info()) << "Clio version: " << Build::getClioFullVersionString();
auto const threads = config.valueOr("io_threads", 2);
if (threads <= 0)
{
if (threads <= 0) {
LOG(LogService::fatal()) << "io_threads is less than 1";
return EXIT_FAILURE;
}
@@ -204,13 +193,16 @@ try
auto workQueue = rpc::WorkQueue::make_WorkQueue(config);
auto counters = rpc::Counters::make_Counters(workQueue);
auto const handlerProvider = std::make_shared<rpc::detail::ProductionHandlerProvider const>(
config, backend, subscriptions, balancer, etl, counters);
config, backend, subscriptions, balancer, etl, counters
);
auto const rpcEngine = rpc::RPCEngine::make_RPCEngine(
backend, subscriptions, balancer, dosGuard, workQueue, counters, handlerProvider);
backend, subscriptions, balancer, dosGuard, workQueue, counters, handlerProvider
);
// Init the web server
auto handler = std::make_shared<web::RPCServerHandler<rpc::RPCEngine, etl::ETLService>>(
config, backend, rpcEngine, etl, subscriptions);
config, backend, rpcEngine, etl, subscriptions
);
auto ctx = parseCerts(config);
auto const ctxRef = ctx ? std::optional<std::reference_wrapper<ssl::context>>{ctx.value()} : std::nullopt;
auto const httpServer = web::make_HttpServer(config, ioc, ctxRef, dosGuard, handler);
@@ -221,8 +213,6 @@ try
start(ioc, threads);
return EXIT_SUCCESS;
}
catch (std::exception const& e)
{
} catch (std::exception const& e) {
LOG(LogService::fatal()) << "Exit on exception: " << e.what();
}

3
src/rpc/Amendments.h
View File

@@ -30,8 +30,7 @@ namespace rpc {
/**
* @brief Represents a list of amendments in the XRPL.
*/
struct Amendments
{
struct Amendments {
/**
* @param name The name of the amendment
* @return The corresponding amendment Id

22
src/rpc/BookChangesHelper.h
View File

@@ -29,8 +29,7 @@ namespace rpc {
/**
* @brief Represents an entry in the book_changes' changes array.
*/
struct BookChange
{
struct BookChange {
ripple::STAmount sideAVolume;
ripple::STAmount sideBVolume;
ripple::STAmount highRate;
@@ -42,8 +41,7 @@ struct BookChange
/**
* @brief Encapsulates the book_changes computations and transformations.
*/
class BookChanges final
{
class BookChanges final {
public:
BookChanges() = delete; // only accessed via static handle function
@@ -60,8 +58,7 @@ public:
}
private:
class HandlerImpl final
{
class HandlerImpl final {
std::map<std::string, BookChange> tally_ = {};
std::optional<uint32_t> offerCancel_ = {};
@@ -78,7 +75,8 @@ private:
std::make_move_iterator(std::begin(tally_)),
std::make_move_iterator(std::end(tally_)),
std::back_inserter(changes),
[](auto obj) { return obj.second; });
[](auto obj) { return obj.second; }
);
return changes;
}
@@ -148,8 +146,7 @@ private:
second = -second;
auto const key = noswap ? (g + '|' + p) : (p + '|' + g);
if (tally_.contains(key))
{
if (tally_.contains(key)) {
auto& entry = tally_.at(key);
entry.sideAVolume += first;
@@ -162,9 +159,7 @@ private:
entry.lowRate = rate;
entry.closeRate = rate;
}
else
{
} else {
tally_[key] = {
.sideAVolume = first,
.sideBVolume = second,
@@ -191,8 +186,7 @@ private:
static std::optional<uint32_t>
shouldCancelOffer(std::shared_ptr<ripple::STTx const> const& tx)
{
switch (tx->getFieldU16(ripple::sfTransactionType))
{
switch (tx->getFieldU16(ripple::sfTransactionType)) {
// in future if any other ways emerge to cancel an offer
// this switch makes them easy to add
case ripple::ttOFFER_CANCEL:

3
src/rpc/Counters.cpp
View File

@@ -112,8 +112,7 @@ Counters::report() const
obj[JS(rpc)] = boost::json::object{};
auto& rpc = obj[JS(rpc)].as_object();
for (auto const& [method, info] : methodInfo_)
{
for (auto const& [method, info] : methodInfo_) {
auto counters = boost::json::object{};
counters[JS(started)] = std::to_string(info.started);
counters[JS(finished)] = std::to_string(info.finished);

8
src/rpc/Counters.h
View File

@@ -33,13 +33,11 @@ namespace rpc {
/**
* @brief Holds information about successful, failed, forwarded, etc. RPC handler calls.
*/
class Counters
{
class Counters {
/**
* @brief All counters the system keeps track of for each RPC method.
*/
struct MethodInfo
{
struct MethodInfo {
std::uint64_t started = 0u;
std::uint64_t finished = 0u;
std::uint64_t failed = 0u;
@@ -59,7 +57,7 @@ class Counters
std::atomic_uint64_t unknownCommandCounter_;
std::atomic_uint64_t internalErrorCounter_;
std::reference_wrapper<const WorkQueue> workQueue_;
std::reference_wrapper<WorkQueue const> workQueue_;
std::chrono::time_point<std::chrono::system_clock> startupTime_;
public:

14
src/rpc/Errors.cpp
View File

@@ -26,8 +26,7 @@ using namespace std;
namespace {
template <class... Ts>
struct overloadSet : Ts...
{
struct overloadSet : Ts... {
using Ts::operator()...;
};
@@ -44,9 +43,8 @@ getWarningInfo(WarningCode code)
constexpr static WarningInfo infos[]{
{warnUNKNOWN, "Unknown warning"},
{warnRPC_CLIO,
"This is a clio server. clio only serves validated data. If you "
"want to talk to rippled, include 'ledger_index':'current' in your "
"request"},
"This is a clio server. clio only serves validated data. If you want to talk to rippled, include "
"'ledger_index':'current' in your request"},
{warnRPC_OUTDATED, "This server may be out of date"},
{warnRPC_RATE_LIMIT, "You are about to be rate limited"},
};
@@ -143,10 +141,10 @@ makeError(Status const& status)
return makeError(err, wrapOptional(status.error), wrapOptional(status.message));
},
},
status.code);
status.code
);
if (status.extraInfo)
{
if (status.extraInfo) {
for (auto& [key, value] : status.extraInfo.value())
res[key] = value;
}

25
src/rpc/Errors.h
View File

@@ -52,8 +52,7 @@ enum class ClioError {
};
/** @brief Holds info about a particular @ref ClioError. */
struct ClioErrorInfo
{
struct ClioErrorInfo {
ClioError const code;
std::string_view const error;
std::string_view const message;
@@ -71,8 +70,7 @@ using RippledError = ripple::error_code_i;
using CombinedError = std::variant<RippledError, ClioError>;
/** @brief A status returned from any RPC handler. */
struct Status
{
struct Status {
CombinedError code = RippledError::rpcSUCCESS;
std::string error;
std::string message;
@@ -141,8 +139,7 @@ struct Status
enum WarningCode { warnUNKNOWN = -1, warnRPC_CLIO = 2001, warnRPC_OUTDATED = 2002, warnRPC_RATE_LIMIT = 2003 };
/** @brief Holds information about a clio warning. */
struct WarningInfo
{
struct WarningInfo {
constexpr WarningInfo() = default;
constexpr WarningInfo(WarningCode code, char const* message) : code(code), message(message)
{
@@ -153,8 +150,7 @@ struct WarningInfo
};
/** @brief Invalid parameters error. */
class InvalidParamsError : public std::exception
{
class InvalidParamsError : public std::exception {
std::string msg;
public:
@@ -162,7 +158,7 @@ public:
{
}
const char*
char const*
what() const throw() override
{
return msg.c_str();
@@ -170,8 +166,7 @@ public:
};
/** @brief Account not found error. */
class AccountNotFoundError : public std::exception
{
class AccountNotFoundError : public std::exception {
std::string account;
public:
@@ -179,7 +174,7 @@ public:
{
}
const char*
char const*
what() const throw() override
{
return account.c_str();
@@ -235,7 +230,8 @@ boost::json::object
makeError(
RippledError err,
std::optional<std::string_view> customError = std::nullopt,
std::optional<std::string_view> customMessage = std::nullopt);
std::optional<std::string_view> customMessage = std::nullopt
);
/**
* @brief Generate JSON from a @ref rpc::ClioError.
@@ -247,6 +243,7 @@ boost::json::object
makeError(
ClioError err,
std::optional<std::string_view> customError = std::nullopt,
std::optional<std::string_view> customMessage = std::nullopt);
std::optional<std::string_view> customMessage = std::nullopt
);
} // namespace rpc

16
src/rpc/Factories.cpp
View File

@@ -33,15 +33,13 @@ make_WsContext(
TagDecoratorFactory const& tagFactory,
data::LedgerRange const& range,
string const& clientIp,
std::reference_wrapper<APIVersionParser const> apiVersionParser)
std::reference_wrapper<APIVersionParser const> apiVersionParser
)
{
boost::json::value commandValue = nullptr;
if (!request.contains("command") && request.contains("method"))
{
if (!request.contains("command") && request.contains("method")) {
commandValue = request.at("method");
}
else if (request.contains("command") && !request.contains("method"))
{
} else if (request.contains("command") && !request.contains("method")) {
commandValue = request.at("command");
}
@@ -64,7 +62,8 @@ make_HttpContext(
data::LedgerRange const& range,
string const& clientIp,
std::reference_wrapper<APIVersionParser const> apiVersionParser,
bool const isAdmin)
bool const isAdmin
)
{
if (!request.contains("method"))
return Error{{ClioError::rpcCOMMAND_IS_MISSING}};
@@ -93,7 +92,8 @@ make_HttpContext(
return Error{{ClioError::rpcINVALID_API_VERSION, apiVersion.error()}};
return web::Context(
yc, command, *apiVersion, array.at(0).as_object(), nullptr, tagFactory, range, clientIp, isAdmin);
yc, command, *apiVersion, array.at(0).as_object(), nullptr, tagFactory, range, clientIp, isAdmin
);
}
} // namespace rpc

6
src/rpc/Factories.h
View File

@@ -60,7 +60,8 @@ make_WsContext(
util::TagDecoratorFactory const& tagFactory,
data::LedgerRange const& range,
std::string const& clientIp,
std::reference_wrapper<APIVersionParser const> apiVersionParser);
std::reference_wrapper<APIVersionParser const> apiVersionParser
);
/**
* @brief A factory function that creates a HTTP context.
@@ -81,6 +82,7 @@ make_HttpContext(
data::LedgerRange const& range,
std::string const& clientIp,
std::reference_wrapper<APIVersionParser const> apiVersionParser,
bool isAdmin);
bool isAdmin
);
} // namespace rpc

29
src/rpc/RPCEngine.h
View File

@@ -59,8 +59,7 @@ namespace rpc {
/**
* @brief The RPC engine that ties all RPC-related functionality together.
*/
class RPCEngine
{
class RPCEngine {
util::Logger perfLog_{"Performance"};
util::Logger log_{"RPC"};
@@ -83,7 +82,8 @@ public:
web::DOSGuard const& dosGuard,
WorkQueue& workQueue,
Counters& counters,
std::shared_ptr<HandlerProvider const> const& handlerProvider)
std::shared_ptr<HandlerProvider const> const& handlerProvider
)
: backend_{backend}
, subscriptions_{subscriptions}
, balancer_{balancer}
@@ -103,10 +103,12 @@ public:
web::DOSGuard const& dosGuard,
WorkQueue& workQueue,
Counters& counters,
std::shared_ptr<HandlerProvider const> const& handlerProvider)
std::shared_ptr<HandlerProvider const> const& handlerProvider
)
{
return std::make_shared<RPCEngine>(
backend, subscriptions, balancer, dosGuard, workQueue, counters, handlerProvider);
backend, subscriptions, balancer, dosGuard, workQueue, counters, handlerProvider
);
}
/**
@@ -121,22 +123,19 @@ public:
if (forwardingProxy_.shouldForward(ctx))
return forwardingProxy_.forward(ctx);
if (backend_->isTooBusy())
{
if (backend_->isTooBusy()) {
LOG(log_.error()) << "Database is too busy. Rejecting request";
notifyTooBusy(); // TODO: should we add ctx.method if we have it?
return Status{RippledError::rpcTOO_BUSY};
}
auto const method = handlerProvider_->getHandler(ctx.method);
if (!method)
{
if (!method) {
notifyUnknownCommand();
return Status{RippledError::rpcUNKNOWN_COMMAND};
}
try
{
try {
LOG(perfLog_.debug()) << ctx.tag() << " start executing rpc `" << ctx.method << '`';
auto const context = Context{ctx.yield, ctx.session, ctx.isAdmin, ctx.clientIp, ctx.apiVersion};
@@ -149,16 +148,12 @@ public:
notifyErrored(ctx.method);
return Status{v.error()};
}
catch (data::DatabaseTimeout const& t)
{
} catch (data::DatabaseTimeout const& t) {
LOG(log_.error()) << "Database timeout";
notifyTooBusy();
return Status{RippledError::rpcTOO_BUSY};
}
catch (std::exception const& ex)
{
} catch (std::exception const& ex) {
LOG(log_.error()) << ctx.tag() << "Caught exception: " << ex.what();
notifyInternalError();

386
src/rpc/RPCHelpers.cpp
View File

@@ -60,12 +60,9 @@ parseAccountCursor(std::optional<std::string> jsonCursor)
if (!std::getline(cursor, value, ','))
return {};
try
{
try {
startHint = boost::lexical_cast<std::uint64_t>(value);
}
catch (boost::bad_lexical_cast&)
{
} catch (boost::bad_lexical_cast&) {
return {};
}
@@ -77,12 +74,12 @@ getDeliveredAmount(
std::shared_ptr<ripple::STTx const> const& txn,
std::shared_ptr<ripple::TxMeta const> const& meta,
std::uint32_t const ledgerSequence,
uint32_t date)
uint32_t date
)
{
if (meta->hasDeliveredAmount())
return meta->getDeliveredAmount();
if (txn->isFieldPresent(ripple::sfAmount))
{
if (txn->isFieldPresent(ripple::sfAmount)) {
using namespace std::chrono_literals;
// Ledger 4594095 is the first ledger in which the DeliveredAmount field
@@ -95,8 +92,7 @@ getDeliveredAmount(
// 446000000 is in Feb 2014, well after DeliveredAmount went live
static std::uint32_t constexpr FIRST_LEDGER_WITH_DELIVERED_AMOUNT = 4594095;
static std::uint32_t constexpr DELIVERED_AMOUNT_LIVE_DATE = 446000000;
if (ledgerSequence >= FIRST_LEDGER_WITH_DELIVERED_AMOUNT || date > DELIVERED_AMOUNT_LIVE_DATE)
{
if (ledgerSequence >= FIRST_LEDGER_WITH_DELIVERED_AMOUNT || date > DELIVERED_AMOUNT_LIVE_DATE) {
return txn->getFieldAmount(ripple::sfAmount);
}
}
@@ -106,7 +102,8 @@ getDeliveredAmount(
bool
canHaveDeliveredAmount(
std::shared_ptr<ripple::STTx const> const& txn,
std::shared_ptr<ripple::TxMeta const> const& meta)
std::shared_ptr<ripple::TxMeta const> const& meta
)
{
ripple::TxType const tt{txn->getTxnType()};
if (tt != ripple::ttPAYMENT && tt != ripple::ttCHECK_CASH && tt != ripple::ttACCOUNT_DELETE)
@@ -124,27 +121,20 @@ accountFromStringStrict(std::string const& account)
auto blob = ripple::strUnHex(account);
std::optional<ripple::PublicKey> publicKey = {};
if (blob && ripple::publicKeyType(ripple::makeSlice(*blob)))
{
if (blob && ripple::publicKeyType(ripple::makeSlice(*blob))) {
publicKey = ripple::PublicKey(ripple::Slice{blob->data(), blob->size()});
}
else
{
} else {
publicKey = ripple::parseBase58<ripple::PublicKey>(ripple::TokenType::AccountPublic, account);
}
std::optional<ripple::AccountID> result;
if (publicKey)
{
if (publicKey) {
result = ripple::calcAccountID(*publicKey);
}
else
{
} else {
result = ripple::parseBase58<ripple::AccountID>(account);
}
if (result)
{
if (result) {
return result.value();
}
return {};
@@ -153,8 +143,7 @@ accountFromStringStrict(std::string const& account)
std::pair<std::shared_ptr<ripple::STTx const>, std::shared_ptr<ripple::STObject const>>
deserializeTxPlusMeta(data::TransactionAndMetadata const& blobs)
{
try
{
try {
std::pair<std::shared_ptr<ripple::STTx const>, std::shared_ptr<ripple::STObject const>> result;
{
ripple::SerialIter s{blobs.transaction.data(), blobs.transaction.size()};
@@ -165,9 +154,7 @@ deserializeTxPlusMeta(data::TransactionAndMetadata const& blobs)
result.second = std::make_shared<ripple::STObject const>(s, ripple::sfMetadata);
}
return result;
}
catch (std::exception const& e)
{
} catch (std::exception const& e) {
std::stringstream txn;
std::stringstream meta;
std::copy(blobs.transaction.begin(), blobs.transaction.end(), std::ostream_iterator<unsigned char>(txn));
@@ -205,24 +192,20 @@ toExpandedJson(data::TransactionAndMetadata const& blobs, NFTokenjson nftEnabled
auto metaJson = toJson(*meta);
insertDeliveredAmount(metaJson, txn, meta, blobs.date);
if (nftEnabled == NFTokenjson::ENABLE)
{
if (nftEnabled == NFTokenjson::ENABLE) {
Json::Value nftJson;
ripple::insertNFTSyntheticInJson(nftJson, txn, *meta);
// if there is no nft fields, the nftJson will be {"meta":null}
auto const nftBoostJson = toBoostJson(nftJson).as_object();
if (nftBoostJson.contains(JS(meta)) and nftBoostJson.at(JS(meta)).is_object())
{
if (nftBoostJson.contains(JS(meta)) and nftBoostJson.at(JS(meta)).is_object()) {
for (auto const& [k, v] : nftBoostJson.at(JS(meta)).as_object())
metaJson.insert_or_assign(k, v);
}
}
if (networkId)
{
if (networkId) {
// networkId is available, insert ctid field to tx
if (auto const ctid = rpc::encodeCTID(meta->getLgrSeq(), meta->getIndex(), *networkId))
{
if (auto const ctid = rpc::encodeCTID(meta->getLgrSeq(), meta->getIndex(), *networkId)) {
txnJson[JS(ctid)] = *ctid;
}
}
@@ -249,16 +232,13 @@ insertDeliveredAmount(
boost::json::object& metaJson,
std::shared_ptr<ripple::STTx const> const& txn,
std::shared_ptr<ripple::TxMeta const> const& meta,
uint32_t date)
uint32_t date
)
{
if (canHaveDeliveredAmount(txn, meta))
{
if (auto amt = getDeliveredAmount(txn, meta, meta->getLgrSeq(), date))
{
if (canHaveDeliveredAmount(txn, meta)) {
if (auto amt = getDeliveredAmount(txn, meta, meta->getLgrSeq(), date)) {
metaJson["delivered_amount"] = toBoostJson(amt->getJson(ripple::JsonOptions::include_date));
}
else
{
} else {
metaJson["delivered_amount"] = "unavailable";
}
return true;
@@ -286,10 +266,8 @@ boost::json::object
toJson(ripple::SLE const& sle)
{
boost::json::value value = boost::json::parse(sle.getJson(ripple::JsonOptions::none).toStyledString());
if (sle.getType() == ripple::ltACCOUNT_ROOT)
{
if (sle.isFieldPresent(ripple::sfEmailHash))
{
if (sle.getType() == ripple::ltACCOUNT_ROOT) {
if (sle.isFieldPresent(ripple::sfEmailHash)) {
auto const& hash = sle.getFieldH128(ripple::sfEmailHash);
std::string md5 = strHex(hash);
boost::algorithm::to_lower(md5);
@@ -322,12 +300,9 @@ std::optional<std::uint32_t>
parseStringAsUInt(std::string const& value)
{
std::optional<std::uint32_t> index = {};
try
{
try {
index = boost::lexical_cast<std::uint32_t>(value);
}
catch (boost::bad_lexical_cast const&)
{
} catch (boost::bad_lexical_cast const&) {
}
return index;
@@ -338,8 +313,7 @@ ledgerInfoFromRequest(std::shared_ptr<data::BackendInterface const> const& backe
{
auto hashValue = ctx.params.contains("ledger_hash") ? ctx.params.at("ledger_hash") : nullptr;
if (!hashValue.is_null())
{
if (!hashValue.is_null()) {
if (!hashValue.is_string())
return Status{RippledError::rpcINVALID_PARAMS, "ledgerHashNotString"};
@@ -358,25 +332,17 @@ ledgerInfoFromRequest(std::shared_ptr<data::BackendInterface const> const& backe
auto indexValue = ctx.params.contains("ledger_index") ? ctx.params.at("ledger_index") : nullptr;
std::optional<std::uint32_t> ledgerSequence = {};
if (!indexValue.is_null())
{
if (indexValue.is_string())
{
if (!indexValue.is_null()) {
if (indexValue.is_string()) {
boost::json::string const& stringIndex = indexValue.as_string();
if (stringIndex == "validated")
{
if (stringIndex == "validated") {
ledgerSequence = ctx.range.maxSequence;
}
else
{
} else {
ledgerSequence = parseStringAsUInt(stringIndex.c_str());
}
}
else if (indexValue.is_int64())
} else if (indexValue.is_int64())
ledgerSequence = indexValue.as_int64();
}
else
{
} else {
ledgerSequence = ctx.range.maxSequence;
}
@@ -398,12 +364,12 @@ getLedgerInfoFromHashOrSeq(
boost::asio::yield_context yield,
std::optional<std::string> ledgerHash,
std::optional<uint32_t> ledgerIndex,
uint32_t maxSeq)
uint32_t maxSeq
)
{
std::optional<ripple::LedgerHeader> lgrInfo;
auto const err = Status{RippledError::rpcLGR_NOT_FOUND, "ledgerNotFound"};
if (ledgerHash)
{
if (ledgerHash) {
// invoke uint256's constructor to parse the hex string , instead of
// copying buffer
ripple::uint256 const ledgerHash256{std::string_view(*ledgerHash)};
@@ -446,10 +412,8 @@ ledgerInfoToBlob(ripple::LedgerHeader const& info, bool includeHash)
std::uint64_t
getStartHint(ripple::SLE const& sle, ripple::AccountID const& accountID)
{
if (sle.getType() == ripple::ltRIPPLE_STATE)
{
if (sle.getFieldAmount(ripple::sfLowLimit).getIssuer() == accountID)
{
if (sle.getType() == ripple::ltRIPPLE_STATE) {
if (sle.getFieldAmount(ripple::sfLowLimit).getIssuer() == accountID) {
return sle.getFieldU64(ripple::sfLowNode);
}
if (sle.getFieldAmount(ripple::sfHighLimit).getIssuer() == accountID)
@@ -473,7 +437,8 @@ traverseNFTObjects(
ripple::uint256 nextPage,
std::uint32_t limit,
boost::asio::yield_context yield,
std::function<void(ripple::SLE&&)> atOwnedNode)
std::function<void(ripple::SLE&&)> atOwnedNode
)
{
auto const firstNFTPage = ripple::keylet::nftpage_min(accountID);
auto const lastNFTPage = ripple::keylet::nftpage_max(accountID);
@@ -488,10 +453,8 @@ traverseNFTObjects(
// read the current page
auto page = backend.fetchLedgerObject(currentPage, sequence, yield);
if (!page)
{
if (nextPage == beast::zero)
{ // no nft objects in lastNFTPage
if (!page) {
if (nextPage == beast::zero) { // no nft objects in lastNFTPage
return AccountCursor{beast::zero, 0};
} // marker is in the right range, but still invalid
return Status{RippledError::rpcINVALID_PARAMS, "Invalid marker."};
@@ -502,8 +465,7 @@ traverseNFTObjects(
auto count = 0u;
// traverse the nft page linked list until the start of the list or reach the limit
while (true)
{
while (true) {
auto const nftPreviousPage = pageSLE.getFieldH256(ripple::sfPreviousPageMin);
atOwnedNode(std::move(pageSLE));
count++;
@@ -527,7 +489,8 @@ traverseOwnedNodes(
std::optional<std::string> jsonCursor,
boost::asio::yield_context yield,
std::function<void(ripple::SLE&&)> atOwnedNode,
bool nftIncluded)
bool nftIncluded
)
{
auto const maybeCursor = parseAccountCursor(jsonCursor);
@@ -543,8 +506,7 @@ traverseOwnedNodes(
// if we need to traverse nft objects and the marker is still in nft page -> traverse nft objects
// if we need to traverse nft objects and the marker is still in nft page but next page is zero -> owned nodes
// if we need to traverse nft objects and the marker is not in nft page -> traverse owned nodes
if (nftIncluded and (!jsonCursor or isNftMarkerNonZero))
{
if (nftIncluded and (!jsonCursor or isNftMarkerNonZero)) {
auto const cursorMaybe = traverseNFTObjects(backend, sequence, accountID, hexCursor, limit, yield, atOwnedNode);
if (auto const status = std::get_if<Status>(&cursorMaybe))
@@ -560,16 +522,15 @@ traverseOwnedNodes(
limit -= nftsCount;
hexCursor = beast::zero;
startHint = 0;
}
else if (nftIncluded and isNftMarkerZero)
{
} else if (nftIncluded and isNftMarkerZero) {
// the last request happen to fetch all the nft, adjust marker to continue traversing owned nodes
hexCursor = beast::zero;
startHint = 0;
}
return traverseOwnedNodes(
backend, ripple::keylet::ownerDir(accountID), hexCursor, startHint, sequence, limit, yield, atOwnedNode);
backend, ripple::keylet::ownerDir(accountID), hexCursor, startHint, sequence, limit, yield, atOwnedNode
);
}
std::variant<Status, AccountCursor>
@@ -581,7 +542,8 @@ traverseOwnedNodes(
std::uint32_t sequence,
std::uint32_t limit,
boost::asio::yield_context yield,
std::function<void(ripple::SLE&&)> atOwnedNode)
std::function<void(ripple::SLE&&)> atOwnedNode
)
{
auto cursor = AccountCursor({beast::zero, 0});
@@ -600,8 +562,7 @@ traverseOwnedNodes(
auto start = std::chrono::system_clock::now();
// If startAfter is not zero try jumping to that page using the hint
if (hexMarker.isNonZero())
{
if (hexMarker.isNonZero()) {
auto const hintIndex = ripple::keylet::page(rootIndex, startHint);
auto hintDir = backend.fetchLedgerObject(hintIndex.key, sequence, yield);
@@ -612,16 +573,14 @@ traverseOwnedNodes(
ripple::SLE const hintDirSle{hintDirIt, hintIndex.key};
if (auto const& indexes = hintDirSle.getFieldV256(ripple::sfIndexes);
std::find(std::begin(indexes), std::end(indexes), hexMarker) == std::end(indexes))
{
std::find(std::begin(indexes), std::end(indexes), hexMarker) == std::end(indexes)) {
// the index specified by marker is not in the page specified by marker
return Status(ripple::rpcINVALID_PARAMS, "Invalid marker.");
}
currentIndex = hintIndex;
bool found = false;
for (;;)
{
for (;;) {
auto const ownerDir = backend.fetchLedgerObject(currentIndex.key, sequence, yield);
if (!ownerDir)
@@ -630,26 +589,20 @@ traverseOwnedNodes(
ripple::SerialIter ownedDirIt{ownerDir->data(), ownerDir->size()};
ripple::SLE const ownedDirSle{ownedDirIt, currentIndex.key};
for (auto const& key : ownedDirSle.getFieldV256(ripple::sfIndexes))
{
if (!found)
{
for (auto const& key : ownedDirSle.getFieldV256(ripple::sfIndexes)) {
if (!found) {
if (key == hexMarker)
found = true;
}
else
{
} else {
keys.push_back(key);
if (--limit == 0)
{
if (--limit == 0) {
break;
}
}
}
if (limit == 0)
{
if (limit == 0) {
cursor = AccountCursor({keys.back(), currentPage});
break;
}
@@ -661,11 +614,8 @@ traverseOwnedNodes(
currentIndex = ripple::keylet::page(rootIndex, uNodeNext);
currentPage = uNodeNext;
}
}
else
{
for (;;)
{
} else {
for (;;) {
auto const ownerDir = backend.fetchLedgerObject(currentIndex.key, sequence, yield);
if (!ownerDir)
@@ -674,16 +624,14 @@ traverseOwnedNodes(
ripple::SerialIter ownedDirIt{ownerDir->data(), ownerDir->size()};
ripple::SLE const ownedDirSle{ownedDirIt, currentIndex.key};
for (auto const& key : ownedDirSle.getFieldV256(ripple::sfIndexes))
{
for (auto const& key : ownedDirSle.getFieldV256(ripple::sfIndexes)) {
keys.push_back(key);
if (--limit == 0)
break;
}
if (limit == 0)
{
if (limit == 0) {
cursor = AccountCursor({keys.back(), currentPage});
break;
}
@@ -701,14 +649,14 @@ traverseOwnedNodes(
LOG(gLog.debug()) << fmt::format(
"Time loading owned directories: {} milliseconds, entries size: {}",
std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count(),
keys.size());
keys.size()
);
auto [objects, timeDiff] = util::timed([&]() { return backend.fetchLedgerObjects(keys, sequence, yield); });
LOG(gLog.debug()) << "Time loading owned entries: " << timeDiff << " milliseconds";
for (auto i = 0u; i < objects.size(); ++i)
{
for (auto i = 0u; i < objects.size(); ++i) {
ripple::SerialIter it{objects[i].data(), objects[i].size()};
atOwnedNode(ripple::SLE{it, keys[i]});
}
@@ -724,10 +672,10 @@ read(
std::shared_ptr<data::BackendInterface const> const& backend,
ripple::Keylet const& keylet,
ripple::LedgerHeader const& lgrInfo,
web::Context const& context)
web::Context const& context
)
{
if (auto const blob = backend->fetchLedgerObject(keylet.key, lgrInfo.seq, context.yield); blob)
{
if (auto const blob = backend->fetchLedgerObject(keylet.key, lgrInfo.seq, context.yield); blob) {
return std::make_shared<ripple::SLE const>(ripple::SerialIter{blob->data(), blob->size()}, keylet.key);
}
@@ -766,10 +714,8 @@ keypairFromRequst(boost::json::object const& request)
// Identify which secret type is in use.
std::string secretType;
int count = 0;
for (const auto& t : secretTypes)
{
if (request.contains(t))
{
for (auto const& t : secretTypes) {
if (request.contains(t)) {
++count;
secretType = t;
}
@@ -778,19 +724,16 @@ keypairFromRequst(boost::json::object const& request)
if (count == 0)
return Status{RippledError::rpcINVALID_PARAMS, "missing field secret"};
if (count > 1)
{
if (count > 1) {
return Status{
RippledError::rpcINVALID_PARAMS,
"Exactly one of the following must be specified: "
" passphrase, secret, seed, or seed_hex"};
"Exactly one of the following must be specified: passphrase, secret, seed, or seed_hex"};
}
std::optional<ripple::KeyType> keyType;
std::optional<ripple::Seed> seed;
if (has_key_type)
{
if (has_key_type) {
if (!request.at("key_type").is_string())
return Status{RippledError::rpcINVALID_PARAMS, "keyTypeNotString"};
@@ -807,12 +750,10 @@ keypairFromRequst(boost::json::object const& request)
// ripple-lib encodes seed used to generate an Ed25519 wallet in a
// non-standard way. While we never encode seeds that way, we try
// to detect such keys to avoid user confusion.
if (secretType != "seed_hex")
{
if (secretType != "seed_hex") {
seed = parseRippleLibSeed(request.at(secretType));
if (seed)
{
if (seed) {
// If the user passed in an Ed25519 seed but *explicitly*
// requested another key type, return an error.
if (keyType.value_or(ripple::KeyType::ed25519) != ripple::KeyType::ed25519)
@@ -825,32 +766,23 @@ keypairFromRequst(boost::json::object const& request)
if (!keyType)
keyType = ripple::KeyType::secp256k1;
if (!seed)
{
if (has_key_type)
{
if (!seed) {
if (has_key_type) {
if (!request.at(secretType).is_string())
return Status{RippledError::rpcINVALID_PARAMS, "secret value must be string"};
std::string const key = request.at(secretType).as_string().c_str();
if (secretType == "seed")
{
if (secretType == "seed") {
seed = ripple::parseBase58<ripple::Seed>(key);
}
else if (secretType == "passphrase")
{
} else if (secretType == "passphrase") {
seed = ripple::parseGenericSeed(key);
}
else if (secretType == "seed_hex")
{
} else if (secretType == "seed_hex") {
ripple::uint128 s;
if (s.parseHex(key))
seed.emplace(ripple::Slice(s.data(), ripple::uint128::size()));
}
}
else
{
} else {
if (!request.at("secret").is_string())
return Status{RippledError::rpcINVALID_PARAMS, "field secret should be a string"};
@@ -872,18 +804,13 @@ std::vector<ripple::AccountID>
getAccountsFromTransaction(boost::json::object const& transaction)
{
std::vector<ripple::AccountID> accounts = {};
for (auto const& [key, value] : transaction)
{
if (value.is_object())
{
for (auto const& [key, value] : transaction) {
if (value.is_object()) {
auto inObject = getAccountsFromTransaction(value.as_object());
accounts.insert(accounts.end(), inObject.begin(), inObject.end());
}
else if (value.is_string())
{
} else if (value.is_string()) {
auto account = ripple::parseBase58<ripple::AccountID>(value.as_string().c_str());
if (account)
{
if (account) {
accounts.push_back(*account);
}
}
@@ -897,7 +824,8 @@ isGlobalFrozen(
BackendInterface const& backend,
std::uint32_t sequence,
ripple::AccountID const& issuer,
boost::asio::yield_context yield)
boost::asio::yield_context yield
)
{
if (ripple::isXRP(issuer))
return false;
@@ -921,7 +849,8 @@ isFrozen(
ripple::AccountID const& account,
ripple::Currency const& currency,
ripple::AccountID const& issuer,
boost::asio::yield_context yield)
boost::asio::yield_context yield
)
{
if (ripple::isXRP(currency))
return false;
@@ -938,8 +867,7 @@ isFrozen(
if (sle.isFlag(ripple::lsfGlobalFreeze))
return true;
if (issuer != account)
{
if (issuer != account) {
key = ripple::keylet::line(account, issuer, currency).key;
blob = backend.fetchLedgerObject(key, sequence, yield);
@@ -963,7 +891,8 @@ xrpLiquid(
BackendInterface const& backend,
std::uint32_t sequence,
ripple::AccountID const& id,
boost::asio::yield_context yield)
boost::asio::yield_context yield
)
{
auto key = ripple::keylet::account(id).key;
auto blob = backend.fetchLedgerObject(key, sequence, yield);
@@ -993,10 +922,10 @@ accountFunds(
std::uint32_t const sequence,
ripple::STAmount const& amount,
ripple::AccountID const& id,
boost::asio::yield_context yield)
boost::asio::yield_context yield
)
{
if (!amount.native() && amount.getIssuer() == id)
{
if (!amount.native() && amount.getIssuer() == id) {
return amount;
}
@@ -1011,19 +940,18 @@ accountHolds(
ripple::Currency const& currency,
ripple::AccountID const& issuer,
bool const zeroIfFrozen,
boost::asio::yield_context yield)
boost::asio::yield_context yield
)
{
ripple::STAmount amount;
if (ripple::isXRP(currency))
{
if (ripple::isXRP(currency)) {
return {xrpLiquid(backend, sequence, account, yield)};
}
auto key = ripple::keylet::line(account, issuer, currency).key;
auto const blob = backend.fetchLedgerObject(key, sequence, yield);
if (!blob)
{
if (!blob) {
amount.clear({currency, issuer});
return amount;
}
@@ -1031,15 +959,11 @@ accountHolds(
ripple::SerialIter it{blob->data(), blob->size()};
ripple::SLE const sle{it, key};
if (zeroIfFrozen && isFrozen(backend, sequence, account, currency, issuer, yield))
{
if (zeroIfFrozen && isFrozen(backend, sequence, account, currency, issuer, yield)) {
amount.clear(ripple::Issue(currency, issuer));
}
else
{
} else {
amount = sle.getFieldAmount(ripple::sfBalance);
if (account > issuer)
{
if (account > issuer) {
// Put balance in account terms.
amount.negate();
}
@@ -1054,13 +978,13 @@ transferRate(
BackendInterface const& backend,
std::uint32_t sequence,
ripple::AccountID const& issuer,
boost::asio::yield_context yield)
boost::asio::yield_context yield
)
{
auto key = ripple::keylet::account(issuer).key;
auto blob = backend.fetchLedgerObject(key, sequence, yield);
if (blob)
{
if (blob) {
ripple::SerialIter it{blob->data(), blob->size()};
ripple::SLE const sle{it, key};
@@ -1078,7 +1002,8 @@ postProcessOrderBook(
ripple::AccountID const& takerID,
data::BackendInterface const& backend,
std::uint32_t const ledgerSequence,
boost::asio::yield_context yield)
boost::asio::yield_context yield
)
{
boost::json::array jsonOffers;
@@ -1089,10 +1014,8 @@ postProcessOrderBook(
auto rate = transferRate(backend, ledgerSequence, book.out.account, yield);
for (auto const& obj : offers)
{
try
{
for (auto const& obj : offers) {
try {
ripple::SerialIter it{obj.blob.data(), obj.blob.size()};
ripple::SLE const offer{it, obj.key};
ripple::uint256 const bookDir = offer.getFieldH256(ripple::sfBookDirectory);
@@ -1103,32 +1026,25 @@ postProcessOrderBook(
ripple::STAmount saOwnerFunds;
bool firstOwnerOffer = true;
if (book.out.account == uOfferOwnerID)
{
if (book.out.account == uOfferOwnerID) {
// If an offer is selling issuer's own IOUs, it is fully
// funded.
saOwnerFunds = saTakerGets;
}
else if (globalFreeze)
{
} else if (globalFreeze) {
// If either asset is globally frozen, consider all offers
// that aren't ours to be totally unfunded
saOwnerFunds.clear(book.out);
}
else
{
} else {
auto umBalanceEntry = umBalance.find(uOfferOwnerID);
if (umBalanceEntry != umBalance.end())
{
if (umBalanceEntry != umBalance.end()) {
// Found in running balance table.
saOwnerFunds = umBalanceEntry->second;
firstOwnerOffer = false;
}
else
{
} else {
saOwnerFunds = accountHolds(
backend, ledgerSequence, uOfferOwnerID, book.out.currency, book.out.account, true, yield);
backend, ledgerSequence, uOfferOwnerID, book.out.currency, book.out.account, true, yield
);
if (saOwnerFunds < beast::zero)
saOwnerFunds.clear();
@@ -1154,13 +1070,10 @@ postProcessOrderBook(
saOwnerFundsLimit = ripple::divide(saOwnerFunds, offerRate);
}
if (saOwnerFundsLimit >= saTakerGets)
{
if (saOwnerFundsLimit >= saTakerGets) {
// Sufficient funds no shenanigans.
saTakerGetsFunded = saTakerGets;
}
else
{
} else {
saTakerGetsFunded = saOwnerFundsLimit;
offerJson["taker_gets_funded"] = toBoostJson(saTakerGetsFunded.getJson(ripple::JsonOptions::none));
offerJson["taker_pays_funded"] =
@@ -1180,9 +1093,7 @@ postProcessOrderBook(
offerJson["quality"] = dirRate.getText();
jsonOffers.push_back(offerJson);
}
catch (std::exception const& e)
{
} catch (std::exception const& e) {
LOG(gLog.error()) << "caught exception: " << e.what();
}
}
@@ -1193,26 +1104,22 @@ postProcessOrderBook(
std::variant<Status, ripple::Book>
parseBook(ripple::Currency pays, ripple::AccountID payIssuer, ripple::Currency gets, ripple::AccountID getIssuer)
{
if (isXRP(pays) && !isXRP(payIssuer))
{
if (isXRP(pays) && !isXRP(payIssuer)) {
return Status{
RippledError::rpcSRC_ISR_MALFORMED, "Unneeded field 'taker_pays.issuer' for XRP currency specification."};
}
if (!isXRP(pays) && isXRP(payIssuer))
{
if (!isXRP(pays) && isXRP(payIssuer)) {
return Status{
RippledError::rpcSRC_ISR_MALFORMED, "Invalid field 'taker_pays.issuer', expected non-XRP issuer."};
}
if (ripple::isXRP(gets) && !ripple::isXRP(getIssuer))
{
if (ripple::isXRP(gets) && !ripple::isXRP(getIssuer)) {
return Status{
RippledError::rpcDST_ISR_MALFORMED, "Unneeded field 'taker_gets.issuer' for XRP currency specification."};
}
if (!ripple::isXRP(gets) && ripple::isXRP(getIssuer))
{
if (!ripple::isXRP(gets) && ripple::isXRP(getIssuer)) {
return Status{
RippledError::rpcDST_ISR_MALFORMED, "Invalid field 'taker_gets.issuer', expected non-XRP issuer."};
}
@@ -1249,8 +1156,7 @@ parseBook(boost::json::object const& request)
if (!taker_gets.contains("currency"))
return Status{RippledError::rpcDST_AMT_MALFORMED};
if (!taker_gets.at("currency").is_string())
{
if (!taker_gets.at("currency").is_string()) {
return Status{
RippledError::rpcDST_AMT_MALFORMED,
};
@@ -1265,8 +1171,7 @@ parseBook(boost::json::object const& request)
return Status{RippledError::rpcDST_AMT_MALFORMED};
ripple::AccountID pay_issuer;
if (taker_pays.contains("issuer"))
{
if (taker_pays.contains("issuer")) {
if (!taker_pays.at("issuer").is_string())
return Status{RippledError::rpcINVALID_PARAMS, "takerPaysIssuerNotString"};
@@ -1275,20 +1180,16 @@ parseBook(boost::json::object const& request)
if (pay_issuer == ripple::noAccount())
return Status{RippledError::rpcSRC_ISR_MALFORMED};
}
else
{
} else {
pay_issuer = ripple::xrpAccount();
}
if (isXRP(pay_currency) && !isXRP(pay_issuer))
{
if (isXRP(pay_currency) && !isXRP(pay_issuer)) {
return Status{
RippledError::rpcSRC_ISR_MALFORMED, "Unneeded field 'taker_pays.issuer' for XRP currency specification."};
}
if (!isXRP(pay_currency) && isXRP(pay_issuer))
{
if (!isXRP(pay_currency) && isXRP(pay_issuer)) {
return Status{
RippledError::rpcSRC_ISR_MALFORMED, "Invalid field 'taker_pays.issuer', expected non-XRP issuer."};
}
@@ -1298,33 +1199,27 @@ parseBook(boost::json::object const& request)
ripple::AccountID get_issuer;
if (taker_gets.contains("issuer"))
{
if (taker_gets.contains("issuer")) {
if (!taker_gets["issuer"].is_string())
return Status{RippledError::rpcINVALID_PARAMS, "taker_gets.issuer should be string"};
if (!ripple::to_issuer(get_issuer, taker_gets.at("issuer").as_string().c_str()))
return Status{RippledError::rpcDST_ISR_MALFORMED, "Invalid field 'taker_gets.issuer', bad issuer."};
if (get_issuer == ripple::noAccount())
{
if (get_issuer == ripple::noAccount()) {
return Status{
RippledError::rpcDST_ISR_MALFORMED, "Invalid field 'taker_gets.issuer', bad issuer account one."};
}
}
else
{
} else {
get_issuer = ripple::xrpAccount();
}
if (ripple::isXRP(get_currency) && !ripple::isXRP(get_issuer))
{
if (ripple::isXRP(get_currency) && !ripple::isXRP(get_issuer)) {
return Status{
RippledError::rpcDST_ISR_MALFORMED, "Unneeded field 'taker_gets.issuer' for XRP currency specification."};
}
if (!ripple::isXRP(get_currency) && ripple::isXRP(get_issuer))
{
if (!ripple::isXRP(get_currency) && ripple::isXRP(get_issuer)) {
return Status{
RippledError::rpcDST_ISR_MALFORMED, "Invalid field 'taker_gets.issuer', expected non-XRP issuer."};
}
@@ -1351,11 +1246,9 @@ parseTaker(boost::json::value const& taker)
bool
specifiesCurrentOrClosedLedger(boost::json::object const& request)
{
if (request.contains("ledger_index"))
{
if (request.contains("ledger_index")) {
auto indexValue = request.at("ledger_index");
if (indexValue.is_string())
{
if (indexValue.is_string()) {
std::string const index = indexValue.as_string().c_str();
return index == "current" || index == "closed";
}
@@ -1384,7 +1277,8 @@ isAmendmentEnabled(
std::shared_ptr<data::BackendInterface const> const& backend,
boost::asio::yield_context yield,
uint32_t seq,
ripple::uint256 amendmentId)
ripple::uint256 amendmentId
)
{
// the amendments should always be present in ledger
auto const& amendments = backend->fetchLedgerObject(ripple::keylet::amendments().key, seq, yield);

63
src/rpc/RPCHelpers.h
View File

@@ -74,14 +74,16 @@ std::pair<boost::json::object, boost::json::object>
toExpandedJson(
data::TransactionAndMetadata const& blobs,
NFTokenjson nftEnabled = NFTokenjson::DISABLE,
std::optional<uint16_t> networkId = std::nullopt);
std::optional<uint16_t> networkId = std::nullopt
);
bool
insertDeliveredAmount(
boost::json::object& metaJson,
std::shared_ptr<ripple::STTx const> const& txn,
std::shared_ptr<ripple::TxMeta const> const& meta,
uint32_t date);
uint32_t date
);
boost::json::object
toJson(ripple::STBase const& obj);
@@ -104,7 +106,8 @@ generatePubLedgerMessage(
ripple::LedgerHeader const& lgrInfo,
ripple::Fees const& fees,
std::string const& ledgerRange,
std::uint32_t txnCount);
std::uint32_t txnCount
);
std::variant<Status, ripple::LedgerHeader>
ledgerInfoFromRequest(std::shared_ptr<data::BackendInterface const> const& backend, web::Context const& ctx);
@@ -115,7 +118,8 @@ getLedgerInfoFromHashOrSeq(
boost::asio::yield_context yield,
std::optional<std::string> ledgerHash,
std::optional<uint32_t> ledgerIndex,
uint32_t maxSeq);
uint32_t maxSeq
);
std::variant<Status, AccountCursor>
traverseOwnedNodes(
@@ -126,7 +130,8 @@ traverseOwnedNodes(
std::uint32_t sequence,
std::uint32_t limit,
boost::asio::yield_context yield,
std::function<void(ripple::SLE&&)> atOwnedNode);
std::function<void(ripple::SLE&&)> atOwnedNode
);
// Remove the account check from traverseOwnedNodes
// Account check has been done by framework,remove it from internal function
@@ -139,14 +144,16 @@ traverseOwnedNodes(
std::optional<std::string> jsonCursor,
boost::asio::yield_context yield,
std::function<void(ripple::SLE&&)> atOwnedNode,
bool nftIncluded = false);
bool nftIncluded = false
);
std::shared_ptr<ripple::SLE const>
read(
std::shared_ptr<data::BackendInterface const> const& backend,
ripple::Keylet const& keylet,
ripple::LedgerHeader const& lgrInfo,
web::Context const& context);
web::Context const& context
);
std::variant<Status, std::pair<ripple::PublicKey, ripple::SecretKey>>
keypairFromRequst(boost::json::object const& request);
@@ -162,7 +169,8 @@ isGlobalFrozen(
BackendInterface const& backend,
std::uint32_t seq,
ripple::AccountID const& issuer,
boost::asio::yield_context yield);
boost::asio::yield_context yield
);
bool
isFrozen(
@@ -171,7 +179,8 @@ isFrozen(
ripple::AccountID const& account,
ripple::Currency const& currency,
ripple::AccountID const& issuer,
boost::asio::yield_context yield);
boost::asio::yield_context yield
);
ripple::STAmount
accountFunds(
@@ -179,7 +188,8 @@ accountFunds(
std::uint32_t sequence,
ripple::STAmount const& amount,
ripple::AccountID const& id,
boost::asio::yield_context yield);
boost::asio::yield_context yield
);
ripple::STAmount
accountHolds(
@@ -189,21 +199,24 @@ accountHolds(
ripple::Currency const& currency,
ripple::AccountID const& issuer,
bool zeroIfFrozen,
boost::asio::yield_context yield);
boost::asio::yield_context yield
);
ripple::Rate
transferRate(
BackendInterface const& backend,
std::uint32_t sequence,
ripple::AccountID const& issuer,
boost::asio::yield_context yield);
boost::asio::yield_context yield
);
ripple::XRPAmount
xrpLiquid(
BackendInterface const& backend,
std::uint32_t sequence,
ripple::AccountID const& id,
boost::asio::yield_context yield);
boost::asio::yield_context yield
);
boost::json::array
postProcessOrderBook(
@@ -212,7 +225,8 @@ postProcessOrderBook(
ripple::AccountID const& takerID,
data::BackendInterface const& backend,
std::uint32_t ledgerSequence,
boost::asio::yield_context yield);
boost::asio::yield_context yield
);
std::variant<Status, ripple::Book>
parseBook(ripple::Currency pays, ripple::AccountID payIssuer, ripple::Currency gets, ripple::AccountID getIssuer);
@@ -234,7 +248,8 @@ isAmendmentEnabled(
std::shared_ptr<data::BackendInterface const> const& backend,
boost::asio::yield_context yield,
uint32_t seq,
ripple::uint256 amendmentId);
ripple::uint256 amendmentId
);
std::optional<std::string>
encodeCTID(uint32_t ledgerSeq, uint16_t txnIndex, uint16_t networkId) noexcept;
@@ -244,8 +259,7 @@ inline std::optional<std::tuple<uint32_t, uint16_t, uint16_t>>
decodeCTID(T const ctid) noexcept
{
auto const getCTID64 = [](T const ctid) noexcept -> std::optional<uint64_t> {
if constexpr (std::is_convertible_v<T, std::string>)
{
if constexpr (std::is_convertible_v<T, std::string>) {
std::string const ctidString(ctid);
static std::size_t constexpr CTID_STRING_LENGTH = 16;
if (ctidString.length() != CTID_STRING_LENGTH)
@@ -289,19 +303,14 @@ logDuration(web::Context const& ctx, T const& dur)
auto const millis = std::chrono::duration_cast<std::chrono::milliseconds>(dur).count();
auto const seconds = std::chrono::duration_cast<std::chrono::seconds>(dur).count();
auto const msg = fmt::format(
"Request processing duration = {} milliseconds. request = {}",
millis,
serialize(util::removeSecret(ctx.params)));
"Request processing duration = {} milliseconds. request = {}", millis, serialize(util::removeSecret(ctx.params))
);
if (seconds > DURATION_ERROR_THRESHOLD_SECONDS)
{
if (seconds > DURATION_ERROR_THRESHOLD_SECONDS) {
LOG(log.error()) << ctx.tag() << msg;
}
else if (seconds > 1)
{
} else if (seconds > 1) {
LOG(log.warn()) << ctx.tag() << msg;
}
else
} else
LOG(log.info()) << ctx.tag() << msg;
}

9
src/rpc/WorkQueue.h
View File

@@ -37,8 +37,7 @@ namespace rpc {
/**
* @brief An asynchronous, thread-safe queue for RPC requests.
*/
class WorkQueue
{
class WorkQueue {
// these are cumulative for the lifetime of the process
std::atomic_uint64_t queued_ = 0;
std::atomic_uint64_t durationUs_ = 0;
@@ -90,8 +89,7 @@ public:
bool
postCoro(FnType&& func, bool isWhiteListed)
{
if (curSize_ >= maxSize_ && !isWhiteListed)
{
if (curSize_ >= maxSize_ && !isWhiteListed) {
LOG(log_.warn()) << "Queue is full. rejecting job. current size = " << curSize_
<< "; max size = " << maxSize_;
return false;
@@ -113,7 +111,8 @@ public:
func(yield);
--curSize_;
});
}
);
return true;
}

3
src/rpc/common/APIVersion.h
View File

@@ -49,8 +49,7 @@ static constexpr uint32_t API_VERSION_MAX = 2u;
/**
* @brief A baseclass for API version helper
*/
class APIVersionParser
{
class APIVersionParser {
public:
virtual ~APIVersionParser() = default;

9
src/rpc/common/AnyHandler.h
View File

@@ -32,8 +32,7 @@ namespace rpc {
* Support for copying was added in order to allow storing in a
* map/unordered_map using the initializer_list constructor.
*/
class AnyHandler final
{
class AnyHandler final {
public:
/**
* @brief Type-erases any handler class.
@@ -79,8 +78,7 @@ public:
}
private:
struct Concept
{
struct Concept {
virtual ~Concept() = default;
[[nodiscard]] virtual ReturnType
@@ -91,8 +89,7 @@ private:
};
template <typename HandlerType, typename ProcessorType>
struct Model : Concept
{
struct Model : Concept {
HandlerType handler;
ProcessorType processor;

46
src/rpc/common/Concepts.h
View File

@@ -33,22 +33,22 @@ struct RpcSpec;
/**
* @brief Specifies what a requirement used with @ref rpc::FieldSpec must provide.
*/
// clang-format off
template <typename T>
concept SomeRequirement = requires(T a, boost::json::value lval) {
{ a.verify(lval, std::string{}) } -> std::same_as<MaybeError>;
{
a.verify(lval, std::string{})
} -> std::same_as<MaybeError>;
};
// clang-format on
/**
* @brief Specifies what a modifier used with @ref rpc::FieldSpec must provide.
*/
// clang-format off
template <typename T>
concept SomeModifier = requires(T a, boost::json::value lval) {
{ a.modify(lval, std::string{}) } -> std::same_as<MaybeError>;
{
a.modify(lval, std::string{})
} -> std::same_as<MaybeError>;
};
// clang-format on
/**
* @brief The requirements of a processor to be used with @ref rpc::FieldSpec.
@@ -59,54 +59,44 @@ concept SomeProcessor = (SomeRequirement<T> or SomeModifier<T>);
/**
* @brief A process function that expects both some Input and a Context.
*/
// clang-format off
template <typename T>
concept SomeContextProcessWithInput = requires(T a, typename T::Input in, typename T::Output out, Context const& ctx) {
{ a.process(in, ctx) } -> std::same_as<HandlerReturnType<decltype(out)>>;
{
a.process(in, ctx)
} -> std::same_as<HandlerReturnType<decltype(out)>>;
};
// clang-format on
/**
* @brief A process function that expects no Input but does take a Context.
*/
// clang-format off
template <typename T>
concept SomeContextProcessWithoutInput = requires(T a, typename T::Output out, Context const& ctx) {
{ a.process(ctx) } -> std::same_as<HandlerReturnType<decltype(out)>>;
{
a.process(ctx)
} -> std::same_as<HandlerReturnType<decltype(out)>>;
};
// clang-format on
/**
* @brief Specifies what a Handler with Input must provide.
*/
// clang-format off
template <typename T>
concept SomeHandlerWithInput = requires(T a, uint32_t version) {
{ a.spec(version) } -> std::same_as<RpcSpecConstRef>;
}
and SomeContextProcessWithInput<T>
and boost::json::has_value_to<typename T::Input>::value;
// clang-format on
{
a.spec(version)
} -> std::same_as<RpcSpecConstRef>;
} and SomeContextProcessWithInput<T> and boost::json::has_value_to<typename T::Input>::value;
/**
* @brief Specifies what a Handler without Input must provide.
*/
// clang-format off
template <typename T>
concept SomeHandlerWithoutInput = SomeContextProcessWithoutInput<T>;
// clang-format on
/**
* @brief Specifies what a Handler type must provide.
*/
// clang-format off
template <typename T>
concept SomeHandler =
(
SomeHandlerWithInput<T> or
SomeHandlerWithoutInput<T>
)
and boost::json::has_value_from<typename T::Output>::value;
// clang-format on
concept SomeHandler =
(SomeHandlerWithInput<T> or SomeHandlerWithoutInput<T>)and boost::json::has_value_from<typename T::Output>::value;
} // namespace rpc

11
src/rpc/common/JsonBool.h
View File

@@ -1,13 +1,13 @@
#pragma once
#include <boost/json/value_to.hpp>
namespace rpc {
/**
* @brief A wrapper around bool that allows to convert from any JSON value
*/
struct JsonBool
{
struct JsonBool {
bool value = false;
operator bool() const
@@ -19,8 +19,7 @@ struct JsonBool
inline JsonBool
tag_invoke(boost::json::value_to_tag<JsonBool> const&, boost::json::value const& jsonValue)
{
switch (jsonValue.kind())
{
switch (jsonValue.kind()) {
case boost::json::kind::null:
return JsonBool{false};
case boost::json::kind::bool_:
@@ -32,8 +31,8 @@ tag_invoke(boost::json::value_to_tag<JsonBool> const&, boost::json::value const&
case boost::json::kind::double_:
return JsonBool{jsonValue.as_double() != 0.0};
case boost::json::kind::string:
// Also should be `jsonValue.as_string() != "false"` but rippled doesn't do that. Anyway for v2 api we have
// bool validation
// Also should be `jsonValue.as_string() != "false"` but rippled doesn't do
// that. Anyway for v2 api we have bool validation
return JsonBool{!jsonValue.as_string().empty() && jsonValue.as_string()[0] != 0};
case boost::json::kind::array:
return JsonBool{!jsonValue.as_array().empty()};

6
src/rpc/common/MetaProcessors.cpp
View File

@@ -38,8 +38,7 @@ Section::verify(boost::json::value& value, std::string_view key) const
if (!res.is_object())
return {};
for (auto const& spec : specs)
{
for (auto const& spec : specs) {
if (auto const ret = spec.process(res); not ret)
return Error{ret.error()};
}
@@ -61,8 +60,7 @@ ValidateArrayAt::verify(boost::json::value& value, std::string_view key) const
return Error{Status{RippledError::rpcINVALID_PARAMS}};
auto& res = arr.at(idx_);
for (auto const& spec : specs_)
{
for (auto const& spec : specs_) {
if (auto const ret = spec.process(res); not ret)
return Error{ret.error()};
}

36
src/rpc/common/MetaProcessors.h
View File

@@ -33,8 +33,7 @@ namespace rpc::meta {
/**
* @brief A meta-processor that acts as a spec for a sub-object/section.
*/
class Section final
{
class Section final {
std::vector<FieldSpec> specs;
public:
@@ -61,8 +60,7 @@ public:
/**
* @brief A meta-processor that specifies a list of specs to run against the object at the given index in the array.
*/
class ValidateArrayAt final
{
class ValidateArrayAt final {
std::size_t idx_;
std::vector<FieldSpec> specs_;
@@ -93,8 +91,7 @@ public:
* parameter.
*/
template <typename Type>
class IfType final
{
class IfType final {
public:
/**
* @brief Constructs a validator that validates the specs if the type matches.
@@ -103,26 +100,28 @@ public:
template <SomeRequirement... Requirements>
IfType(Requirements&&... requirements)
: processor_(
[... r = std::forward<Requirements>(
requirements)](boost::json::value& j, std::string_view key) -> MaybeError {
[... r = std::forward<Requirements>(requirements
)](boost::json::value& j, std::string_view key) -> MaybeError {
std::optional<Status> firstFailure = std::nullopt;
// the check logic is the same as fieldspec
// clang-format off
([&j, &key, &firstFailure, req = &r]() {
if (firstFailure)
return;
(
[&j, &key, &firstFailure, req = &r]() {
if (firstFailure)
return;
if (auto const res = req->verify(j, key); not res)
firstFailure = res.error();
}(), ...);
// clang-format on
if (auto const res = req->verify(j, key); not res)
firstFailure = res.error();
}(),
...
);
if (firstFailure)
return Error{firstFailure.value()};
return {};
})
}
)
{
}
@@ -153,8 +152,7 @@ private:
* @brief A meta-processor that wraps a validator and produces a custom error in case the wrapped validator fails.
*/
template <typename SomeRequirement>
class WithCustomError final
{
class WithCustomError final {
SomeRequirement requirement;
Status error;

6
src/rpc/common/Modifiers.h
View File

@@ -32,8 +32,7 @@ namespace rpc::modifiers {
* @brief Clamp value between min and max.
*/
template <typename Type>
class Clamp final
{
class Clamp final {
Type min_;
Type max_;
@@ -76,8 +75,7 @@ public:
*
* Note: the conversion is only performed if the input value is a string.
*/
struct ToLower final
{
struct ToLower final {
/**
* @brief Update the input string to lower case.
*

3
src/rpc/common/Specs.cpp
View File

@@ -32,8 +32,7 @@ FieldSpec::process(boost::json::value& value) const
[[nodiscard]] MaybeError
RpcSpec::process(boost::json::value& value) const
{
for (auto const& field : fields_)
{
for (auto const& field : fields_) {
if (auto ret = field.process(value); not ret)
return Error{ret.error()};
}

8
src/rpc/common/Specs.h
View File

@@ -31,8 +31,7 @@ namespace rpc {
/**
* @brief Represents a Specification for one field of an RPC command.
*/
struct FieldSpec final
{
struct FieldSpec final {
/**
* @brief Construct a field specification out of a set of processors.
*
@@ -65,8 +64,7 @@ private:
* Note: this should really be all constexpr and handlers would expose
* static constexpr RpcSpec spec instead. Maybe some day in the future.
*/
struct RpcSpec final
{
struct RpcSpec final {
/**
* @brief Construct a full RPC request specification.
*
@@ -82,7 +80,7 @@ struct RpcSpec final
* @param other The other spec to copy fields from
* @param additionalFields The additional fields to add to the spec
*/
RpcSpec(const RpcSpec& other, std::initializer_list<FieldSpec> additionalFields) : fields_{other.fields_}
RpcSpec(RpcSpec const& other, std::initializer_list<FieldSpec> additionalFields) : fields_{other.fields_}
{
for (auto& f : additionalFields)
fields_.push_back(f);

13
src/rpc/common/Types.h
View File

@@ -75,15 +75,12 @@ using RpcSpecConstRef = RpcSpec const&;
/**
* @brief An empty type used as Output for handlers than don't actually produce output.
*/
struct VoidOutput
{
};
struct VoidOutput {};
/**
* @brief Context of an RPC call.
*/
struct Context
{
struct Context {
boost::asio::yield_context yield;
std::shared_ptr<web::ConnectionBase> session = {};
bool isAdmin = false;
@@ -99,8 +96,7 @@ using Result = std::variant<Status, boost::json::object>;
/**
* @brief A cursor object used to traverse nodes owned by an account.
*/
struct AccountCursor
{
struct AccountCursor {
ripple::uint256 index;
std::uint32_t hint{};
@@ -120,8 +116,7 @@ struct AccountCursor
/**
* @brief Interface for the provider of RPC handlers.
*/
class HandlerProvider
{
class HandlerProvider {
public:
virtual ~HandlerProvider() = default;

12
src/rpc/common/Validators.cpp
View File

@@ -112,8 +112,7 @@ CustomValidator AccountMarkerValidator =
// TODO: we are using parseAccountCursor from RPCHelpers, after we
// remove all old handler, this function can be moved to here
if (!parseAccountCursor(value.as_string().c_str()))
{
if (!parseAccountCursor(value.as_string().c_str())) {
// align with the current error message
return Error{Status{RippledError::rpcINVALID_PARAMS, "Malformed cursor."}};
}
@@ -144,8 +143,7 @@ CustomValidator IssuerValidator =
if (!ripple::to_issuer(issuer, value.as_string().c_str()))
return Error{Status{RippledError::rpcINVALID_PARAMS, fmt::format("Invalid field '{}', bad issuer.", key)}};
if (issuer == ripple::noAccount())
{
if (issuer == ripple::noAccount()) {
return Error{Status{
RippledError::rpcINVALID_PARAMS, fmt::format("Invalid field '{}', bad issuer account one.", key)}};
}
@@ -163,8 +161,7 @@ CustomValidator SubscribeStreamValidator =
static std::unordered_set<std::string> const reportingNotSupportStreams = {
"peer_status", "consensus", "server"};
for (auto const& v : value.as_array())
{
for (auto const& v : value.as_array()) {
if (!v.is_string())
return Error{Status{RippledError::rpcINVALID_PARAMS, "streamNotString"}};
@@ -186,8 +183,7 @@ CustomValidator SubscribeAccountsValidator =
if (value.as_array().empty())
return Error{Status{RippledError::rpcACT_MALFORMED, std::string(key) + " malformed."}};
for (auto const& v : value.as_array())
{
for (auto const& v : value.as_array()) {
auto obj = boost::json::object();
auto const keyItem = std::string(key) + "'sItem";

66
src/rpc/common/Validators.h
View File

@@ -40,38 +40,26 @@ template <typename Expected>
[[nodiscard]] bool static checkType(boost::json::value const& value)
{
auto hasError = false;
if constexpr (std::is_same_v<Expected, bool>)
{
if constexpr (std::is_same_v<Expected, bool>) {
if (not value.is_bool())
hasError = true;
}
else if constexpr (std::is_same_v<Expected, std::string>)
{
} else if constexpr (std::is_same_v<Expected, std::string>) {
if (not value.is_string())
hasError = true;
}
else if constexpr (std::is_same_v<Expected, double> or std::is_same_v<Expected, float>)
{
} else if constexpr (std::is_same_v<Expected, double> or std::is_same_v<Expected, float>) {
if (not value.is_double())
hasError = true;
}
else if constexpr (std::is_same_v<Expected, boost::json::array>)
{
} else if constexpr (std::is_same_v<Expected, boost::json::array>) {
if (not value.is_array())
hasError = true;
}
else if constexpr (std::is_same_v<Expected, boost::json::object>)
{
} else if constexpr (std::is_same_v<Expected, boost::json::object>) {
if (not value.is_object())
hasError = true;
}
else if constexpr (std::is_convertible_v<Expected, uint64_t> or std::is_convertible_v<Expected, int64_t>)
{
} else if constexpr (std::is_convertible_v<Expected, uint64_t> or std::is_convertible_v<Expected, int64_t>) {
if (not value.is_int64() && not value.is_uint64())
hasError = true;
// specify the type is unsigened, it can not be negative
if constexpr (std::is_unsigned_v<Expected>)
{
if constexpr (std::is_unsigned_v<Expected>) {
if (value.is_int64() and value.as_int64() < 0)
hasError = true;
}
@@ -83,8 +71,7 @@ template <typename Expected>
/**
* @brief A validator that simply requires a field to be present.
*/
struct Required final
{
struct Required final {
[[nodiscard]] static MaybeError
verify(boost::json::value const& value, std::string_view key);
};
@@ -102,8 +89,7 @@ class NotSupported;
* @brief A specialized NotSupported validator that forbids a field to be present when the value equals the given value.
*/
template <typename T>
class NotSupported<T> final
{
class NotSupported<T> final {
T value_;
public:
@@ -126,12 +112,10 @@ public:
[[nodiscard]] MaybeError
verify(boost::json::value const& value, std::string_view key) const
{
if (value.is_object() and value.as_object().contains(key.data()))
{
if (value.is_object() and value.as_object().contains(key.data())) {
using boost::json::value_to;
auto const res = value_to<T>(value.as_object().at(key.data()));
if (value_ == res)
{
if (value_ == res) {
return Error{Status{
RippledError::rpcNOT_SUPPORTED,
fmt::format("Not supported field '{}'s value '{}'", std::string{key}, res)}};
@@ -145,8 +129,7 @@ public:
* @brief A specialized NotSupported validator that forbids a field to be present.
*/
template <>
class NotSupported<> final
{
class NotSupported<> final {
public:
/**
* @brief Verify whether the field is supported or not.
@@ -175,8 +158,7 @@ NotSupported(T&&... t) -> NotSupported<T...>;
* @brief Validates that the type of the value is one of the given types.
*/
template <typename... Types>
struct Type final
{
struct Type final {
/**
* @brief Verify that the JSON value is (one) of specified type(s).
*
@@ -204,8 +186,7 @@ struct Type final
* @brief Validate that value is between specified min and max.
*/
template <typename Type>
class Between final
{
class Between final {
Type min_;
Type max_;
@@ -250,8 +231,7 @@ public:
* @brief Validate that value is equal or greater than the specified min.
*/
template <typename Type>
class Min final
{
class Min final {
Type min_;
public:
@@ -292,8 +272,7 @@ public:
* @brief Validate that value is not greater than max.
*/
template <typename Type>
class Max final
{
class Max final {
Type max_;
public:
@@ -334,8 +313,7 @@ public:
* @brief Validates that the value is equal to the one passed in.
*/
template <typename Type>
class EqualTo final
{
class EqualTo final {
Type original_;
public:
@@ -374,14 +352,13 @@ public:
/**
* @brief Deduction guide to help disambiguate what it means to EqualTo a "string" without specifying the type.
*/
EqualTo(char const*)->EqualTo<std::string>;
EqualTo(char const*) -> EqualTo<std::string>;
/**
* @brief Validates that the value is one of the values passed in.
*/
template <typename Type>
class OneOf final
{
class OneOf final {
std::vector<Type> options_;
public:
@@ -430,13 +407,12 @@ public:
/**
* @brief Deduction guide to help disambiguate what it means to OneOf a few "strings" without specifying the type.
*/
OneOf(std::initializer_list<char const*>)->OneOf<std::string>;
OneOf(std::initializer_list<char const*>) -> OneOf<std::string>;
/**
* @brief A meta-validator that allows to specify a custom validation function.
*/
class CustomValidator final
{
class CustomValidator final {
std::function<MaybeError(boost::json::value const&, std::string_view)> validator_;
public:

6
src/rpc/common/impl/APIVersionParser.cpp
View File

@@ -30,7 +30,8 @@ ProductionAPIVersionParser::ProductionAPIVersionParser(util::Config const& confi
: ProductionAPIVersionParser(
config.valueOr("default", API_VERSION_DEFAULT),
config.valueOr("min", API_VERSION_MIN),
config.valueOr("max", API_VERSION_MAX))
config.valueOr("max", API_VERSION_MAX)
)
{
}
@@ -39,8 +40,7 @@ ProductionAPIVersionParser::parse(boost::json::object const& request) const
{
using Error = util::Unexpected<std::string>;
if (request.contains("api_version"))
{
if (request.contains("api_version")) {
if (!request.at("api_version").is_int64())
return Error{"API version must be an integer"};

9
src/rpc/common/impl/APIVersionParser.h
View File

@@ -29,8 +29,7 @@
namespace rpc::detail {
class ProductionAPIVersionParser : public APIVersionParser
{
class ProductionAPIVersionParser : public APIVersionParser {
util::Logger log_{"RPC"};
uint32_t defaultVersion_;
@@ -42,15 +41,15 @@ public:
ProductionAPIVersionParser(
uint32_t defaultVersion = API_VERSION_DEFAULT,
uint32_t minVersion = API_VERSION_MIN,
uint32_t maxVersion = API_VERSION_MAX)
uint32_t maxVersion = API_VERSION_MAX
)
: defaultVersion_{defaultVersion}, minVersion_{minVersion}, maxVersion_{maxVersion}
{
#ifndef UNITTEST_BUILD
// in production, we don't want the ability to misconfigure clio with bogus versions
// that are not actually supported by the code itself. for testing it is desired however.
auto checkRange = [this](uint32_t version, std::string label) {
if (std::clamp(version, API_VERSION_MIN, API_VERSION_MAX) != version)
{
if (std::clamp(version, API_VERSION_MIN, API_VERSION_MAX) != version) {
LOG(log_.error()) << "API version settings issue detected: " << label << " version with value "
<< version << " is outside of supported range " << API_VERSION_MIN << "-"
<< API_VERSION_MAX << "; Falling back to hardcoded values.";

32
src/rpc/common/impl/Factories.h
View File

@@ -40,23 +40,23 @@ makeFieldProcessor(std::string const& key, Processors&&... procs)
// This expands in order of Requirements and stops evaluating after first failure which is stored in
// `firstFailure` and can be checked later on to see whether the verification failed as a whole or not.
// clang-format off
([&j, &key, &firstFailure, req = &proc]() {
if (firstFailure)
return; // already failed earlier - skip
(
[&j, &key, &firstFailure, req = &proc]() {
if (firstFailure)
return; // already failed earlier - skip
if constexpr (SomeRequirement<decltype(*req)>) {
if (auto const res = req->verify(j, key); not res)
firstFailure = res.error();
} else if constexpr (SomeModifier<decltype(*req)>) {
if (auto const res = req->modify(j, key); not res)
firstFailure = res.error();
} else {
static_assert(unsupported_v<decltype(*req)>);
}
}(), ...);
// clang-format on
if constexpr (SomeRequirement<decltype(*req)>) {
if (auto const res = req->verify(j, key); not res)
firstFailure = res.error();
} else if constexpr (SomeModifier<decltype(*req)>) {
if (auto const res = req->modify(j, key); not res)
firstFailure = res.error();
} else {
static_assert(unsupported_v<decltype(*req)>);
}
}(),
...
);
if (firstFailure)
return Error{firstFailure.value()};

13
src/rpc/common/impl/ForwardingProxy.h
View File

@@ -33,8 +33,7 @@
namespace rpc::detail {
template <typename LoadBalancerType, typename CountersType, typename HandlerProviderType>
class ForwardingProxy
{
class ForwardingProxy {
util::Logger log_{"RPC"};
std::shared_ptr<LoadBalancerType> balancer_;
@@ -45,7 +44,8 @@ public:
ForwardingProxy(
std::shared_ptr<LoadBalancerType> const& balancer,
CountersType& counters,
std::shared_ptr<HandlerProviderType const> const& handlerProvider)
std::shared_ptr<HandlerProviderType const> const& handlerProvider
)
: balancer_{balancer}, counters_{std::ref(counters)}, handlerProvider_{handlerProvider}
{
}
@@ -76,8 +76,8 @@ public:
return ctx.method == "ledger" and
((request.contains("queue") and request.at("queue").is_bool() and request.at("queue").as_bool()) or
(request.contains("full") and request.at("full").is_bool() and request.at("full").as_bool()) or
(request.contains("accounts") and request.at("accounts").is_bool() and
request.at("accounts").as_bool()));
(request.contains("accounts") and request.at("accounts").is_bool() and request.at("accounts").as_bool()
));
};
return static_cast<bool>(checkAccountInfoForward() or checkLedgerForward());
@@ -90,8 +90,7 @@ public:
toForward["command"] = ctx.method;
auto const res = balancer_->forwardToRippled(toForward, ctx.clientIp, ctx.yield);
if (not res)
{
if (not res) {
notifyFailedToForward(ctx.method);
return Status{RippledError::rpcFAILED_TO_FORWARD};
}

3
src/rpc/common/impl/HandlerProvider.cpp
View File

@@ -61,7 +61,8 @@ ProductionHandlerProvider::ProductionHandlerProvider(
std::shared_ptr<feed::SubscriptionManager> const& subscriptionManager,
std::shared_ptr<etl::LoadBalancer> const& balancer,
std::shared_ptr<etl::ETLService const> const& etl,
Counters const& counters)
Counters const& counters
)
: handlerMap_{
{"account_channels", {AccountChannelsHandler{backend}}},
{"account_currencies", {AccountCurrenciesHandler{backend}}},

9
src/rpc/common/impl/HandlerProvider.h
View File

@@ -41,10 +41,8 @@ class SubscriptionManager;
namespace rpc::detail {
class ProductionHandlerProvider final : public HandlerProvider
{
struct Handler
{
class ProductionHandlerProvider final : public HandlerProvider {
struct Handler {
AnyHandler handler;
bool isClioOnly = false;
};
@@ -58,7 +56,8 @@ public:
std::shared_ptr<feed::SubscriptionManager> const& subscriptionManager,
std::shared_ptr<etl::LoadBalancer> const& balancer,
std::shared_ptr<etl::ETLService const> const& etl,
Counters const& counters);
Counters const& counters
);
bool
contains(std::string const& method) const override;

20
src/rpc/common/impl/Processors.h
View File

@@ -29,15 +29,13 @@ template <typename>
static constexpr bool unsupported_handler_v = false;
template <SomeHandler HandlerType>
struct DefaultProcessor final
{
struct DefaultProcessor final {
[[nodiscard]] ReturnType
operator()(HandlerType const& handler, boost::json::value const& value, Context const& ctx) const
{
using boost::json::value_from;
using boost::json::value_to;
if constexpr (SomeHandlerWithInput<HandlerType>)
{
if constexpr (SomeHandlerWithInput<HandlerType>) {
// first we run validation against specified API version
auto const spec = handler.spec(ctx.apiVersion);
auto input = value; // copy here, spec require mutable data
@@ -49,24 +47,18 @@ struct DefaultProcessor final
auto const ret = handler.process(inData, ctx);
// real handler is given expected Input, not json
if (!ret)
{
if (!ret) {
return Error{ret.error()}; // forward Status
}
return value_from(ret.value());
}
else if constexpr (SomeHandlerWithoutInput<HandlerType>)
{
} else if constexpr (SomeHandlerWithoutInput<HandlerType>) {
// no input to pass, ignore the value
auto const ret = handler.process(ctx);
if (not ret)
{
if (not ret) {
return Error{ret.error()}; // forward Status
}
return value_from(ret.value());
}
else
{
} else {
// when concept SomeHandlerWithInput and SomeHandlerWithoutInput not cover all Handler case
static_assert(unsupported_handler_v<HandlerType>);
}

22
src/rpc/handlers/AccountChannels.cpp
View File

@@ -33,8 +33,7 @@ AccountChannelsHandler::addChannel(std::vector<ChannelResponse>& jsonChannels, r
channel.balance = channelSle[ripple::sfBalance].getText();
channel.settleDelay = channelSle[ripple::sfSettleDelay];
if (publicKeyType(channelSle[ripple::sfPublicKey]))
{
if (publicKeyType(channelSle[ripple::sfPublicKey])) {
ripple::PublicKey const pk(channelSle[ripple::sfPublicKey]);
channel.publicKey = toBase58(ripple::TokenType::AccountPublic, pk);
channel.publicKeyHex = strHex(pk);
@@ -60,7 +59,8 @@ AccountChannelsHandler::process(AccountChannelsHandler::Input input, Context con
{
auto const range = sharedPtrBackend_->fetchLedgerRange();
auto const lgrInfoOrStatus = getLedgerInfoFromHashOrSeq(
*sharedPtrBackend_, ctx.yield, input.ledgerHash, input.ledgerIndex, range->maxSequence);
*sharedPtrBackend_, ctx.yield, input.ledgerHash, input.ledgerIndex, range->maxSequence
);
if (auto status = std::get_if<Status>(&lgrInfoOrStatus))
return Error{*status};
@@ -79,8 +79,7 @@ AccountChannelsHandler::process(AccountChannelsHandler::Input input, Context con
Output response;
auto const addToResponse = [&](ripple::SLE&& sle) {
if (sle.getType() == ripple::ltPAYCHAN && sle.getAccountID(ripple::sfAccount) == accountID &&
(!destAccountID || *destAccountID == sle.getAccountID(ripple::sfDestination)))
{
(!destAccountID || *destAccountID == sle.getAccountID(ripple::sfDestination))) {
addChannel(response.channels, sle);
}
@@ -88,7 +87,8 @@ AccountChannelsHandler::process(AccountChannelsHandler::Input input, Context con
};
auto const next = traverseOwnedNodes(
*sharedPtrBackend_, *accountID, lgrInfo.seq, input.limit, input.marker, ctx.yield, addToResponse);
*sharedPtrBackend_, *accountID, lgrInfo.seq, input.limit, input.marker, ctx.yield, addToResponse
);
if (auto status = std::get_if<Status>(&next))
return Error{*status};
@@ -125,14 +125,10 @@ tag_invoke(boost::json::value_to_tag<AccountChannelsHandler::Input>, boost::json
if (jsonObject.contains(JS(destination_account)))
input.destinationAccount = jv.at(JS(destination_account)).as_string().c_str();
if (jsonObject.contains(JS(ledger_index)))
{
if (!jsonObject.at(JS(ledger_index)).is_string())
{
if (jsonObject.contains(JS(ledger_index))) {
if (!jsonObject.at(JS(ledger_index)).is_string()) {
input.ledgerIndex = jv.at(JS(ledger_index)).as_int64();
}
else if (jsonObject.at(JS(ledger_index)).as_string() != "validated")
{
} else if (jsonObject.at(JS(ledger_index)).as_string() != "validated") {
input.ledgerIndex = std::stoi(jv.at(JS(ledger_index)).as_string().c_str());
}
}

12
src/rpc/handlers/AccountChannels.h
View File

@@ -36,8 +36,7 @@ namespace rpc {
*
* For more details see: https://xrpl.org/account_channels.html
*/
class AccountChannelsHandler
{
class AccountChannelsHandler {
// dependencies
std::shared_ptr<BackendInterface> const sharedPtrBackend_;
@@ -47,8 +46,7 @@ public:
static constexpr auto LIMIT_DEFAULT = 200;
// type align with SField.h
struct ChannelResponse
{
struct ChannelResponse {
std::string channelID;
std::string account;
std::string accountDestination;
@@ -63,8 +61,7 @@ public:
std::optional<uint32_t> destinationTag;
};
struct Output
{
struct Output {
std::vector<ChannelResponse> channels;
std::string account;
std::string ledgerHash;
@@ -75,8 +72,7 @@ public:
std::optional<std::string> marker;
};
struct Input
{
struct Input {
std::string account;
std::optional<std::string> destinationAccount;
std::optional<std::string> ledgerHash;

19
src/rpc/handlers/AccountCurrencies.cpp
View File

@@ -25,7 +25,8 @@ AccountCurrenciesHandler::process(AccountCurrenciesHandler::Input input, Context
{
auto const range = sharedPtrBackend_->fetchLedgerRange();
auto const lgrInfoOrStatus = getLedgerInfoFromHashOrSeq(
*sharedPtrBackend_, ctx.yield, input.ledgerHash, input.ledgerIndex, range->maxSequence);
*sharedPtrBackend_, ctx.yield, input.ledgerHash, input.ledgerIndex, range->maxSequence
);
if (auto const status = std::get_if<Status>(&lgrInfoOrStatus))
return Error{*status};
@@ -40,8 +41,7 @@ AccountCurrenciesHandler::process(AccountCurrenciesHandler::Input input, Context
Output response;
auto const addToResponse = [&](ripple::SLE&& sle) {
if (sle.getType() == ripple::ltRIPPLE_STATE)
{
if (sle.getType() == ripple::ltRIPPLE_STATE) {
auto balance = sle.getFieldAmount(ripple::sfBalance);
auto const lowLimit = sle.getFieldAmount(ripple::sfLowLimit);
auto const highLimit = sle.getFieldAmount(ripple::sfHighLimit);
@@ -70,7 +70,8 @@ AccountCurrenciesHandler::process(AccountCurrenciesHandler::Input input, Context
std::numeric_limits<std::uint32_t>::max(),
{},
ctx.yield,
addToResponse);
addToResponse
);
response.ledgerHash = ripple::strHex(lgrInfo.hash);
response.ledgerIndex = lgrInfo.seq;
@@ -103,14 +104,10 @@ tag_invoke(boost::json::value_to_tag<AccountCurrenciesHandler::Input>, boost::js
if (jsonObject.contains(JS(ledger_hash)))
input.ledgerHash = jv.at(JS(ledger_hash)).as_string().c_str();
if (jsonObject.contains(JS(ledger_index)))
{
if (!jsonObject.at(JS(ledger_index)).is_string())
{
if (jsonObject.contains(JS(ledger_index))) {
if (!jsonObject.at(JS(ledger_index)).is_string()) {
input.ledgerIndex = jv.at(JS(ledger_index)).as_int64();
}
else if (jsonObject.at(JS(ledger_index)).as_string() != "validated")
{
} else if (jsonObject.at(JS(ledger_index)).as_string() != "validated") {
input.ledgerIndex = std::stoi(jv.at(JS(ledger_index)).as_string().c_str());
}
}

9
src/rpc/handlers/AccountCurrencies.h
View File

@@ -35,14 +35,12 @@ namespace rpc {
*
* For more details see: https://xrpl.org/account_currencies.html
*/
class AccountCurrenciesHandler
{
class AccountCurrenciesHandler {
// dependencies
std::shared_ptr<BackendInterface> sharedPtrBackend_;
public:
struct Output
{
struct Output {
std::string ledgerHash;
uint32_t ledgerIndex{};
std::set<std::string> receiveCurrencies;
@@ -51,8 +49,7 @@ public:
bool validated = true;
};
struct Input
{
struct Input {
std::string account;
std::optional<std::string> ledgerHash;
std::optional<uint32_t> ledgerIndex;

Some files were not shown because too many files have changed in this diff Show More