ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2026-04-29 15:37:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reformat codebase with 120 char limit (#583)

Browse Source
This commit is contained in:
Alex Kremer
2023-04-06 11:24:36 +01:00
committed by GitHub
parent e60fd3e58e
commit d816ef54ab
174 changed files with 5591 additions and 10450 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.clang-format
View File

@@ -34,7 +34,7 @@ BreakBeforeBinaryOperators: false
BreakBeforeBraces: Custom
BreakBeforeTernaryOperators: true
BreakConstructorInitializersBeforeComma: true
ColumnLimit: 80
ColumnLimit: 120
CommentPragmas: '^ IWYU pragma:'
ConstructorInitializerAllOnOneLineOrOnePerLine: true
ConstructorInitializerIndentWidth: 4

1
.git-blame-ignore-revs
View File

@@ -7,3 +7,4 @@
# clang-format
e41150248a97e4bdc1cf21b54650c4bb7c63928e
2e542e7b0d94451a933c88778461cc8d3d7e6417
cde682b42d1b3f798abb3d5a0b729e48a9cbeb27

4
src/backend/BackendFactory.h
View File

@@ -48,8 +48,8 @@ make_Backend(boost::asio::io_context& ioc, clio::Config const& config)
{
auto cfg = config.section("database." + type);
auto ttl = config.valueOr<uint16_t>("online_delete", 0) * 4;
backend = std::make_shared<Backend::Cassandra::CassandraBackend>(
Backend::Cassandra::SettingsProvider{cfg, ttl});
backend =
std::make_shared<Backend::Cassandra::CassandraBackend>(Backend::Cassandra::SettingsProvider{cfg, ttl});
}
if (!backend)

66
src/backend/BackendInterface.cpp
View File

@@ -37,25 +37,20 @@ BackendInterface::finishWrites(std::uint32_t const ledgerSequence)
auto commitRes = doFinishWrites();
if (commitRes)
{
gLog.debug() << "Successfully commited. Updating range now to "
<< ledgerSequence;
gLog.debug() << "Successfully commited. Updating range now to " << ledgerSequence;
updateRange(ledgerSequence);
}
return commitRes;
}
void
BackendInterface::writeLedgerObject(
std::string&& key,
std::uint32_t const seq,
std::string&& blob)
BackendInterface::writeLedgerObject(std::string&& key, std::uint32_t const seq, std::string&& blob)
{
assert(key.size() == sizeof(ripple::uint256));
doWriteLedgerObject(std::move(key), seq, std::move(blob));
}
std::optional<LedgerRange>
BackendInterface::hardFetchLedgerRangeNoThrow(
boost::asio::yield_context& yield) const
BackendInterface::hardFetchLedgerRangeNoThrow(boost::asio::yield_context& yield) const
{
gLog.trace() << "called";
while (true)
@@ -120,8 +115,7 @@ BackendInterface::fetchLedgerObjects(
else
misses.push_back(keys[i]);
}
gLog.trace() << "Cache hits = " << keys.size() - misses.size()
<< " - cache misses = " << misses.size();
gLog.trace() << "Cache hits = " << keys.size() - misses.size() << " - cache misses = " << misses.size();
if (misses.size())
{
@@ -184,10 +178,7 @@ BackendInterface::fetchBookOffers(
const ripple::uint256 bookEnd = ripple::getQualityNext(book);
ripple::uint256 uTipIndex = book;
std::vector<ripple::uint256> keys;
auto getMillis = [](auto diff) {
return std::chrono::duration_cast<std::chrono::milliseconds>(diff)
.count();
};
auto getMillis = [](auto diff) { return std::chrono::duration_cast<std::chrono::milliseconds>(diff).count(); };
auto begin = std::chrono::system_clock::now();
std::uint32_t numSucc = 0;
std::uint32_t numPages = 0;
@@ -202,18 +193,14 @@ BackendInterface::fetchBookOffers(
succMillis += getMillis(mid2 - mid1);
if (!offerDir || offerDir->key >= bookEnd)
{
gLog.trace() << "offerDir.has_value() " << offerDir.has_value()
<< " breaking";
gLog.trace() << "offerDir.has_value() " << offerDir.has_value() << " breaking";
break;
}
uTipIndex = offerDir->key;
while (keys.size() < limit)
{
++numPages;
ripple::STLedgerEntry sle{
ripple::SerialIter{
offerDir->blob.data(), offerDir->blob.size()},
offerDir->key};
ripple::STLedgerEntry sle{ripple::SerialIter{offerDir->blob.data(), offerDir->blob.size()}, offerDir->key};
auto indexes = sle.getFieldV256(ripple::sfIndexes);
keys.insert(keys.end(), indexes.begin(), indexes.end());
auto next = sle.getFieldU64(ripple::sfIndexNext);
@@ -223,8 +210,7 @@ BackendInterface::fetchBookOffers(
break;
}
auto nextKey = ripple::keylet::page(uTipIndex, next);
auto nextDir =
fetchLedgerObject(nextKey.key, ledgerSequence, yield);
auto nextDir = fetchLedgerObject(nextKey.key, ledgerSequence, yield);
assert(nextDir);
offerDir->blob = *nextDir;
offerDir->key = nextKey.key;
@@ -236,26 +222,20 @@ BackendInterface::fetchBookOffers(
auto objs = fetchLedgerObjects(keys, ledgerSequence, yield);
for (size_t i = 0; i < keys.size() && i < limit; ++i)
{
gLog.trace() << "Key = " << ripple::strHex(keys[i])
<< " blob = " << ripple::strHex(objs[i])
gLog.trace() << "Key = " << ripple::strHex(keys[i]) << " blob = " << ripple::strHex(objs[i])
<< " ledgerSequence = " << ledgerSequence;
assert(objs[i].size());
page.offers.push_back({keys[i], objs[i]});
}
auto end = std::chrono::system_clock::now();
gLog.debug() << "Fetching " << std::to_string(keys.size())
<< " offers took " << std::to_string(getMillis(mid - begin))
<< " milliseconds. Fetching next dir took "
<< std::to_string(succMillis)
<< " milliseonds. Fetched next dir " << std::to_string(numSucc)
gLog.debug() << "Fetching " << std::to_string(keys.size()) << " offers took "
<< std::to_string(getMillis(mid - begin)) << " milliseconds. Fetching next dir took "
<< std::to_string(succMillis) << " milliseonds. Fetched next dir " << std::to_string(numSucc)
<< " times"
<< " Fetching next page of dir took "
<< std::to_string(pageMillis) << " milliseconds"
<< ". num pages = " << std::to_string(numPages)
<< ". Fetching all objects took "
<< " Fetching next page of dir took " << std::to_string(pageMillis) << " milliseconds"
<< ". num pages = " << std::to_string(numPages) << ". Fetching all objects took "
<< std::to_string(getMillis(end - mid))
<< " milliseconds. total time = "
<< std::to_string(getMillis(end - begin)) << " milliseconds"
<< " milliseconds. total time = " << std::to_string(getMillis(end - begin)) << " milliseconds"
<< " book = " << ripple::strHex(book);
return page;
@@ -275,11 +255,8 @@ BackendInterface::fetchLedgerPage(
bool reachedEnd = false;
while (keys.size() < limit && !reachedEnd)
{
ripple::uint256 const& curCursor = keys.size() ? keys.back()
: cursor ? *cursor
: firstKey;
std::uint32_t const seq =
outOfOrder ? range->maxSequence : ledgerSequence;
ripple::uint256 const& curCursor = keys.size() ? keys.back() : cursor ? *cursor : firstKey;
std::uint32_t const seq = outOfOrder ? range->maxSequence : ledgerSequence;
auto succ = fetchSuccessorKey(curCursor, seq, yield);
if (!succ)
reachedEnd = true;
@@ -294,9 +271,8 @@ BackendInterface::fetchLedgerPage(
page.objects.push_back({std::move(keys[i]), std::move(objects[i])});
else if (!outOfOrder)
{
gLog.error()
<< "Deleted or non-existent object in successor table. key = "
<< ripple::strHex(keys[i]) << " - seq = " << ledgerSequence;
gLog.error() << "Deleted or non-existent object in successor table. key = " << ripple::strHex(keys[i])
<< " - seq = " << ledgerSequence;
std::stringstream msg;
for (size_t j = 0; j < objects.size(); ++j)
{
@@ -312,9 +288,7 @@ BackendInterface::fetchLedgerPage(
}
std::optional<ripple::Fees>
BackendInterface::fetchFees(
std::uint32_t const seq,
boost::asio::yield_context& yield) const
BackendInterface::fetchFees(std::uint32_t const seq, boost::asio::yield_context& yield) const
{
ripple::Fees fees;

111
src/backend/BackendInterface.h
View File

@@ -72,8 +72,7 @@ retryOnTimeout(F func, size_t waitMs = 500)
}
catch (DatabaseTimeout& t)
{
log.error()
<< "Database request timed out. Sleeping and retrying ... ";
log.error() << "Database request timed out. Sleeping and retrying ... ";
std::this_thread::sleep_for(std::chrono::milliseconds(waitMs));
}
}
@@ -122,11 +121,10 @@ synchronous(F&& f)
* executing coroutine, yield. The different type is returned.
*/
R res;
boost::asio::spawn(
strand, [&f, &work, &res](boost::asio::yield_context yield) {
res = f(yield);
work.reset();
});
boost::asio::spawn(strand, [&f, &work, &res](boost::asio::yield_context yield) {
res = f(yield);
work.reset();
});
ctx.run();
return res;
@@ -134,11 +132,10 @@ synchronous(F&& f)
else
{
/*! @brief When the corutine type is different, run as normal. */
boost::asio::spawn(
strand, [&f, &work](boost::asio::yield_context yield) {
f(yield);
work.reset();
});
boost::asio::spawn(strand, [&f, &work](boost::asio::yield_context yield) {
f(yield);
work.reset();
});
ctx.run();
}
@@ -209,15 +206,11 @@ public:
/*! @brief Fetches a specific ledger by sequence number. */
virtual std::optional<ripple::LedgerInfo>
fetchLedgerBySequence(
std::uint32_t const sequence,
boost::asio::yield_context& yield) const = 0;
fetchLedgerBySequence(std::uint32_t const sequence, boost::asio::yield_context& yield) const = 0;
/*! @brief Fetches a specific ledger by hash. */
virtual std::optional<ripple::LedgerInfo>
fetchLedgerByHash(
ripple::uint256 const& hash,
boost::asio::yield_context& yield) const = 0;
fetchLedgerByHash(ripple::uint256 const& hash, boost::asio::yield_context& yield) const = 0;
/*! @brief Fetches the latest ledger sequence. */
virtual std::optional<std::uint32_t>
@@ -269,9 +262,7 @@ public:
* @return std::optional<TransactionAndMetadata>
*/
virtual std::optional<TransactionAndMetadata>
fetchTransaction(
ripple::uint256 const& hash,
boost::asio::yield_context& yield) const = 0;
fetchTransaction(ripple::uint256 const& hash, boost::asio::yield_context& yield) const = 0;
/**
* @brief Fetches multiple transactions.
@@ -281,9 +272,7 @@ public:
* @return std::vector<TransactionAndMetadata>
*/
virtual std::vector<TransactionAndMetadata>
fetchTransactions(
std::vector<ripple::uint256> const& hashes,
boost::asio::yield_context& yield) const = 0;
fetchTransactions(std::vector<ripple::uint256> const& hashes, boost::asio::yield_context& yield) const = 0;
/**
* @brief Fetches all transactions for a specific account
@@ -314,9 +303,7 @@ public:
* @return std::vector<TransactionAndMetadata>
*/
virtual std::vector<TransactionAndMetadata>
fetchAllTransactionsInLedger(
std::uint32_t const ledgerSequence,
boost::asio::yield_context& yield) const = 0;
fetchAllTransactionsInLedger(std::uint32_t const ledgerSequence, boost::asio::yield_context& yield) const = 0;
/**
* @brief Fetches all transaction hashes from a specific ledger.
@@ -326,9 +313,7 @@ public:
* @return std::vector<ripple::uint256>
*/
virtual std::vector<ripple::uint256>
fetchAllTransactionHashesInLedger(
std::uint32_t const ledgerSequence,
boost::asio::yield_context& yield) const = 0;
fetchAllTransactionHashesInLedger(std::uint32_t const ledgerSequence, boost::asio::yield_context& yield) const = 0;
/*! @brief NFT methods */
/**
@@ -340,10 +325,8 @@ public:
* @return std::optional<NFT>
*/
virtual std::optional<NFT>
fetchNFT(
ripple::uint256 const& tokenID,
std::uint32_t const ledgerSequence,
boost::asio::yield_context& yield) const = 0;
fetchNFT(ripple::uint256 const& tokenID, std::uint32_t const ledgerSequence, boost::asio::yield_context& yield)
const = 0;
/**
* @brief Fetches all transactions for a specific NFT.
@@ -373,10 +356,8 @@ public:
* @return std::optional<Blob>
*/
std::optional<Blob>
fetchLedgerObject(
ripple::uint256 const& key,
std::uint32_t const sequence,
boost::asio::yield_context& yield) const;
fetchLedgerObject(ripple::uint256 const& key, std::uint32_t const sequence, boost::asio::yield_context& yield)
const;
/**
* @brief Fetches all ledger objects: a vector of vectors of unsigned chars.
@@ -394,10 +375,8 @@ public:
/*! @brief Virtual function version of fetchLedgerObject */
virtual std::optional<Blob>
doFetchLedgerObject(
ripple::uint256 const& key,
std::uint32_t const sequence,
boost::asio::yield_context& yield) const = 0;
doFetchLedgerObject(ripple::uint256 const& key, std::uint32_t const sequence, boost::asio::yield_context& yield)
const = 0;
/*! @brief Virtual function version of fetchLedgerObjects */
virtual std::vector<Blob>
@@ -417,9 +396,7 @@ public:
* @return std::vector<LedgerObject>
*/
virtual std::vector<LedgerObject>
fetchLedgerDiff(
std::uint32_t const ledgerSequence,
boost::asio::yield_context& yield) const = 0;
fetchLedgerDiff(std::uint32_t const ledgerSequence, boost::asio::yield_context& yield) const = 0;
/**
* @brief Fetches a page of ledger objects, ordered by key/index.
@@ -441,24 +418,17 @@ public:
/*! @brief Fetches successor object from key/index. */
std::optional<LedgerObject>
fetchSuccessorObject(
ripple::uint256 key,
std::uint32_t const ledgerSequence,
boost::asio::yield_context& yield) const;
fetchSuccessorObject(ripple::uint256 key, std::uint32_t const ledgerSequence, boost::asio::yield_context& yield)
const;
/*! @brief Fetches successor key from key/index. */
std::optional<ripple::uint256>
fetchSuccessorKey(
ripple::uint256 key,
std::uint32_t const ledgerSequence,
boost::asio::yield_context& yield) const;
fetchSuccessorKey(ripple::uint256 key, std::uint32_t const ledgerSequence, boost::asio::yield_context& yield) const;
/*! @brief Virtual function version of fetchSuccessorKey. */
virtual std::optional<ripple::uint256>
doFetchSuccessorKey(
ripple::uint256 key,
std::uint32_t const ledgerSequence,
boost::asio::yield_context& yield) const = 0;
doFetchSuccessorKey(ripple::uint256 key, std::uint32_t const ledgerSequence, boost::asio::yield_context& yield)
const = 0;
/**
* @brief Fetches book offers.
@@ -490,9 +460,7 @@ public:
std::optional<LedgerRange>
hardFetchLedgerRange() const
{
return synchronous([&](boost::asio::yield_context yield) {
return hardFetchLedgerRange(yield);
});
return synchronous([&](boost::asio::yield_context yield) { return hardFetchLedgerRange(yield); });
}
/*! @brief Virtual function equivalent of hardFetchLedgerRange. */
@@ -513,9 +481,7 @@ public:
* @param ledgerHeader r-value string representing ledger header.
*/
virtual void
writeLedger(
ripple::LedgerInfo const& ledgerInfo,
std::string&& ledgerHeader) = 0;
writeLedger(ripple::LedgerInfo const& ledgerInfo, std::string&& ledgerHeader) = 0;
/**
* @brief Writes a new ledger object.
@@ -527,10 +493,7 @@ public:
* @param blob r-value vector of unsigned characters (blob).
*/
virtual void
writeLedgerObject(
std::string&& key,
std::uint32_t const seq,
std::string&& blob);
writeLedgerObject(std::string&& key, std::uint32_t const seq, std::string&& blob);
/**
* @brief Writes a new transaction.
@@ -581,10 +544,7 @@ public:
* @param successor Passed in as an r-value reference.
*/
virtual void
writeSuccessor(
std::string&& key,
std::uint32_t const seq,
std::string&& successor) = 0;
writeSuccessor(std::string&& key, std::uint32_t const seq, std::string&& successor) = 0;
/*! @brief Tells database we will write data for a specific ledger. */
virtual void
@@ -613,9 +573,7 @@ public:
* @return false
*/
virtual bool
doOnlineDelete(
std::uint32_t numLedgersToKeep,
boost::asio::yield_context& yield) const = 0;
doOnlineDelete(std::uint32_t numLedgersToKeep, boost::asio::yield_context& yield) const = 0;
/**
* @brief Opens the database
@@ -645,10 +603,7 @@ private:
* @param blob r-value vector of unsigned chars.
*/
virtual void
doWriteLedgerObject(
std::string&& key,
std::uint32_t const seq,
std::string&& blob) = 0;
doWriteLedgerObject(std::string&& key, std::uint32_t const seq, std::string&& blob) = 0;
virtual bool
doFinishWrites() = 0;

317
src/backend/CassandraBackend.cpp
View File

@@ -48,22 +48,15 @@ processAsyncWriteResponse(T& requestParams, CassFuture* fut, F func)
if (rc != CASS_OK)
{
// exponential backoff with a max wait of 2^10 ms (about 1 second)
auto wait = std::chrono::milliseconds(
lround(std::pow(2, std::min(10u, requestParams.currentRetries))));
log.error() << "ERROR!!! Cassandra write error: " << rc << ", "
<< cass_error_desc(rc)
<< " id= " << requestParams.toString()
<< ", current retries " << requestParams.currentRetries
auto wait = std::chrono::milliseconds(lround(std::pow(2, std::min(10u, requestParams.currentRetries))));
log.error() << "ERROR!!! Cassandra write error: " << rc << ", " << cass_error_desc(rc)
<< " id= " << requestParams.toString() << ", current retries " << requestParams.currentRetries
<< ", retrying in " << wait.count() << " milliseconds";
++requestParams.currentRetries;
std::shared_ptr<boost::asio::steady_timer> timer =
std::make_shared<boost::asio::steady_timer>(
backend.getIOContext(),
std::chrono::steady_clock::now() + wait);
timer->async_wait([timer, &requestParams, func](
const boost::system::error_code& error) {
func(requestParams, true);
});
std::shared_ptr<boost::asio::steady_timer> timer = std::make_shared<boost::asio::steady_timer>(
backend.getIOContext(), std::chrono::steady_clock::now() + wait);
timer->async_wait(
[timer, &requestParams, func](const boost::system::error_code& error) { func(requestParams, true); });
}
else
{
@@ -91,21 +84,13 @@ struct WriteCallbackData
std::atomic<int> refs = 1;
std::string id;
WriteCallbackData(
CassandraBackend const* b,
T&& d,
B bind,
std::string const& identifier)
WriteCallbackData(CassandraBackend const* b, T&& d, B bind, std::string const& identifier)
: backend(b), data(std::move(d)), id(identifier)
{
retry = [bind, this](auto& params, bool isRetry) {
auto statement = bind(params);
backend->executeAsyncWrite(
statement,
processAsyncWrite<
typename std::remove_reference<decltype(params)>::type>,
params,
isRetry);
statement, processAsyncWrite<typename std::remove_reference<decltype(params)>::type>, params, isRetry);
};
}
virtual void
@@ -146,10 +131,7 @@ struct BulkWriteCallbackData : public WriteCallbackData<T, B>
std::atomic_int& r,
std::mutex& m,
std::condition_variable& c)
: WriteCallbackData<T, B>(b, std::move(d), bind, "bulk")
, numRemaining(r)
, mtx(m)
, cv(c)
: WriteCallbackData<T, B>(b, std::move(d), bind, "bulk"), numRemaining(r), mtx(m), cv(c)
{
}
void
@@ -173,11 +155,7 @@ struct BulkWriteCallbackData : public WriteCallbackData<T, B>
template <class T, class B>
void
makeAndExecuteAsyncWrite(
CassandraBackend const* b,
T&& d,
B bind,
std::string const& id)
makeAndExecuteAsyncWrite(CassandraBackend const* b, T&& d, B bind, std::string const& id)
{
auto* cb = new WriteCallbackData<T, B>(b, std::move(d), bind, id);
cb->start();
@@ -193,17 +171,13 @@ makeAndExecuteBulkAsyncWrite(
std::mutex& m,
std::condition_variable& c)
{
auto cb = std::make_shared<BulkWriteCallbackData<T, B>>(
b, std::move(d), bind, r, m, c);
auto cb = std::make_shared<BulkWriteCallbackData<T, B>>(b, std::move(d), bind, r, m, c);
cb->start();
return cb;
}
void
CassandraBackend::doWriteLedgerObject(
std::string&& key,
std::uint32_t const seq,
std::string&& blob)
CassandraBackend::doWriteLedgerObject(std::string&& key, std::uint32_t const seq, std::string&& blob)
{
log_.trace() << "Writing ledger object to cassandra";
if (range)
@@ -235,14 +209,10 @@ CassandraBackend::doWriteLedgerObject(
}
void
CassandraBackend::writeSuccessor(
std::string&& key,
std::uint32_t const seq,
std::string&& successor)
CassandraBackend::writeSuccessor(std::string&& key, std::uint32_t const seq, std::string&& successor)
{
log_.trace() << "Writing successor. key = " << key.size() << " bytes. "
<< " seq = " << std::to_string(seq)
<< " successor = " << successor.size() << " bytes.";
<< " seq = " << std::to_string(seq) << " successor = " << successor.size() << " bytes.";
assert(key.size() != 0);
assert(successor.size() != 0);
makeAndExecuteAsyncWrite(
@@ -260,9 +230,7 @@ CassandraBackend::writeSuccessor(
"successor");
}
void
CassandraBackend::writeLedger(
ripple::LedgerInfo const& ledgerInfo,
std::string&& header)
CassandraBackend::writeLedger(ripple::LedgerInfo const& ledgerInfo, std::string&& header)
{
makeAndExecuteAsyncWrite(
this,
@@ -290,8 +258,7 @@ CassandraBackend::writeLedger(
}
void
CassandraBackend::writeAccountTransactions(
std::vector<AccountTransactionsData>&& data)
CassandraBackend::writeAccountTransactions(std::vector<AccountTransactionsData>&& data)
{
for (auto& record : data)
{
@@ -299,11 +266,7 @@ CassandraBackend::writeAccountTransactions(
{
makeAndExecuteAsyncWrite(
this,
std::make_tuple(
std::move(account),
record.ledgerSequence,
record.transactionIndex,
record.txHash),
std::make_tuple(std::move(account), record.ledgerSequence, record.transactionIndex, record.txHash),
[this](auto& params) {
CassandraStatement statement(insertAccountTx_);
auto& [account, lgrSeq, txnIdx, hash] = params.data;
@@ -324,11 +287,7 @@ CassandraBackend::writeNFTTransactions(std::vector<NFTTransactionsData>&& data)
{
makeAndExecuteAsyncWrite(
this,
std::make_tuple(
record.tokenID,
record.ledgerSequence,
record.transactionIndex,
record.txHash),
std::make_tuple(record.tokenID, record.ledgerSequence, record.transactionIndex, record.txHash),
[this](auto const& params) {
CassandraStatement statement(insertNFTTx_);
auto const& [tokenID, lgrSeq, txnIdx, txHash] = params.data;
@@ -364,12 +323,7 @@ CassandraBackend::writeTransaction(
"ledger_transaction");
makeAndExecuteAsyncWrite(
this,
std::make_tuple(
std::move(hash),
seq,
date,
std::move(transaction),
std::move(metadata)),
std::make_tuple(std::move(hash), seq, date, std::move(transaction), std::move(metadata)),
[this](auto& params) {
CassandraStatement statement{insertTransaction_};
auto& [hash, sequence, date, transaction, metadata] = params.data;
@@ -390,11 +344,7 @@ CassandraBackend::writeNFTs(std::vector<NFTsData>&& data)
{
makeAndExecuteAsyncWrite(
this,
std::make_tuple(
record.tokenID,
record.ledgerSequence,
record.owner,
record.isBurned),
std::make_tuple(record.tokenID, record.ledgerSequence, record.owner, record.isBurned),
[this](auto const& params) {
CassandraStatement statement{insertNFT_};
auto const& [tokenID, lgrSeq, owner, isBurned] = params.data;
@@ -420,8 +370,7 @@ CassandraBackend::writeNFTs(std::vector<NFTsData>&& data)
CassandraStatement statement{insertIssuerNFT_};
auto const& [tokenID] = params.data;
statement.bindNextBytes(ripple::nft::getIssuer(tokenID));
statement.bindNextInt(
ripple::nft::toUInt32(ripple::nft::getTaxon(tokenID)));
statement.bindNextInt(ripple::nft::toUInt32(ripple::nft::getTaxon(tokenID)));
statement.bindNextBytes(tokenID);
return statement;
},
@@ -429,8 +378,7 @@ CassandraBackend::writeNFTs(std::vector<NFTsData>&& data)
makeAndExecuteAsyncWrite(
this,
std::make_tuple(
record.tokenID, record.ledgerSequence, record.uri.value()),
std::make_tuple(record.tokenID, record.ledgerSequence, record.uri.value()),
[this](auto const& params) {
CassandraStatement statement{insertNFTURI_};
auto const& [tokenID, lgrSeq, uri] = params.data;
@@ -470,9 +418,8 @@ CassandraBackend::hardFetchLedgerRange(boost::asio::yield_context& yield) const
}
std::vector<TransactionAndMetadata>
CassandraBackend::fetchAllTransactionsInLedger(
std::uint32_t const ledgerSequence,
boost::asio::yield_context& yield) const
CassandraBackend::fetchAllTransactionsInLedger(std::uint32_t const ledgerSequence, boost::asio::yield_context& yield)
const
{
auto hashes = fetchAllTransactionHashesInLedger(ledgerSequence, yield);
return fetchTransactions(hashes, yield);
@@ -517,26 +464,21 @@ struct ReadCallbackData
void
resume()
{
boost::asio::post(
boost::asio::get_associated_executor(handler),
[handler = std::move(handler)]() mutable {
handler(boost::system::error_code{});
});
boost::asio::post(boost::asio::get_associated_executor(handler), [handler = std::move(handler)]() mutable {
handler(boost::system::error_code{});
});
}
};
void
processAsyncRead(CassFuture* fut, void* cbData)
{
ReadCallbackData<result_type>& cb =
*static_cast<ReadCallbackData<result_type>*>(cbData);
ReadCallbackData<result_type>& cb = *static_cast<ReadCallbackData<result_type>*>(cbData);
cb.finish(fut);
}
std::vector<TransactionAndMetadata>
CassandraBackend::fetchTransactions(
std::vector<ripple::uint256> const& hashes,
boost::asio::yield_context& yield) const
CassandraBackend::fetchTransactions(std::vector<ripple::uint256> const& hashes, boost::asio::yield_context& yield) const
{
if (hashes.size() == 0)
return {};
@@ -556,14 +498,10 @@ CassandraBackend::fetchTransactions(
CassandraStatement statement{selectTransaction_};
statement.bindNextBytes(hashes[i]);
cbs.push_back(std::make_shared<ReadCallbackData<result_type>>(
numOutstanding, handler, [i, &results](auto& result) {
cbs.push_back(
std::make_shared<ReadCallbackData<result_type>>(numOutstanding, handler, [i, &results](auto& result) {
if (result.hasResult())
results[i] = {
result.getBytes(),
result.getBytes(),
result.getUInt32(),
result.getUInt32()};
results[i] = {result.getBytes(), result.getBytes(), result.getUInt32(), result.getUInt32()};
}));
executeAsyncRead(statement, processAsyncRead, *cbs[i]);
@@ -580,9 +518,7 @@ CassandraBackend::fetchTransactions(
throw DatabaseTimeout();
}
log_.debug() << "Fetched " << numHashes
<< " transactions from Cassandra in " << timeDiff
<< " milliseconds";
log_.debug() << "Fetched " << numHashes << " transactions from Cassandra in " << timeDiff << " milliseconds";
return results;
}
@@ -608,12 +544,8 @@ CassandraBackend::fetchAllTransactionHashesInLedger(
{
hashes.push_back(result.getUInt256());
} while (result.nextRow());
log_.debug() << "Fetched " << hashes.size()
<< " transaction hashes from Cassandra in "
<< std::chrono::duration_cast<std::chrono::milliseconds>(
end - start)
.count()
<< " milliseconds";
log_.debug() << "Fetched " << hashes.size() << " transaction hashes from Cassandra in "
<< std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count() << " milliseconds";
return hashes;
}
@@ -668,29 +600,23 @@ CassandraBackend::fetchNFTTransactions(
if (!rng)
return {{}, {}};
CassandraStatement statement = forward
? CassandraStatement(selectNFTTxForward_)
: CassandraStatement(selectNFTTx_);
CassandraStatement statement = forward ? CassandraStatement(selectNFTTxForward_) : CassandraStatement(selectNFTTx_);
statement.bindNextBytes(tokenID);
if (cursor)
{
statement.bindNextIntTuple(
cursor->ledgerSequence, cursor->transactionIndex);
log_.debug() << "token_id = " << ripple::strHex(tokenID)
<< " tuple = " << cursor->ledgerSequence
statement.bindNextIntTuple(cursor->ledgerSequence, cursor->transactionIndex);
log_.debug() << "token_id = " << ripple::strHex(tokenID) << " tuple = " << cursor->ledgerSequence
<< cursor->transactionIndex;
}
else
{
int const seq = forward ? rng->minSequence : rng->maxSequence;
int const placeHolder =
forward ? 0 : std::numeric_limits<std::uint32_t>::max();
int const placeHolder = forward ? 0 : std::numeric_limits<std::uint32_t>::max();
statement.bindNextIntTuple(placeHolder, placeHolder);
log_.debug() << "token_id = " << ripple::strHex(tokenID)
<< " idx = " << seq << " tuple = " << placeHolder;
log_.debug() << "token_id = " << ripple::strHex(tokenID) << " idx = " << seq << " tuple = " << placeHolder;
}
statement.bindNextUInt(limit);
@@ -713,9 +639,7 @@ CassandraBackend::fetchNFTTransactions(
{
log_.debug() << "Setting cursor";
auto const [lgrSeq, txnIdx] = result.getInt64Tuple();
cursor = {
static_cast<std::uint32_t>(lgrSeq),
static_cast<std::uint32_t>(txnIdx)};
cursor = {static_cast<std::uint32_t>(lgrSeq), static_cast<std::uint32_t>(txnIdx)};
// Only modify if forward because forward query
// (selectNFTTxForward_) orders by ledger/tx sequence >= whereas
@@ -760,21 +684,17 @@ CassandraBackend::fetchAccountTransactions(
statement.bindNextBytes(account);
if (cursor)
{
statement.bindNextIntTuple(
cursor->ledgerSequence, cursor->transactionIndex);
log_.debug() << "account = " << ripple::strHex(account)
<< " tuple = " << cursor->ledgerSequence
statement.bindNextIntTuple(cursor->ledgerSequence, cursor->transactionIndex);
log_.debug() << "account = " << ripple::strHex(account) << " tuple = " << cursor->ledgerSequence
<< cursor->transactionIndex;
}
else
{
int const seq = forward ? rng->minSequence : rng->maxSequence;
int const placeHolder =
forward ? 0 : std::numeric_limits<std::uint32_t>::max();
int const placeHolder = forward ? 0 : std::numeric_limits<std::uint32_t>::max();
statement.bindNextIntTuple(placeHolder, placeHolder);
log_.debug() << "account = " << ripple::strHex(account)
<< " idx = " << seq << " tuple = " << placeHolder;
log_.debug() << "account = " << ripple::strHex(account) << " idx = " << seq << " tuple = " << placeHolder;
}
statement.bindNextUInt(limit);
@@ -796,9 +716,7 @@ CassandraBackend::fetchAccountTransactions(
{
log_.debug() << "Setting cursor";
auto [lgrSeq, txnIdx] = result.getInt64Tuple();
cursor = {
static_cast<std::uint32_t>(lgrSeq),
static_cast<std::uint32_t>(txnIdx)};
cursor = {static_cast<std::uint32_t>(lgrSeq), static_cast<std::uint32_t>(txnIdx)};
// Only modify if forward because forward query
// (selectAccountTxForward_) orders by ledger/tx sequence >= whereas
@@ -890,8 +808,8 @@ CassandraBackend::doFetchLedgerObjects(
cbs.reserve(numKeys);
for (std::size_t i = 0; i < keys.size(); ++i)
{
cbs.push_back(std::make_shared<ReadCallbackData<result_type>>(
numOutstanding, handler, [i, &results](auto& result) {
cbs.push_back(
std::make_shared<ReadCallbackData<result_type>>(numOutstanding, handler, [i, &results](auto& result) {
if (result.hasResult())
results[i] = result.getBytes();
}));
@@ -917,9 +835,7 @@ CassandraBackend::doFetchLedgerObjects(
}
std::vector<LedgerObject>
CassandraBackend::fetchLedgerDiff(
std::uint32_t const ledgerSequence,
boost::asio::yield_context& yield) const
CassandraBackend::fetchLedgerDiff(std::uint32_t const ledgerSequence, boost::asio::yield_context& yield) const
{
CassandraStatement statement{selectDiff_};
statement.bindNextInt(ledgerSequence);
@@ -939,29 +855,19 @@ CassandraBackend::fetchLedgerDiff(
{
keys.push_back(result.getUInt256());
} while (result.nextRow());
log_.debug() << "Fetched " << keys.size()
<< " diff hashes from Cassandra in "
<< std::chrono::duration_cast<std::chrono::milliseconds>(
end - start)
.count()
<< " milliseconds";
log_.debug() << "Fetched " << keys.size() << " diff hashes from Cassandra in "
<< std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count() << " milliseconds";
auto objs = fetchLedgerObjects(keys, ledgerSequence, yield);
std::vector<LedgerObject> results;
std::transform(
keys.begin(),
keys.end(),
objs.begin(),
std::back_inserter(results),
[](auto const& k, auto const& o) {
keys.begin(), keys.end(), objs.begin(), std::back_inserter(results), [](auto const& k, auto const& o) {
return LedgerObject{k, o};
});
return results;
}
bool
CassandraBackend::doOnlineDelete(
std::uint32_t const numLedgersToKeep,
boost::asio::yield_context& yield) const
CassandraBackend::doOnlineDelete(std::uint32_t const numLedgersToKeep, boost::asio::yield_context& yield) const
{
// calculate TTL
// ledgers close roughly every 4 seconds. We double the TTL so that way
@@ -994,17 +900,15 @@ CassandraBackend::doOnlineDelete(
std::optional<ripple::uint256> cursor;
while (true)
{
auto [objects, curCursor] = retryOnTimeout([&]() {
return fetchLedgerPage(cursor, minLedger, 256, false, yield);
});
auto [objects, curCursor] =
retryOnTimeout([&]() { return fetchLedgerPage(cursor, minLedger, 256, false, yield); });
for (auto& obj : objects)
{
++numOutstanding;
cbs.push_back(makeAndExecuteBulkAsyncWrite(
this,
std::make_tuple(
std::move(obj.key), minLedger, std::move(obj.blob)),
std::make_tuple(std::move(obj.key), minLedger, std::move(obj.blob)),
bind,
numOutstanding,
mtx,
@@ -1012,9 +916,7 @@ CassandraBackend::doOnlineDelete(
std::unique_lock<std::mutex> lck(mtx);
log_.trace() << "Got the mutex";
cv.wait(lck, [&numOutstanding, concurrentLimit]() {
return numOutstanding < concurrentLimit;
});
cv.wait(lck, [&numOutstanding, concurrentLimit]() { return numOutstanding < concurrentLimit; });
}
log_.debug() << "Fetched a page";
cursor = curCursor;
@@ -1052,15 +954,13 @@ CassandraBackend::open(bool readOnly)
if (!cluster)
throw std::runtime_error("nodestore:: Failed to create CassCluster");
std::string secureConnectBundle =
config_.valueOr<std::string>("secure_connect_bundle", "");
std::string secureConnectBundle = config_.valueOr<std::string>("secure_connect_bundle", "");
if (!secureConnectBundle.empty())
{
/* Setup driver to connect to the cloud using the secure connection
* bundle */
if (cass_cluster_set_cloud_secure_connection_bundle(
cluster, secureConnectBundle.c_str()) != CASS_OK)
if (cass_cluster_set_cloud_secure_connection_bundle(cluster, secureConnectBundle.c_str()) != CASS_OK)
{
log_.error() << "Unable to configure cloud using the "
"secure connection bundle: "
@@ -1074,15 +974,12 @@ CassandraBackend::open(bool readOnly)
else
{
std::string contact_points = config_.valueOrThrow<std::string>(
"contact_points",
"nodestore: Missing contact_points in Cassandra config");
CassError rc =
cass_cluster_set_contact_points(cluster, contact_points.c_str());
"contact_points", "nodestore: Missing contact_points in Cassandra config");
CassError rc = cass_cluster_set_contact_points(cluster, contact_points.c_str());
if (rc != CASS_OK)
{
std::stringstream ss;
ss << "nodestore: Error setting Cassandra contact_points: "
<< contact_points << ", result: " << rc << ", "
ss << "nodestore: Error setting Cassandra contact_points: " << contact_points << ", result: " << rc << ", "
<< cass_error_desc(rc);
throw std::runtime_error(ss.str());
@@ -1095,16 +992,15 @@ CassandraBackend::open(bool readOnly)
if (rc != CASS_OK)
{
std::stringstream ss;
ss << "nodestore: Error setting Cassandra port: " << *port
<< ", result: " << rc << ", " << cass_error_desc(rc);
ss << "nodestore: Error setting Cassandra port: " << *port << ", result: " << rc << ", "
<< cass_error_desc(rc);
throw std::runtime_error(ss.str());
}
}
}
cass_cluster_set_token_aware_routing(cluster, cass_true);
CassError rc =
cass_cluster_set_protocol_version(cluster, CASS_PROTOCOL_VERSION_V4);
CassError rc = cass_cluster_set_protocol_version(cluster, CASS_PROTOCOL_VERSION_V4);
if (rc != CASS_OK)
{
std::stringstream ss;
@@ -1119,40 +1015,32 @@ CassandraBackend::open(bool readOnly)
{
log_.debug() << "user = " << *username;
auto password = config_.value<std::string>("password");
cass_cluster_set_credentials(
cluster, username->c_str(), password.c_str());
cass_cluster_set_credentials(cluster, username->c_str(), password.c_str());
}
auto threads =
config_.valueOr<int>("threads", std::thread::hardware_concurrency());
auto threads = config_.valueOr<int>("threads", std::thread::hardware_concurrency());
rc = cass_cluster_set_num_threads_io(cluster, threads);
if (rc != CASS_OK)
{
std::stringstream ss;
ss << "nodestore: Error setting Cassandra io threads to " << threads
<< ", result: " << rc << ", " << cass_error_desc(rc);
ss << "nodestore: Error setting Cassandra io threads to " << threads << ", result: " << rc << ", "
<< cass_error_desc(rc);
throw std::runtime_error(ss.str());
}
maxWriteRequestsOutstanding = config_.valueOr<int>(
"max_write_requests_outstanding", maxWriteRequestsOutstanding);
maxReadRequestsOutstanding = config_.valueOr<int>(
"max_read_requests_outstanding", maxReadRequestsOutstanding);
maxWriteRequestsOutstanding = config_.valueOr<int>("max_write_requests_outstanding", maxWriteRequestsOutstanding);
maxReadRequestsOutstanding = config_.valueOr<int>("max_read_requests_outstanding", maxReadRequestsOutstanding);
syncInterval_ = config_.valueOr<int>("sync_interval", syncInterval_);
log_.info() << "Sync interval is " << syncInterval_
<< ". max write requests outstanding is "
<< maxWriteRequestsOutstanding
<< ". max read requests outstanding is "
<< maxReadRequestsOutstanding;
log_.info() << "Sync interval is " << syncInterval_ << ". max write requests outstanding is "
<< maxWriteRequestsOutstanding << ". max read requests outstanding is " << maxReadRequestsOutstanding;
cass_cluster_set_request_timeout(cluster, 10000);
rc = cass_cluster_set_queue_size_io(
cluster,
maxWriteRequestsOutstanding +
maxReadRequestsOutstanding); // This number needs to scale w/ the
// number of request per sec
maxWriteRequestsOutstanding + maxReadRequestsOutstanding); // This number needs to scale w/ the
// number of request per sec
if (rc != CASS_OK)
{
std::stringstream ss;
@@ -1165,17 +1053,14 @@ CassandraBackend::open(bool readOnly)
if (auto certfile = config_.maybeValue<std::string>("certfile"); certfile)
{
std::ifstream fileStream(
boost::filesystem::path(*certfile).string(), std::ios::in);
std::ifstream fileStream(boost::filesystem::path(*certfile).string(), std::ios::in);
if (!fileStream)
{
std::stringstream ss;
ss << "opening config file " << *certfile;
throw std::system_error(errno, std::generic_category(), ss.str());
}
std::string cert(
std::istreambuf_iterator<char>{fileStream},
std::istreambuf_iterator<char>{});
std::string cert(std::istreambuf_iterator<char>{fileStream}, std::istreambuf_iterator<char>{});
if (fileStream.bad())
{
std::stringstream ss;
@@ -1189,8 +1074,7 @@ CassandraBackend::open(bool readOnly)
if (rc != CASS_OK)
{
std::stringstream ss;
ss << "nodestore: Error setting Cassandra ssl context: " << rc
<< ", " << cass_error_desc(rc);
ss << "nodestore: Error setting Cassandra ssl context: " << rc << ", " << cass_error_desc(rc);
throw std::runtime_error(ss.str());
}
@@ -1226,8 +1110,8 @@ CassandraBackend::open(bool readOnly)
if (rc != CASS_OK && rc != CASS_ERROR_SERVER_INVALID_QUERY)
{
std::stringstream ss;
ss << "nodestore: Error executing simple statement: " << rc << ", "
<< cass_error_desc(rc) << " - " << query;
ss << "nodestore: Error executing simple statement: " << rc << ", " << cass_error_desc(rc) << " - "
<< query;
log_.error() << ss.str();
return false;
}
@@ -1241,15 +1125,13 @@ CassandraBackend::open(bool readOnly)
session_.reset(cass_session_new());
assert(session_);
fut = cass_session_connect_keyspace(
session_.get(), cluster, keyspace.c_str());
fut = cass_session_connect_keyspace(session_.get(), cluster, keyspace.c_str());
rc = cass_future_error_code(fut);
cass_future_free(fut);
if (rc != CASS_OK)
{
std::stringstream ss;
ss << "nodestore: Error connecting Cassandra session keyspace: "
<< rc << ", " << cass_error_desc(rc)
ss << "nodestore: Error connecting Cassandra session keyspace: " << rc << ", " << cass_error_desc(rc)
<< ", trying to create it ourselves";
log_.error() << ss.str();
// if the keyspace doesn't exist, try to create it
@@ -1260,8 +1142,7 @@ CassandraBackend::open(bool readOnly)
if (rc != CASS_OK)
{
std::stringstream ss;
ss << "nodestore: Error connecting Cassandra session at all: "
<< rc << ", " << cass_error_desc(rc);
ss << "nodestore: Error connecting Cassandra session at all: " << rc << ", " << cass_error_desc(rc);
log_.error() << ss.str();
}
else
@@ -1298,16 +1179,14 @@ CassandraBackend::open(bool readOnly)
continue;
query.str("");
query
<< "CREATE TABLE IF NOT EXISTS " << tablePrefix << "transactions"
<< " ( hash blob PRIMARY KEY, ledger_sequence bigint, date bigint, "
"transaction blob, metadata blob)"
<< " WITH default_time_to_live = " << std::to_string(ttl);
query << "CREATE TABLE IF NOT EXISTS " << tablePrefix << "transactions"
<< " ( hash blob PRIMARY KEY, ledger_sequence bigint, date bigint, "
"transaction blob, metadata blob)"
<< " WITH default_time_to_live = " << std::to_string(ttl);
if (!executeSimpleStatement(query.str()))
continue;
query.str("");
query << "CREATE TABLE IF NOT EXISTS " << tablePrefix
<< "ledger_transactions"
query << "CREATE TABLE IF NOT EXISTS " << tablePrefix << "ledger_transactions"
<< " ( ledger_sequence bigint, hash blob, PRIMARY "
"KEY(ledger_sequence, hash))"
<< " WITH default_time_to_live = " << std::to_string(ttl);
@@ -1429,8 +1308,7 @@ CassandraBackend::open(bool readOnly)
continue;
query.str("");
query << "CREATE TABLE IF NOT EXISTS " << tablePrefix
<< "issuer_nf_tokens_v2"
query << "CREATE TABLE IF NOT EXISTS " << tablePrefix << "issuer_nf_tokens_v2"
<< " ("
<< " issuer blob,"
<< " taxon bigint,"
@@ -1466,8 +1344,7 @@ CassandraBackend::open(bool readOnly)
continue;
query.str("");
query << "CREATE TABLE IF NOT EXISTS " << tablePrefix
<< "nf_token_transactions"
query << "CREATE TABLE IF NOT EXISTS " << tablePrefix << "nf_token_transactions"
<< " ("
<< " token_id blob,"
<< " seq_idx tuple<bigint, bigint>,"
@@ -1546,8 +1423,7 @@ CassandraBackend::open(bool readOnly)
continue;
query.str("");
query << "SELECT transaction, metadata, ledger_sequence, date FROM "
<< tablePrefix << "transactions"
query << "SELECT transaction, metadata, ledger_sequence, date FROM " << tablePrefix << "transactions"
<< " WHERE hash = ?";
if (!selectTransaction_.prepareStatement(query, session_.get()))
continue;
@@ -1555,8 +1431,7 @@ CassandraBackend::open(bool readOnly)
query.str("");
query << "SELECT hash FROM " << tablePrefix << "ledger_transactions"
<< " WHERE ledger_sequence = ?";
if (!selectAllTransactionHashesInLedger_.prepareStatement(
query, session_.get()))
if (!selectAllTransactionHashesInLedger_.prepareStatement(query, session_.get()))
continue;
query.str("");
@@ -1701,14 +1576,12 @@ CassandraBackend::open(bool readOnly)
continue;
query.str("");
query << " select header from " << tablePrefix
<< "ledgers where sequence = ?";
query << " select header from " << tablePrefix << "ledgers where sequence = ?";
if (!selectLedgerBySeq_.prepareStatement(query, session_.get()))
continue;
query.str("");
query << " select sequence from " << tablePrefix
<< "ledger_range where is_latest = true";
query << " select sequence from " << tablePrefix << "ledger_range where is_latest = true";
if (!selectLatestLedger_.prepareStatement(query, session_.get()))
continue;

263
src/backend/CassandraBackend.h
View File

@@ -85,8 +85,8 @@ public:
else
{
std::stringstream ss;
ss << "nodestore: Error preparing statement : " << rc << ", "
<< cass_error_desc(rc) << ". query : " << query;
ss << "nodestore: Error preparing statement : " << rc << ", " << cass_error_desc(rc)
<< ". query : " << query;
log_.error() << ss.str();
}
cass_future_free(prepareFuture);
@@ -137,15 +137,12 @@ public:
bindNextBoolean(bool val)
{
if (!statement_)
throw std::runtime_error(
"CassandraStatement::bindNextBoolean - statement_ is null");
CassError rc = cass_statement_bind_bool(
statement_, curBindingIndex_, static_cast<cass_bool_t>(val));
throw std::runtime_error("CassandraStatement::bindNextBoolean - statement_ is null");
CassError rc = cass_statement_bind_bool(statement_, curBindingIndex_, static_cast<cass_bool_t>(val));
if (rc != CASS_OK)
{
std::stringstream ss;
ss << "Error binding boolean to statement: " << rc << ", "
<< cass_error_desc(rc);
ss << "Error binding boolean to statement: " << rc << ", " << cass_error_desc(rc);
log_.error() << ss.str();
throw std::runtime_error(ss.str());
}
@@ -190,18 +187,13 @@ public:
bindNextBytes(const unsigned char* data, std::uint32_t const size)
{
if (!statement_)
throw std::runtime_error(
"CassandraStatement::bindNextBytes - statement_ is null");
CassError rc = cass_statement_bind_bytes(
statement_,
curBindingIndex_,
static_cast<cass_byte_t const*>(data),
size);
throw std::runtime_error("CassandraStatement::bindNextBytes - statement_ is null");
CassError rc =
cass_statement_bind_bytes(statement_, curBindingIndex_, static_cast<cass_byte_t const*>(data), size);
if (rc != CASS_OK)
{
std::stringstream ss;
ss << "Error binding bytes to statement: " << rc << ", "
<< cass_error_desc(rc);
ss << "Error binding bytes to statement: " << rc << ", " << cass_error_desc(rc);
log_.error() << ss.str();
throw std::runtime_error(ss.str());
}
@@ -212,17 +204,13 @@ public:
bindNextUInt(std::uint32_t const value)
{
if (!statement_)
throw std::runtime_error(
"CassandraStatement::bindNextUInt - statement_ is null");
log_.trace() << std::to_string(curBindingIndex_) << " "
<< std::to_string(value);
CassError rc =
cass_statement_bind_int32(statement_, curBindingIndex_, value);
throw std::runtime_error("CassandraStatement::bindNextUInt - statement_ is null");
log_.trace() << std::to_string(curBindingIndex_) << " " << std::to_string(value);
CassError rc = cass_statement_bind_int32(statement_, curBindingIndex_, value);
if (rc != CASS_OK)
{
std::stringstream ss;
ss << "Error binding uint to statement: " << rc << ", "
<< cass_error_desc(rc);
ss << "Error binding uint to statement: " << rc << ", " << cass_error_desc(rc);
log_.error() << ss.str();
throw std::runtime_error(ss.str());
}
@@ -239,15 +227,12 @@ public:
bindNextInt(int64_t value)
{
if (!statement_)
throw std::runtime_error(
"CassandraStatement::bindNextInt - statement_ is null");
CassError rc =
cass_statement_bind_int64(statement_, curBindingIndex_, value);
throw std::runtime_error("CassandraStatement::bindNextInt - statement_ is null");
CassError rc = cass_statement_bind_int64(statement_, curBindingIndex_, value);
if (rc != CASS_OK)
{
std::stringstream ss;
ss << "Error binding int to statement: " << rc << ", "
<< cass_error_desc(rc);
ss << "Error binding int to statement: " << rc << ", " << cass_error_desc(rc);
log_.error() << ss.str();
throw std::runtime_error(ss.str());
}
@@ -262,8 +247,7 @@ public:
if (rc != CASS_OK)
{
std::stringstream ss;
ss << "Error binding int to tuple: " << rc << ", "
<< cass_error_desc(rc);
ss << "Error binding int to tuple: " << rc << ", " << cass_error_desc(rc);
log_.error() << ss.str();
throw std::runtime_error(ss.str());
}
@@ -271,8 +255,7 @@ public:
if (rc != CASS_OK)
{
std::stringstream ss;
ss << "Error binding int to tuple: " << rc << ", "
<< cass_error_desc(rc);
ss << "Error binding int to tuple: " << rc << ", " << cass_error_desc(rc);
log_.error() << ss.str();
throw std::runtime_error(ss.str());
}
@@ -280,8 +263,7 @@ public:
if (rc != CASS_OK)
{
std::stringstream ss;
ss << "Error binding tuple to statement: " << rc << ", "
<< cass_error_desc(rc);
ss << "Error binding tuple to statement: " << rc << ", " << cass_error_desc(rc);
log_.error() << ss.str();
throw std::runtime_error(ss.str());
}
@@ -382,13 +364,11 @@ public:
throw std::runtime_error("CassandraResult::getBytes - no result");
cass_byte_t const* buf;
std::size_t bufSize;
CassError rc = cass_value_get_bytes(
cass_row_get_column(row_, curGetIndex_), &buf, &bufSize);
CassError rc = cass_value_get_bytes(cass_row_get_column(row_, curGetIndex_), &buf, &bufSize);
if (rc != CASS_OK)
{
std::stringstream msg;
msg << "CassandraResult::getBytes - error getting value: " << rc
<< ", " << cass_error_desc(rc);
msg << "CassandraResult::getBytes - error getting value: " << rc << ", " << cass_error_desc(rc);
log_.error() << msg.str();
throw std::runtime_error(msg.str());
}
@@ -403,13 +383,11 @@ public:
throw std::runtime_error("CassandraResult::uint256 - no result");
cass_byte_t const* buf;
std::size_t bufSize;
CassError rc = cass_value_get_bytes(
cass_row_get_column(row_, curGetIndex_), &buf, &bufSize);
CassError rc = cass_value_get_bytes(cass_row_get_column(row_, curGetIndex_), &buf, &bufSize);
if (rc != CASS_OK)
{
std::stringstream msg;
msg << "CassandraResult::getuint256 - error getting value: " << rc
<< ", " << cass_error_desc(rc);
msg << "CassandraResult::getuint256 - error getting value: " << rc << ", " << cass_error_desc(rc);
log_.error() << msg.str();
throw std::runtime_error(msg.str());
}
@@ -423,13 +401,11 @@ public:
if (!row_)
throw std::runtime_error("CassandraResult::getInt64 - no result");
cass_int64_t val;
CassError rc =
cass_value_get_int64(cass_row_get_column(row_, curGetIndex_), &val);
CassError rc = cass_value_get_int64(cass_row_get_column(row_, curGetIndex_), &val);
if (rc != CASS_OK)
{
std::stringstream msg;
msg << "CassandraResult::getInt64 - error getting value: " << rc
<< ", " << cass_error_desc(rc);
msg << "CassandraResult::getInt64 - error getting value: " << rc << ", " << cass_error_desc(rc);
log_.error() << msg.str();
throw std::runtime_error(msg.str());
}
@@ -447,8 +423,7 @@ public:
getInt64Tuple()
{
if (!row_)
throw std::runtime_error(
"CassandraResult::getInt64Tuple - no result");
throw std::runtime_error("CassandraResult::getInt64Tuple - no result");
CassValue const* tuple = cass_row_get_column(row_, curGetIndex_);
CassIterator* tupleIter = cass_iterator_from_tuple(tuple);
@@ -456,8 +431,7 @@ public:
if (!cass_iterator_next(tupleIter))
{
cass_iterator_free(tupleIter);
throw std::runtime_error(
"CassandraResult::getInt64Tuple - failed to iterate tuple");
throw std::runtime_error("CassandraResult::getInt64Tuple - failed to iterate tuple");
}
CassValue const* value = cass_iterator_get_value(tupleIter);
@@ -466,8 +440,7 @@ public:
if (!cass_iterator_next(tupleIter))
{
cass_iterator_free(tupleIter);
throw std::runtime_error(
"CassandraResult::getInt64Tuple - failed to iterate tuple");
throw std::runtime_error("CassandraResult::getInt64Tuple - failed to iterate tuple");
}
value = cass_iterator_get_value(tupleIter);
@@ -486,20 +459,17 @@ public:
std::size_t bufSize;
if (!row_)
throw std::runtime_error(
"CassandraResult::getBytesTuple - no result");
throw std::runtime_error("CassandraResult::getBytesTuple - no result");
CassValue const* tuple = cass_row_get_column(row_, curGetIndex_);
CassIterator* tupleIter = cass_iterator_from_tuple(tuple);
if (!cass_iterator_next(tupleIter))
throw std::runtime_error(
"CassandraResult::getBytesTuple - failed to iterate tuple");
throw std::runtime_error("CassandraResult::getBytesTuple - failed to iterate tuple");
CassValue const* value = cass_iterator_get_value(tupleIter);
cass_value_get_bytes(value, &buf, &bufSize);
Blob first{buf, buf + bufSize};
if (!cass_iterator_next(tupleIter))
throw std::runtime_error(
"CassandraResult::getBytesTuple - failed to iterate tuple");
throw std::runtime_error("CassandraResult::getBytesTuple - failed to iterate tuple");
value = cass_iterator_get_value(tupleIter);
cass_value_get_bytes(value, &buf, &bufSize);
Blob second{buf, buf + bufSize};
@@ -519,8 +489,7 @@ public:
throw std::runtime_error(msg);
}
cass_bool_t val;
CassError rc =
cass_value_get_bool(cass_row_get_column(row_, curGetIndex_), &val);
CassError rc = cass_value_get_bool(cass_row_get_column(row_, curGetIndex_), &val);
if (rc != CASS_OK)
{
std::stringstream msg;
@@ -544,10 +513,8 @@ public:
inline bool
isTimeout(CassError rc)
{
if (rc == CASS_ERROR_LIB_NO_HOSTS_AVAILABLE or
rc == CASS_ERROR_LIB_REQUEST_TIMED_OUT or
rc == CASS_ERROR_SERVER_UNAVAILABLE or
rc == CASS_ERROR_SERVER_OVERLOADED or
if (rc == CASS_ERROR_LIB_NO_HOSTS_AVAILABLE or rc == CASS_ERROR_LIB_REQUEST_TIMED_OUT or
rc == CASS_ERROR_SERVER_UNAVAILABLE or rc == CASS_ERROR_SERVER_OVERLOADED or
rc == CASS_ERROR_SERVER_READ_TIMEOUT)
return true;
return false;
@@ -558,8 +525,7 @@ CassError
cass_future_error_code(CassFuture* fut, CompletionToken&& token)
{
using function_type = void(boost::system::error_code, CassError);
using result_type =
boost::asio::async_result<CompletionToken, function_type>;
using result_type = boost::asio::async_result<CompletionToken, function_type>;
using handler_type = typename result_type::completion_handler_type;
handler_type handler(std::forward<decltype(token)>(token));
@@ -578,12 +544,10 @@ cass_future_error_code(CassFuture* fut, CompletionToken&& token)
HandlerWrapper* hw = (HandlerWrapper*)data;
boost::asio::post(
boost::asio::get_associated_executor(hw->handler),
[fut, hw, handler = std::move(hw->handler)]() mutable {
boost::asio::get_associated_executor(hw->handler), [fut, hw, handler = std::move(hw->handler)]() mutable {
delete hw;
handler(
boost::system::error_code{}, cass_future_error_code(fut));
handler(boost::system::error_code{}, cass_future_error_code(fut));
});
};
@@ -608,13 +572,12 @@ private:
makeStatement(char const* query, std::size_t params)
{
CassStatement* ret = cass_statement_new(query, params);
CassError rc =
cass_statement_set_consistency(ret, CASS_CONSISTENCY_QUORUM);
CassError rc = cass_statement_set_consistency(ret, CASS_CONSISTENCY_QUORUM);
if (rc != CASS_OK)
{
std::stringstream ss;
ss << "nodestore: Error setting query consistency: " << query
<< ", result: " << rc << ", " << cass_error_desc(rc);
ss << "nodestore: Error setting query consistency: " << query << ", result: " << rc << ", "
<< cass_error_desc(rc);
throw std::runtime_error(ss.str());
}
return ret;
@@ -623,15 +586,13 @@ private:
clio::Logger log_{"Backend"};
std::atomic<bool> open_{false};
std::unique_ptr<CassSession, void (*)(CassSession*)> session_{
nullptr,
[](CassSession* session) {
// Try to disconnect gracefully.
CassFuture* fut = cass_session_close(session);
cass_future_wait(fut);
cass_future_free(fut);
cass_session_free(session);
}};
std::unique_ptr<CassSession, void (*)(CassSession*)> session_{nullptr, [](CassSession* session) {
// Try to disconnect gracefully.
CassFuture* fut = cass_session_close(session);
cass_future_wait(fut);
cass_future_free(fut);
cass_session_free(session);
}};
// Database statements cached server side. Using these is more efficient
// than making a new statement
@@ -704,10 +665,7 @@ private:
mutable std::uint32_t ledgerSequence_ = 0;
public:
CassandraBackend(
boost::asio::io_context& ioc,
clio::Config const& config,
uint32_t ttl)
CassandraBackend(boost::asio::io_context& ioc, clio::Config const& config, uint32_t ttl)
: config_(config), ttl_(ttl)
{
work_.emplace(ioContext_);
@@ -777,8 +735,7 @@ public:
statement.bindNextInt(ledgerSequence_ - 1);
if (!executeSyncUpdate(statement))
{
log_.warn() << "Update failed for ledger "
<< std::to_string(ledgerSequence_) << ". Returning";
log_.warn() << "Update failed for ledger " << std::to_string(ledgerSequence_) << ". Returning";
return false;
}
log_.info() << "Committed ledger " << std::to_string(ledgerSequence_);
@@ -792,8 +749,7 @@ public:
// if db is empty, sync. if sync interval is 1, always sync.
// if we've never synced, sync. if its been greater than the configured
// sync interval since we last synced, sync.
if (!range || lastSync_ == 0 ||
ledgerSequence_ - syncInterval_ >= lastSync_)
if (!range || lastSync_ == 0 || ledgerSequence_ - syncInterval_ >= lastSync_)
{
// wait for all other writes to finish
sync();
@@ -815,20 +771,16 @@ public:
statement.bindNextInt(lastSync_);
if (!executeSyncUpdate(statement))
{
log_.warn() << "Update failed for ledger "
<< std::to_string(ledgerSequence_) << ". Returning";
log_.warn() << "Update failed for ledger " << std::to_string(ledgerSequence_) << ". Returning";
return false;
}
log_.info() << "Committed ledger "
<< std::to_string(ledgerSequence_);
log_.info() << "Committed ledger " << std::to_string(ledgerSequence_);
lastSync_ = ledgerSequence_;
}
else
{
log_.info() << "Skipping commit. sync interval is "
<< std::to_string(syncInterval_) << " - last sync is "
<< std::to_string(lastSync_) << " - ledger sequence is "
<< std::to_string(ledgerSequence_);
log_.info() << "Skipping commit. sync interval is " << std::to_string(syncInterval_) << " - last sync is "
<< std::to_string(lastSync_) << " - ledger sequence is " << std::to_string(ledgerSequence_);
}
return true;
}
@@ -842,8 +794,7 @@ public:
return doFinishWritesAsync();
}
void
writeLedger(ripple::LedgerInfo const& ledgerInfo, std::string&& header)
override;
writeLedger(ripple::LedgerInfo const& ledgerInfo, std::string&& header) override;
std::optional<std::uint32_t>
fetchLatestLedgerSequence(boost::asio::yield_context& yield) const override
@@ -853,17 +804,14 @@ public:
CassandraResult result = executeAsyncRead(statement, yield);
if (!result.hasResult())
{
log_.error()
<< "CassandraBackend::fetchLatestLedgerSequence - no rows";
log_.error() << "CassandraBackend::fetchLatestLedgerSequence - no rows";
return {};
}
return result.getUInt32();
}
std::optional<ripple::LedgerInfo>
fetchLedgerBySequence(
std::uint32_t const sequence,
boost::asio::yield_context& yield) const override
fetchLedgerBySequence(std::uint32_t const sequence, boost::asio::yield_context& yield) const override
{
log_.trace() << "called";
CassandraStatement statement{selectLedgerBySeq_};
@@ -879,9 +827,7 @@ public:
}
std::optional<ripple::LedgerInfo>
fetchLedgerByHash(
ripple::uint256 const& hash,
boost::asio::yield_context& yield) const override
fetchLedgerByHash(ripple::uint256 const& hash, boost::asio::yield_context& yield) const override
{
CassandraStatement statement{selectLedgerByHash_};
@@ -904,20 +850,15 @@ public:
hardFetchLedgerRange(boost::asio::yield_context& yield) const override;
std::vector<TransactionAndMetadata>
fetchAllTransactionsInLedger(
std::uint32_t const ledgerSequence,
boost::asio::yield_context& yield) const override;
fetchAllTransactionsInLedger(std::uint32_t const ledgerSequence, boost::asio::yield_context& yield) const override;
std::vector<ripple::uint256>
fetchAllTransactionHashesInLedger(
std::uint32_t const ledgerSequence,
boost::asio::yield_context& yield) const override;
fetchAllTransactionHashesInLedger(std::uint32_t const ledgerSequence, boost::asio::yield_context& yield)
const override;
std::optional<NFT>
fetchNFT(
ripple::uint256 const& tokenID,
std::uint32_t const ledgerSequence,
boost::asio::yield_context& yield) const override;
fetchNFT(ripple::uint256 const& tokenID, std::uint32_t const ledgerSequence, boost::asio::yield_context& yield)
const override;
TransactionsAndCursor
fetchNFTTransactions(
@@ -930,15 +871,11 @@ public:
// Synchronously fetch the object with key key, as of ledger with sequence
// sequence
std::optional<Blob>
doFetchLedgerObject(
ripple::uint256 const& key,
std::uint32_t const sequence,
boost::asio::yield_context& yield) const override;
doFetchLedgerObject(ripple::uint256 const& key, std::uint32_t const sequence, boost::asio::yield_context& yield)
const override;
std::optional<TransactionAndMetadata>
fetchTransaction(
ripple::uint256 const& hash,
boost::asio::yield_context& yield) const override
fetchTransaction(ripple::uint256 const& hash, boost::asio::yield_context& yield) const override
{
log_.trace() << "called";
CassandraStatement statement{selectTransaction_};
@@ -950,23 +887,15 @@ public:
log_.error() << "No rows";
return {};
}
return {
{result.getBytes(),
result.getBytes(),
result.getUInt32(),
result.getUInt32()}};
return {{result.getBytes(), result.getBytes(), result.getUInt32(), result.getUInt32()}};
}
std::optional<ripple::uint256>
doFetchSuccessorKey(
ripple::uint256 key,
std::uint32_t const ledgerSequence,
boost::asio::yield_context& yield) const override;
doFetchSuccessorKey(ripple::uint256 key, std::uint32_t const ledgerSequence, boost::asio::yield_context& yield)
const override;
std::vector<TransactionAndMetadata>
fetchTransactions(
std::vector<ripple::uint256> const& hashes,
boost::asio::yield_context& yield) const override;
fetchTransactions(std::vector<ripple::uint256> const& hashes, boost::asio::yield_context& yield) const override;
std::vector<Blob>
doFetchLedgerObjects(
@@ -975,25 +904,16 @@ public:
boost::asio::yield_context& yield) const override;
std::vector<LedgerObject>
fetchLedgerDiff(
std::uint32_t const ledgerSequence,
boost::asio::yield_context& yield) const override;
fetchLedgerDiff(std::uint32_t const ledgerSequence, boost::asio::yield_context& yield) const override;
void
doWriteLedgerObject(
std::string&& key,
std::uint32_t const seq,
std::string&& blob) override;
doWriteLedgerObject(std::string&& key, std::uint32_t const seq, std::string&& blob) override;
void
writeSuccessor(
std::string&& key,
std::uint32_t const seq,
std::string&& successor) override;
writeSuccessor(std::string&& key, std::uint32_t const seq, std::string&& successor) override;
void
writeAccountTransactions(
std::vector<AccountTransactionsData>&& data) override;
writeAccountTransactions(std::vector<AccountTransactionsData>&& data) override;
void
writeNFTTransactions(std::vector<NFTTransactionsData>&& data) override;
@@ -1023,9 +943,7 @@ public:
}
bool
doOnlineDelete(
std::uint32_t const numLedgersToKeep,
boost::asio::yield_context& yield) const override;
doOnlineDelete(std::uint32_t const numLedgersToKeep, boost::asio::yield_context& yield) const override;
bool
isTooBusy() const override;
@@ -1090,26 +1008,18 @@ public:
template <class T, class S>
void
executeAsyncHelper(
CassandraStatement const& statement,
T callback,
S& callbackData) const
executeAsyncHelper(CassandraStatement const& statement, T callback, S& callbackData) const
{
CassFuture* fut = cass_session_execute(session_.get(), statement.get());
cass_future_set_callback(
fut, callback, static_cast<void*>(&callbackData));
cass_future_set_callback(fut, callback, static_cast<void*>(&callbackData));
cass_future_free(fut);
}
template <class T, class S>
void
executeAsyncWrite(
CassandraStatement const& statement,
T callback,
S& callbackData,
bool isRetry) const
executeAsyncWrite(CassandraStatement const& statement, T callback, S& callbackData, bool isRetry) const
{
if (!isRetry)
incrementOutstandingRequestCount();
@@ -1118,10 +1028,7 @@ public:
template <class T, class S>
void
executeAsyncRead(
CassandraStatement const& statement,
T callback,
S& callbackData) const
executeAsyncRead(CassandraStatement const& statement, T callback, S& callbackData) const
{
executeAsyncHelper(statement, callback, callbackData);
}
@@ -1184,8 +1091,7 @@ public:
if (rc != CASS_OK)
{
cass_result_free(res);
log_.error() << "executeSyncUpdate - error getting result " << rc
<< ", " << cass_error_desc(rc);
log_.error() << "executeSyncUpdate - error getting result " << rc << ", " << cass_error_desc(rc);
return false;
}
cass_result_free(res);
@@ -1206,13 +1112,10 @@ public:
}
CassandraResult
executeAsyncRead(
CassandraStatement const& statement,
boost::asio::yield_context& yield) const
executeAsyncRead(CassandraStatement const& statement, boost::asio::yield_context& yield) const
{
using result = boost::asio::async_result<
boost::asio::yield_context,
void(boost::system::error_code, CassError)>;
using result =
boost::asio::async_result<boost::asio::yield_context, void(boost::system::error_code, CassError)>;
CassFuture* fut;
CassError rc;

343
src/backend/CassandraBackendNew.h
View File

@@ -42,9 +42,7 @@ namespace Backend::Cassandra {
* Eventually we should change the interface so that it does not have to know
* about yield_context.
*/
template <
SomeSettingsProvider SettingsProviderType,
SomeExecutionStrategy ExecutionStrategy>
template <SomeSettingsProvider SettingsProviderType, SomeExecutionStrategy ExecutionStrategy>
class BasicCassandraBackend : public BackendInterface
{
clio::Logger log_{"Backend"};
@@ -71,11 +69,9 @@ public:
, executor_{settingsProvider_.getSettings(), handle_}
{
if (auto const res = handle_.connect(); not res)
throw std::runtime_error(
"Could not connect to Cassandra: " + res.error());
throw std::runtime_error("Could not connect to Cassandra: " + res.error());
if (auto const res = handle_.execute(schema_.createKeyspace); not res)
throw std::runtime_error(
"Could not create keyspace: " + res.error());
throw std::runtime_error("Could not create keyspace: " + res.error());
if (auto const res = handle_.executeEach(schema_.createSchema); not res)
throw std::runtime_error("Could not create schema: " + res.error());
@@ -118,19 +114,16 @@ public:
if (cursor)
{
statement.bindAt(1, cursor->asTuple());
log_.debug() << "account = " << ripple::strHex(account)
<< " tuple = " << cursor->ledgerSequence
log_.debug() << "account = " << ripple::strHex(account) << " tuple = " << cursor->ledgerSequence
<< cursor->transactionIndex;
}
else
{
auto const seq = forward ? rng->minSequence : rng->maxSequence;
auto const placeHolder =
forward ? 0u : std::numeric_limits<std::uint32_t>::max();
auto const placeHolder = forward ? 0u : std::numeric_limits<std::uint32_t>::max();
statement.bindAt(1, std::make_tuple(placeHolder, placeHolder));
log_.debug() << "account = " << ripple::strHex(account)
<< " idx = " << seq << " tuple = " << placeHolder;
log_.debug() << "account = " << ripple::strHex(account) << " idx = " << seq << " tuple = " << placeHolder;
}
// FIXME: Limit is a hack to support uint32_t properly for the time
@@ -149,8 +142,7 @@ public:
auto numRows = results.numRows();
log_.info() << "num_rows = " << numRows;
for (auto [hash, data] :
extract<ripple::uint256, std::tuple<uint32_t, uint32_t>>(results))
for (auto [hash, data] : extract<ripple::uint256, std::tuple<uint32_t, uint32_t>>(results))
{
hashes.push_back(hash);
if (--numRows == 0)
@@ -185,15 +177,10 @@ public:
if (!range)
{
executor_.writeSync(
schema_->updateLedgerRange,
ledgerSequence_,
false,
ledgerSequence_);
executor_.writeSync(schema_->updateLedgerRange, ledgerSequence_, false, ledgerSequence_);
}
if (not executeSyncUpdate(schema_->updateLedgerRange.bind(
ledgerSequence_, true, ledgerSequence_ - 1)))
if (not executeSyncUpdate(schema_->updateLedgerRange.bind(ledgerSequence_, true, ledgerSequence_ - 1)))
{
log_.warn() << "Update failed for ledger " << ledgerSequence_;
return false;
@@ -204,14 +191,11 @@ public:
}
void
writeLedger(ripple::LedgerInfo const& ledgerInfo, std::string&& header)
override
writeLedger(ripple::LedgerInfo const& ledgerInfo, std::string&& header) override
{
executor_.write(
schema_->insertLedgerHeader, ledgerInfo.seq, std::move(header));
executor_.write(schema_->insertLedgerHeader, ledgerInfo.seq, std::move(header));
executor_.write(
schema_->insertLedgerHash, ledgerInfo.hash, ledgerInfo.seq);
executor_.write(schema_->insertLedgerHash, ledgerInfo.hash, ledgerInfo.seq);
ledgerSequence_ = ledgerInfo.seq;
}
@@ -219,13 +203,11 @@ public:
std::optional<std::uint32_t>
fetchLatestLedgerSequence(boost::asio::yield_context& yield) const override
{
if (auto const res = executor_.read(yield, schema_->selectLatestLedger);
res)
if (auto const res = executor_.read(yield, schema_->selectLatestLedger); res)
{
if (auto const& result = res.value(); result)
{
if (auto const maybeValue = result.template get<uint32_t>();
maybeValue)
if (auto const maybeValue = result.template get<uint32_t>(); maybeValue)
return maybeValue;
log_.error() << "Could not fetch latest ledger - no rows";
@@ -243,21 +225,16 @@ public:
}
std::optional<ripple::LedgerInfo>
fetchLedgerBySequence(
std::uint32_t const sequence,
boost::asio::yield_context& yield) const override
fetchLedgerBySequence(std::uint32_t const sequence, boost::asio::yield_context& yield) const override
{
log_.trace() << __func__ << " call for seq " << sequence;
auto const res =
executor_.read(yield, schema_->selectLedgerBySeq, sequence);
auto const res = executor_.read(yield, schema_->selectLedgerBySeq, sequence);
if (res)
{
if (auto const& result = res.value(); result)
{
if (auto const maybeValue =
result.template get<std::vector<unsigned char>>();
maybeValue)
if (auto const maybeValue = result.template get<std::vector<unsigned char>>(); maybeValue)
{
return deserializeHeader(ripple::makeSlice(*maybeValue));
}
@@ -270,28 +247,22 @@ public:
}
else
{
log_.error() << "Could not fetch ledger by sequence: "
<< res.error();
log_.error() << "Could not fetch ledger by sequence: " << res.error();
}
return std::nullopt;
}
std::optional<ripple::LedgerInfo>
fetchLedgerByHash(
ripple::uint256 const& hash,
boost::asio::yield_context& yield) const override
fetchLedgerByHash(ripple::uint256 const& hash, boost::asio::yield_context& yield) const override
{
log_.trace() << __func__ << " call";
if (auto const res =
executor_.read(yield, schema_->selectLedgerByHash, hash);
res)
if (auto const res = executor_.read(yield, schema_->selectLedgerByHash, hash); res)
{
if (auto const& result = res.value(); result)
{
if (auto const maybeValue = result.template get<uint32_t>();
maybeValue)
if (auto const maybeValue = result.template get<uint32_t>(); maybeValue)
return fetchLedgerBySequence(*maybeValue, yield);
log_.error() << "Could not fetch ledger by hash - no rows";
@@ -313,8 +284,7 @@ public:
{
log_.trace() << __func__ << " call";
if (auto const res = executor_.read(yield, schema_->selectLedgerRange);
res)
if (auto const res = executor_.read(yield, schema_->selectLedgerRange); res)
{
auto const& results = res.value();
if (not results.hasRows())
@@ -341,8 +311,7 @@ public:
if (range.minSequence > range.maxSequence)
std::swap(range.minSequence, range.maxSequence);
log_.debug() << "After hardFetchLedgerRange range is "
<< range.minSequence << ":" << range.maxSequence;
log_.debug() << "After hardFetchLedgerRange range is " << range.minSequence << ":" << range.maxSequence;
return range;
}
else
@@ -354,9 +323,7 @@ public:
}
std::vector<TransactionAndMetadata>
fetchAllTransactionsInLedger(
std::uint32_t const ledgerSequence,
boost::asio::yield_context& yield) const override
fetchAllTransactionsInLedger(std::uint32_t const ledgerSequence, boost::asio::yield_context& yield) const override
{
log_.trace() << __func__ << " call";
auto hashes = fetchAllTransactionHashesInLedger(ledgerSequence, yield);
@@ -364,28 +331,24 @@ public:
}
std::vector<ripple::uint256>
fetchAllTransactionHashesInLedger(
std::uint32_t const ledgerSequence,
boost::asio::yield_context& yield) const override
fetchAllTransactionHashesInLedger(std::uint32_t const ledgerSequence, boost::asio::yield_context& yield)
const override
{
log_.trace() << __func__ << " call";
auto start = std::chrono::system_clock::now();
auto const res = executor_.read(
yield, schema_->selectAllTransactionHashesInLedger, ledgerSequence);
auto const res = executor_.read(yield, schema_->selectAllTransactionHashesInLedger, ledgerSequence);
if (not res)
{
log_.error() << "Could not fetch all transaction hashes: "
<< res.error();
log_.error() << "Could not fetch all transaction hashes: " << res.error();
return {};
}
auto const& result = res.value();
if (not result.hasRows())
{
log_.error()
<< "Could not fetch all transaction hashes - no rows; ledger = "
<< std::to_string(ledgerSequence);
log_.error() << "Could not fetch all transaction hashes - no rows; ledger = "
<< std::to_string(ledgerSequence);
return {};
}
@@ -394,36 +357,26 @@ public:
hashes.push_back(std::move(hash));
auto end = std::chrono::system_clock::now();
log_.debug() << "Fetched " << hashes.size()
<< " transaction hashes from Cassandra in "
<< std::chrono::duration_cast<std::chrono::milliseconds>(
end - start)
.count()
<< " milliseconds";
log_.debug() << "Fetched " << hashes.size() << " transaction hashes from Cassandra in "
<< std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count() << " milliseconds";
return hashes;
}
std::optional<NFT>
fetchNFT(
ripple::uint256 const& tokenID,
std::uint32_t const ledgerSequence,
boost::asio::yield_context& yield) const override
fetchNFT(ripple::uint256 const& tokenID, std::uint32_t const ledgerSequence, boost::asio::yield_context& yield)
const override
{
log_.trace() << __func__ << " call";
auto const res =
executor_.read(yield, schema_->selectNFT, tokenID, ledgerSequence);
auto const res = executor_.read(yield, schema_->selectNFT, tokenID, ledgerSequence);
if (not res)
return std::nullopt;
if (auto const maybeRow =
res->template get<uint32_t, ripple::AccountID, bool>();
maybeRow)
if (auto const maybeRow = res->template get<uint32_t, ripple::AccountID, bool>(); maybeRow)
{
auto [seq, owner, isBurned] = *maybeRow;
auto result =
std::make_optional<NFT>(tokenID, seq, owner, isBurned);
auto result = std::make_optional<NFT>(tokenID, seq, owner, isBurned);
// now fetch URI. Usually we will have the URI even for burned NFTs,
// but if the first ledger on this clio included NFTokenBurn
@@ -436,12 +389,10 @@ public:
// a URI because it was burned in the first ledger) to indicate that
// even though we are returning a blank URI, the NFT might have had
// one.
auto uriRes = executor_.read(
yield, schema_->selectNFTURI, tokenID, ledgerSequence);
auto uriRes = executor_.read(yield, schema_->selectNFTURI, tokenID, ledgerSequence);
if (uriRes)
{
if (auto const maybeUri = uriRes->template get<ripple::Blob>();
maybeUri)
if (auto const maybeUri = uriRes->template get<ripple::Blob>(); maybeUri)
result->uri = *maybeUri;
}
@@ -477,19 +428,16 @@ public:
if (cursor)
{
statement.bindAt(1, cursor->asTuple());
log_.debug() << "token_id = " << ripple::strHex(tokenID)
<< " tuple = " << cursor->ledgerSequence
log_.debug() << "token_id = " << ripple::strHex(tokenID) << " tuple = " << cursor->ledgerSequence
<< cursor->transactionIndex;
}
else
{
auto const seq = forward ? rng->minSequence : rng->maxSequence;
auto const placeHolder =
forward ? 0 : std::numeric_limits<std::uint32_t>::max();
auto const placeHolder = forward ? 0 : std::numeric_limits<std::uint32_t>::max();
statement.bindAt(1, std::make_tuple(placeHolder, placeHolder));
log_.debug() << "token_id = " << ripple::strHex(tokenID)
<< " idx = " << seq << " tuple = " << placeHolder;
log_.debug() << "token_id = " << ripple::strHex(tokenID) << " idx = " << seq << " tuple = " << placeHolder;
}
statement.bindAt(2, Limit{limit});
@@ -506,8 +454,7 @@ public:
auto numRows = results.numRows();
log_.info() << "num_rows = " << numRows;
for (auto [hash, data] :
extract<ripple::uint256, std::tuple<uint32_t, uint32_t>>(results))
for (auto [hash, data] : extract<ripple::uint256, std::tuple<uint32_t, uint32_t>>(results))
{
hashes.push_back(hash);
if (--numRows == 0)
@@ -535,16 +482,11 @@ public:
}
std::optional<Blob>
doFetchLedgerObject(
ripple::uint256 const& key,
std::uint32_t const sequence,
boost::asio::yield_context& yield) const override
doFetchLedgerObject(ripple::uint256 const& key, std::uint32_t const sequence, boost::asio::yield_context& yield)
const override
{
log_.debug() << "Fetching ledger object for seq " << sequence
<< ", key = " << ripple::to_string(key);
if (auto const res =
executor_.read(yield, schema_->selectObject, key, sequence);
res)
log_.debug() << "Fetching ledger object for seq " << sequence << ", key = " << ripple::to_string(key);
if (auto const res = executor_.read(yield, schema_->selectObject, key, sequence); res)
{
if (auto const result = res->template get<Blob>(); result)
{
@@ -565,23 +507,16 @@ public:
}
std::optional<TransactionAndMetadata>
fetchTransaction(
ripple::uint256 const& hash,
boost::asio::yield_context& yield) const override
fetchTransaction(ripple::uint256 const& hash, boost::asio::yield_context& yield) const override
{
log_.trace() << __func__ << " call";
if (auto const res =
executor_.read(yield, schema_->selectTransaction, hash);
res)
if (auto const res = executor_.read(yield, schema_->selectTransaction, hash); res)
{
if (auto const maybeValue =
res->template get<Blob, Blob, uint32_t, uint32_t>();
maybeValue)
if (auto const maybeValue = res->template get<Blob, Blob, uint32_t, uint32_t>(); maybeValue)
{
auto [transaction, meta, seq, date] = *maybeValue;
return std::make_optional<TransactionAndMetadata>(
transaction, meta, seq, date);
return std::make_optional<TransactionAndMetadata>(transaction, meta, seq, date);
}
else
{
@@ -597,19 +532,14 @@ public:
}
std::optional<ripple::uint256>
doFetchSuccessorKey(
ripple::uint256 key,
std::uint32_t const ledgerSequence,
boost::asio::yield_context& yield) const override
doFetchSuccessorKey(ripple::uint256 key, std::uint32_t const ledgerSequence, boost::asio::yield_context& yield)
const override
{
log_.trace() << __func__ << " call";
if (auto const res = executor_.read(
yield, schema_->selectSuccessor, key, ledgerSequence);
res)
if (auto const res = executor_.read(yield, schema_->selectSuccessor, key, ledgerSequence); res)
{
if (auto const result = res->template get<ripple::uint256>();
result)
if (auto const result = res->template get<ripple::uint256>(); result)
{
if (*result == lastKey)
return std::nullopt;
@@ -629,9 +559,7 @@ public:
}
std::vector<TransactionAndMetadata>
fetchTransactions(
std::vector<ripple::uint256> const& hashes,
boost::asio::yield_context& yield) const override
fetchTransactions(std::vector<ripple::uint256> const& hashes, boost::asio::yield_context& yield) const override
{
log_.trace() << __func__ << " call";
@@ -645,17 +573,10 @@ public:
std::vector<Statement> statements;
statements.reserve(numHashes);
auto const timeDiff = util::timed([this,
&yield,
&results,
&hashes,
&statements]() {
auto const timeDiff = util::timed([this, &yield, &results, &hashes, &statements]() {
// TODO: seems like a job for "hash IN (list of hashes)" instead?
std::transform(
std::cbegin(hashes),
std::cend(hashes),
std::back_inserter(statements),
[this](auto const& hash) {
std::cbegin(hashes), std::cend(hashes), std::back_inserter(statements), [this](auto const& hash) {
return schema_->selectTransaction.bind(hash);
});
@@ -665,9 +586,7 @@ public:
std::cend(entries),
std::back_inserter(results),
[](auto const& res) -> TransactionAndMetadata {
if (auto const maybeRow =
res.template get<Blob, Blob, uint32_t, uint32_t>();
maybeRow)
if (auto const maybeRow = res.template get<Blob, Blob, uint32_t, uint32_t>(); maybeRow)
return *maybeRow;
else
return {};
@@ -675,9 +594,7 @@ public:
});
assert(numHashes == results.size());
log_.debug() << "Fetched " << numHashes
<< " transactions from Cassandra in " << timeDiff
<< " milliseconds";
log_.debug() << "Fetched " << numHashes << " transactions from Cassandra in " << timeDiff << " milliseconds";
return results;
}
@@ -703,21 +620,14 @@ public:
// TODO: seems like a job for "key IN (list of keys)" instead?
std::transform(
std::cbegin(keys),
std::cend(keys),
std::back_inserter(statements),
[this, &sequence](auto const& key) {
std::cbegin(keys), std::cend(keys), std::back_inserter(statements), [this, &sequence](auto const& key) {
return schema_->selectObject.bind(key, sequence);
});
auto const entries = executor_.readEach(yield, statements);
std::transform(
std::cbegin(entries),
std::cend(entries),
std::back_inserter(results),
[](auto const& res) -> Blob {
if (auto const maybeValue = res.template get<Blob>();
maybeValue)
std::cbegin(entries), std::cend(entries), std::back_inserter(results), [](auto const& res) -> Blob {
if (auto const maybeValue = res.template get<Blob>(); maybeValue)
return *maybeValue;
else
return {};
@@ -728,47 +638,37 @@ public:
}
std::vector<LedgerObject>
fetchLedgerDiff(
std::uint32_t const ledgerSequence,
boost::asio::yield_context& yield) const override
fetchLedgerDiff(std::uint32_t const ledgerSequence, boost::asio::yield_context& yield) const override
{
log_.trace() << __func__ << " call";
auto const [keys, timeDiff] = util::timed(
[this, &ledgerSequence, &yield]() -> std::vector<ripple::uint256> {
auto const res =
executor_.read(yield, schema_->selectDiff, ledgerSequence);
if (not res)
{
log_.error()
<< "Could not fetch ledger diff: " << res.error()
<< "; ledger = " << ledgerSequence;
return {};
}
auto const [keys, timeDiff] = util::timed([this, &ledgerSequence, &yield]() -> std::vector<ripple::uint256> {
auto const res = executor_.read(yield, schema_->selectDiff, ledgerSequence);
if (not res)
{
log_.error() << "Could not fetch ledger diff: " << res.error() << "; ledger = " << ledgerSequence;
return {};
}
auto const& results = res.value();
if (not results)
{
log_.error()
<< "Could not fetch ledger diff - no rows; ledger = "
<< ledgerSequence;
return {};
}
auto const& results = res.value();
if (not results)
{
log_.error() << "Could not fetch ledger diff - no rows; ledger = " << ledgerSequence;
return {};
}
std::vector<ripple::uint256> keys;
for (auto [key] : extract<ripple::uint256>(results))
keys.push_back(key);
std::vector<ripple::uint256> keys;
for (auto [key] : extract<ripple::uint256>(results))
keys.push_back(key);
return keys;
});
return keys;
});
// one of the above errors must have happened
if (keys.empty())
return {};
log_.debug() << "Fetched " << keys.size()
<< " diff hashes from Cassandra in " << timeDiff
<< " milliseconds";
log_.debug() << "Fetched " << keys.size() << " diff hashes from Cassandra in " << timeDiff << " milliseconds";
auto const objs = fetchLedgerObjects(keys, ledgerSequence, yield);
std::vector<LedgerObject> results;
@@ -787,43 +687,29 @@ public:
}
void
doWriteLedgerObject(
std::string&& key,
std::uint32_t const seq,
std::string&& blob) override
doWriteLedgerObject(std::string&& key, std::uint32_t const seq, std::string&& blob) override
{
log_.trace() << " Writing ledger object " << key.size() << ":" << seq
<< " [" << blob.size() << " bytes]";
log_.trace() << " Writing ledger object " << key.size() << ":" << seq << " [" << blob.size() << " bytes]";
if (range)
executor_.write(schema_->insertDiff, seq, key);
executor_.write(
schema_->insertObject, std::move(key), seq, std::move(blob));
executor_.write(schema_->insertObject, std::move(key), seq, std::move(blob));
}
void
writeSuccessor(
std::string&& key,
std::uint32_t const seq,
std::string&& successor) override
writeSuccessor(std::string&& key, std::uint32_t const seq, std::string&& successor) override
{
log_.trace() << "Writing successor. key = " << key.size() << " bytes. "
<< " seq = " << std::to_string(seq)
<< " successor = " << successor.size() << " bytes.";
<< " seq = " << std::to_string(seq) << " successor = " << successor.size() << " bytes.";
assert(key.size() != 0);
assert(successor.size() != 0);
executor_.write(
schema_->insertSuccessor,
std::move(key),
seq,
std::move(successor));
executor_.write(schema_->insertSuccessor, std::move(key), seq, std::move(successor));
}
void
writeAccountTransactions(
std::vector<AccountTransactionsData>&& data) override
writeAccountTransactions(std::vector<AccountTransactionsData>&& data) override
{
std::vector<Statement> statements;
statements.reserve(data.size() * 10); // assume 10 transactions avg
@@ -837,8 +723,7 @@ public:
[this, &record](auto&& account) {
return schema_->insertAccountTx.bind(
std::move(account),
std::make_tuple(
record.ledgerSequence, record.transactionIndex),
std::make_tuple(record.ledgerSequence, record.transactionIndex),
record.txHash);
});
}
@@ -852,17 +737,10 @@ public:
std::vector<Statement> statements;
statements.reserve(data.size());
std::transform(
std::cbegin(data),
std::cend(data),
std::back_inserter(statements),
[this](auto const& record) {
return schema_->insertNFTTx.bind(
record.tokenID,
std::make_tuple(
record.ledgerSequence, record.transactionIndex),
record.txHash);
});
std::transform(std::cbegin(data), std::cend(data), std::back_inserter(statements), [this](auto const& record) {
return schema_->insertNFTTx.bind(
record.tokenID, std::make_tuple(record.ledgerSequence, record.transactionIndex), record.txHash);
});
executor_.write(statements);
}
@@ -879,12 +757,7 @@ public:
executor_.write(schema_->insertLedgerTransaction, seq, hash);
executor_.write(
schema_->insertTransaction,
std::move(hash),
seq,
date,
std::move(transaction),
std::move(metadata));
schema_->insertTransaction, std::move(hash), seq, date, std::move(transaction), std::move(metadata));
}
void
@@ -895,11 +768,8 @@ public:
for (NFTsData const& record : data)
{
statements.push_back(schema_->insertNFT.bind(
record.tokenID,
record.ledgerSequence,
record.owner,
record.isBurned));
statements.push_back(
schema_->insertNFT.bind(record.tokenID, record.ledgerSequence, record.owner, record.isBurned));
// If `uri` is set (and it can be set to an empty uri), we know this
// is a net-new NFT. That is, this NFT has not been seen before by
@@ -910,11 +780,10 @@ public:
{
statements.push_back(schema_->insertIssuerNFT.bind(
ripple::nft::getIssuer(record.tokenID),
static_cast<uint32_t>(
ripple::nft::getTaxon(record.tokenID)),
static_cast<uint32_t>(ripple::nft::getTaxon(record.tokenID)),
record.tokenID));
statements.push_back(schema_->insertNFTURI.bind(
record.tokenID, record.ledgerSequence, record.uri.value()));
statements.push_back(
schema_->insertNFTURI.bind(record.tokenID, record.ledgerSequence, record.uri.value()));
}
}
@@ -930,9 +799,7 @@ public:
/*! Unused in this implementation */
bool
doOnlineDelete(
std::uint32_t const numLedgersToKeep,
boost::asio::yield_context& yield) const override
doOnlineDelete(std::uint32_t const numLedgersToKeep, boost::asio::yield_context& yield) const override
{
log_.trace() << __func__ << " call";
return true;
@@ -958,8 +825,7 @@ private:
if (not maybeSuccess.value())
{
log_.warn()
<< "Update failed. Checking if DB state is what we expect";
log_.warn() << "Update failed. Checking if DB state is what we expect";
// error may indicate that another writer wrote something.
// in this case let's just compare the current state of things
@@ -973,7 +839,6 @@ private:
}
};
using CassandraBackend =
BasicCassandraBackend<SettingsProvider, detail::DefaultExecutionStrategy<>>;
using CassandraBackend = BasicCassandraBackend<SettingsProvider, detail::DefaultExecutionStrategy<>>;
} // namespace Backend::Cassandra

41
src/backend/DBHelpers.h
View File

@@ -39,10 +39,7 @@ struct AccountTransactionsData
std::uint32_t transactionIndex;
ripple::uint256 txHash;
AccountTransactionsData(
ripple::TxMeta& meta,
ripple::uint256 const& txHash,
beast::Journal& j)
AccountTransactionsData(ripple::TxMeta& meta, ripple::uint256 const& txHash, beast::Journal& j)
: accounts(meta.getAffectedAccounts())
, ledgerSequence(meta.getLgrSeq())
, transactionIndex(meta.getIndex())
@@ -62,14 +59,8 @@ struct NFTTransactionsData
std::uint32_t transactionIndex;
ripple::uint256 txHash;
NFTTransactionsData(
ripple::uint256 const& tokenID,
ripple::TxMeta const& meta,
ripple::uint256 const& txHash)
: tokenID(tokenID)
, ledgerSequence(meta.getLgrSeq())
, transactionIndex(meta.getIndex())
, txHash(txHash)
NFTTransactionsData(ripple::uint256 const& tokenID, ripple::TxMeta const& meta, ripple::uint256 const& txHash)
: tokenID(tokenID), ledgerSequence(meta.getLgrSeq()), transactionIndex(meta.getIndex()), txHash(txHash)
{
}
};
@@ -110,21 +101,13 @@ struct NFTsData
ripple::AccountID const& owner,
ripple::Blob const& uri,
ripple::TxMeta const& meta)
: tokenID(tokenID)
, ledgerSequence(meta.getLgrSeq())
, transactionIndex(meta.getIndex())
, owner(owner)
, uri(uri)
: tokenID(tokenID), ledgerSequence(meta.getLgrSeq()), transactionIndex(meta.getIndex()), owner(owner), uri(uri)
{
}
// This constructor is used when parsing an NFTokenBurn or
// NFTokenAcceptOffer tx
NFTsData(
ripple::uint256 const& tokenID,
ripple::AccountID const& owner,
ripple::TxMeta const& meta,
bool isBurned)
NFTsData(ripple::uint256 const& tokenID, ripple::AccountID const& owner, ripple::TxMeta const& meta, bool isBurned)
: tokenID(tokenID)
, ledgerSequence(meta.getLgrSeq())
, transactionIndex(meta.getIndex())
@@ -144,10 +127,7 @@ struct NFTsData
std::uint32_t const ledgerSequence,
ripple::AccountID const& owner,
ripple::Blob const& uri)
: tokenID(tokenID)
, ledgerSequence(ledgerSequence)
, owner(owner)
, uri(uri)
: tokenID(tokenID), ledgerSequence(ledgerSequence), owner(owner), uri(uri)
{
}
};
@@ -188,8 +168,7 @@ isBookDir(T const& key, R const& object)
if (!isDirNode(object))
return false;
ripple::STLedgerEntry const sle{
ripple::SerialIter{object.data(), object.size()}, key};
ripple::STLedgerEntry const sle{ripple::SerialIter{object.data(), object.size()}, key};
return !sle[~ripple::sfOwner].has_value();
}
@@ -228,10 +207,8 @@ deserializeHeader(ripple::Slice data)
info.parentHash = sit.get256();
info.txHash = sit.get256();
info.accountHash = sit.get256();
info.parentCloseTime =
ripple::NetClock::time_point{ripple::NetClock::duration{sit.get32()}};
info.closeTime =
ripple::NetClock::time_point{ripple::NetClock::duration{sit.get32()}};
info.parentCloseTime = ripple::NetClock::time_point{ripple::NetClock::duration{sit.get32()}};
info.closeTime = ripple::NetClock::time_point{ripple::NetClock::duration{sit.get32()}};
info.closeTimeResolution = ripple::NetClock::duration{sit.get8()};
info.closeFlags = sit.get8();

5
src/backend/SimpleCache.cpp
View File

@@ -28,10 +28,7 @@ SimpleCache::latestLedgerSequence() const
}
void
SimpleCache::update(
std::vector<LedgerObject> const& objs,
uint32_t seq,
bool isBackground)
SimpleCache::update(std::vector<LedgerObject> const& objs, uint32_t seq, bool isBackground)
{
if (disabled_)
return;

5
src/backend/SimpleCache.h
View File

@@ -56,10 +56,7 @@ public:
// Update the cache with new ledger objects
// set isBackground to true when writing old data from a background thread
void
update(
std::vector<LedgerObject> const& blobs,
uint32_t seq,
bool isBackground = false);
update(std::vector<LedgerObject> const& blobs, uint32_t seq, bool isBackground = false);
std::optional<Blob>
get(ripple::uint256 const& key, uint32_t seq) const;

35
src/backend/Types.h
View File

@@ -65,15 +65,11 @@ struct TransactionAndMetadata
Blob const& metadata,
std::uint32_t ledgerSequence,
std::uint32_t date)
: transaction{transaction}
, metadata{metadata}
, ledgerSequence{ledgerSequence}
, date{date}
: transaction{transaction}, metadata{metadata}, ledgerSequence{ledgerSequence}, date{date}
{
}
TransactionAndMetadata(
std::tuple<Blob, Blob, std::uint32_t, std::uint32_t> data)
TransactionAndMetadata(std::tuple<Blob, Blob, std::uint32_t, std::uint32_t> data)
: transaction{std::get<0>(data)}
, metadata{std::get<1>(data)}
, ledgerSequence{std::get<2>(data)}
@@ -95,9 +91,7 @@ struct TransactionsCursor
std::uint32_t transactionIndex;
TransactionsCursor() = default;
TransactionsCursor(
std::uint32_t ledgerSequence,
std::uint32_t transactionIndex)
TransactionsCursor(std::uint32_t ledgerSequence, std::uint32_t transactionIndex)
: ledgerSequence{ledgerSequence}, transactionIndex{transactionIndex}
{
}
@@ -140,18 +134,11 @@ struct NFT
ripple::AccountID const& owner,
Blob const& uri,
bool isBurned)
: tokenID{tokenID}
, ledgerSequence{ledgerSequence}
, owner{owner}
, uri{uri}
, isBurned{isBurned}
: tokenID{tokenID}, ledgerSequence{ledgerSequence}, owner{owner}, uri{uri}, isBurned{isBurned}
{
}
NFT(ripple::uint256 const& tokenID,
std::uint32_t ledgerSequence,
ripple::AccountID const& owner,
bool isBurned)
NFT(ripple::uint256 const& tokenID, std::uint32_t ledgerSequence, ripple::AccountID const& owner, bool isBurned)
: NFT(tokenID, ledgerSequence, owner, {}, isBurned)
{
}
@@ -162,8 +149,7 @@ struct NFT
bool
operator==(NFT const& other) const
{
return tokenID == other.tokenID &&
ledgerSequence == other.ledgerSequence;
return tokenID == other.tokenID && ledgerSequence == other.ledgerSequence;
}
};
@@ -172,10 +158,7 @@ struct LedgerRange
std::uint32_t minSequence;
std::uint32_t maxSequence;
};
constexpr ripple::uint256 firstKey{
"0000000000000000000000000000000000000000000000000000000000000000"};
constexpr ripple::uint256 lastKey{
"FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"};
constexpr ripple::uint256 hi192{
"0000000000000000000000000000000000000000000000001111111111111111"};
constexpr ripple::uint256 firstKey{"0000000000000000000000000000000000000000000000000000000000000000"};
constexpr ripple::uint256 lastKey{"FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"};
constexpr ripple::uint256 hi192{"0000000000000000000000000000000000000000000000001111111111111111"};
} // namespace Backend

19
src/backend/cassandra/Error.h
View File

@@ -35,27 +35,20 @@ class CassandraError
public:
CassandraError() = default; // default constructible required by Expected
CassandraError(std::string message, uint32_t code)
: message_{message}, code_{code}
CassandraError(std::string message, uint32_t code) : message_{message}, code_{code}
{
}
template <typename T>
friend std::string
operator+(
T const& lhs,
CassandraError const&
rhs) requires std::is_convertible_v<T, std::string>
operator+(T const& lhs, CassandraError const& rhs) requires std::is_convertible_v<T, std::string>
{
return lhs + rhs.message();
}
template <typename T>
friend bool
operator==(
T const& lhs,
CassandraError const&
rhs) requires std::is_convertible_v<T, std::string>
operator==(T const& lhs, CassandraError const& rhs) requires std::is_convertible_v<T, std::string>
{
return lhs == rhs.message();
}
@@ -89,10 +82,8 @@ public:
bool
isTimeout() const
{
if (code_ == CASS_ERROR_LIB_NO_HOSTS_AVAILABLE or
code_ == CASS_ERROR_LIB_REQUEST_TIMED_OUT or
code_ == CASS_ERROR_SERVER_UNAVAILABLE or
code_ == CASS_ERROR_SERVER_OVERLOADED or
if (code_ == CASS_ERROR_LIB_NO_HOSTS_AVAILABLE or code_ == CASS_ERROR_LIB_REQUEST_TIMED_OUT or
code_ == CASS_ERROR_SERVER_UNAVAILABLE or code_ == CASS_ERROR_SERVER_OVERLOADED or
code_ == CASS_ERROR_SERVER_READ_TIMEOUT)
return true;
return false;

25
src/backend/cassandra/Handle.cpp
View File

@@ -25,8 +25,7 @@ Handle::Handle(Settings clusterSettings) : cluster_{clusterSettings}
{
}
Handle::Handle(std::string_view contactPoints)
: Handle{Settings::defaultSettings().withContactPoints(contactPoints)}
Handle::Handle(std::string_view contactPoints) : Handle{Settings::defaultSettings().withContactPoints(contactPoints)}
{
}
@@ -75,9 +74,7 @@ Handle::FutureType
Handle::asyncReconnect(std::string_view keyspace) const
{
if (auto rc = asyncDisconnect().await(); not rc) // sync
throw std::logic_error(
"Reconnect to keyspace '" + std::string{keyspace} +
"' failed: " + rc.error());
throw std::logic_error("Reconnect to keyspace '" + std::string{keyspace} + "' failed: " + rc.error());
return asyncConnect(keyspace);
}
@@ -99,8 +96,7 @@ Handle::asyncExecuteEach(std::vector<Statement> const& statements) const
Handle::MaybeErrorType
Handle::executeEach(std::vector<Statement> const& statements) const
{
for (auto futures = asyncExecuteEach(statements);
auto const& future : futures)
for (auto futures = asyncExecuteEach(statements); auto const& future : futures)
{
if (auto const rc = future.await(); not rc)
return rc;
@@ -116,12 +112,9 @@ Handle::asyncExecute(Statement const& statement) const
}
Handle::FutureWithCallbackType
Handle::asyncExecute(
Statement const& statement,
std::function<void(Handle::ResultOrErrorType)>&& cb) const
Handle::asyncExecute(Statement const& statement, std::function<void(Handle::ResultOrErrorType)>&& cb) const
{
return Handle::FutureWithCallbackType{
cass_session_execute(session_, statement), std::move(cb)};
return Handle::FutureWithCallbackType{cass_session_execute(session_, statement), std::move(cb)};
}
Handle::ResultOrErrorType
@@ -143,12 +136,10 @@ Handle::execute(std::vector<Statement> const& statements) const
}
Handle::FutureWithCallbackType
Handle::asyncExecute(
std::vector<Statement> const& statements,
std::function<void(Handle::ResultOrErrorType)>&& cb) const
Handle::asyncExecute(std::vector<Statement> const& statements, std::function<void(Handle::ResultOrErrorType)>&& cb)
const
{
return Handle::FutureWithCallbackType{
cass_session_execute_batch(session_, Batch{statements}), std::move(cb)};
return Handle::FutureWithCallbackType{cass_session_execute_batch(session_, Batch{statements}), std::move(cb)};
}
Handle::PreparedStatementType

11
src/backend/cassandra/Handle.h
View File

@@ -212,8 +212,7 @@ public:
[[maybe_unused]] ResultOrErrorType
execute(PreparedStatementType const& statement, Args&&... args) const
{
return asyncExecute<Args...>(statement, std::forward<Args>(args)...)
.get();
return asyncExecute<Args...>(statement, std::forward<Args>(args)...).get();
}
/**
@@ -231,9 +230,7 @@ public:
* @return A future that holds onto the callback provided
*/
[[nodiscard]] FutureWithCallbackType
asyncExecute(
StatementType const& statement,
std::function<void(ResultOrErrorType)>&& cb) const;
asyncExecute(StatementType const& statement, std::function<void(ResultOrErrorType)>&& cb) const;
/**
* @brief Synchonous version of the above
@@ -268,9 +265,7 @@ public:
* @return A future that holds onto the callback provided
*/
[[nodiscard]] FutureWithCallbackType
asyncExecute(
std::vector<StatementType> const& statements,
std::function<void(ResultOrErrorType)>&& cb) const;
asyncExecute(std::vector<StatementType> const& statements, std::function<void(ResultOrErrorType)>&& cb) const;
/**
* @brief Prepare a statement

41
src/backend/cassandra/Schema.h
View File

@@ -32,15 +32,9 @@
namespace Backend::Cassandra {
template <SomeSettingsProvider SettingsProviderType>
[[nodiscard]] std::string inline qualifiedTableName(
SettingsProviderType const& provider,
std::string_view name)
[[nodiscard]] std::string inline qualifiedTableName(SettingsProviderType const& provider, std::string_view name)
{
return fmt::format(
"{}.{}{}",
provider.getKeyspace(),
provider.getTablePrefix().value_or(""),
name);
return fmt::format("{}.{}{}", provider.getKeyspace(), provider.getTablePrefix().value_or(""), name);
}
/**
@@ -58,8 +52,7 @@ class Schema
std::reference_wrapper<SettingsProviderType const> settingsProvider_;
public:
explicit Schema(SettingsProviderType const& settingsProvider)
: settingsProvider_{std::cref(settingsProvider)}
explicit Schema(SettingsProviderType const& settingsProvider) : settingsProvider_{std::cref(settingsProvider)}
{
}
@@ -229,8 +222,7 @@ public:
PRIMARY KEY (issuer, taxon, token_id)
)
)",
qualifiedTableName(
settingsProvider_.get(), "issuer_nf_tokens_v2")));
qualifiedTableName(settingsProvider_.get(), "issuer_nf_tokens_v2")));
statements.emplace_back(fmt::format(
R"(
@@ -259,8 +251,7 @@ public:
WITH CLUSTERING ORDER BY (seq_idx DESC)
AND default_time_to_live = {}
)",
qualifiedTableName(
settingsProvider_.get(), "nf_token_transactions"),
qualifiedTableName(settingsProvider_.get(), "nf_token_transactions"),
settingsProvider_.get().getTtl()));
return statements;
@@ -275,9 +266,7 @@ public:
std::reference_wrapper<Handle const> handle_;
public:
Statements(
SettingsProviderType const& settingsProvider,
Handle const& handle)
Statements(SettingsProviderType const& settingsProvider, Handle const& handle)
: settingsProvider_{settingsProvider}, handle_{std::cref(handle)}
{
}
@@ -313,8 +302,7 @@ public:
(ledger_sequence, hash)
VALUES (?, ?)
)",
qualifiedTableName(
settingsProvider_.get(), "ledger_transactions")));
qualifiedTableName(settingsProvider_.get(), "ledger_transactions")));
}();
PreparedStatement insertSuccessor = [this]() {
@@ -364,8 +352,7 @@ public:
(issuer, taxon, token_id)
VALUES (?, ?, ?)
)",
qualifiedTableName(
settingsProvider_.get(), "issuer_nf_tokens_v2")));
qualifiedTableName(settingsProvider_.get(), "issuer_nf_tokens_v2")));
}();
PreparedStatement insertNFTURI = [this]() {
@@ -385,8 +372,7 @@ public:
(token_id, seq_idx, hash)
VALUES (?, ?, ?)
)",
qualifiedTableName(
settingsProvider_.get(), "nf_token_transactions")));
qualifiedTableName(settingsProvider_.get(), "nf_token_transactions")));
}();
PreparedStatement insertLedgerHeader = [this]() {
@@ -491,8 +477,7 @@ public:
FROM {}
WHERE ledger_sequence = ?
)",
qualifiedTableName(
settingsProvider_.get(), "ledger_transactions")));
qualifiedTableName(settingsProvider_.get(), "ledger_transactions")));
}();
PreparedStatement selectLedgerPageKeys = [this]() {
@@ -595,8 +580,7 @@ public:
ORDER BY seq_idx DESC
LIMIT ?
)",
qualifiedTableName(
settingsProvider_.get(), "nf_token_transactions")));
qualifiedTableName(settingsProvider_.get(), "nf_token_transactions")));
}();
PreparedStatement selectNFTTxForward = [this]() {
@@ -609,8 +593,7 @@ public:
ORDER BY seq_idx ASC
LIMIT ?
)",
qualifiedTableName(
settingsProvider_.get(), "nf_token_transactions")));
qualifiedTableName(settingsProvider_.get(), "nf_token_transactions")));
}();
PreparedStatement selectLedgerByHash = [this]() {

45
src/backend/cassandra/SettingsProvider.cpp
View File

@@ -31,9 +31,7 @@ namespace Backend::Cassandra {
namespace detail {
inline Settings::ContactPoints
tag_invoke(
boost::json::value_to_tag<Settings::ContactPoints>,
boost::json::value const& value)
tag_invoke(boost::json::value_to_tag<Settings::ContactPoints>, boost::json::value const& value)
{
if (not value.is_object())
throw std::runtime_error(
@@ -43,17 +41,14 @@ tag_invoke(
clio::Config obj{value};
Settings::ContactPoints out;
out.contactPoints = obj.valueOrThrow<std::string>(
"contact_points", "`contact_points` must be a string");
out.contactPoints = obj.valueOrThrow<std::string>("contact_points", "`contact_points` must be a string");
out.port = obj.maybeValue<uint16_t>("port");
return out;
}
inline Settings::SecureConnectionBundle
tag_invoke(
boost::json::value_to_tag<Settings::SecureConnectionBundle>,
boost::json::value const& value)
tag_invoke(boost::json::value_to_tag<Settings::SecureConnectionBundle>, boost::json::value const& value)
{
if (not value.is_string())
throw std::runtime_error("`secure_connect_bundle` must be a string");
@@ -80,28 +75,19 @@ SettingsProvider::getSettings() const
std::optional<std::string>
SettingsProvider::parseOptionalCertificate() const
{
if (auto const certPath = config_.maybeValue<std::string>("certfile");
certPath)
if (auto const certPath = config_.maybeValue<std::string>("certfile"); certPath)
{
auto const path = std::filesystem::path(*certPath);
std::ifstream fileStream(path.string(), std::ios::in);
if (!fileStream)
{
throw std::system_error(
errno,
std::generic_category(),
"Opening certificate " + path.string());
throw std::system_error(errno, std::generic_category(), "Opening certificate " + path.string());
}
std::string contents(
std::istreambuf_iterator<char>{fileStream},
std::istreambuf_iterator<char>{});
std::string contents(std::istreambuf_iterator<char>{fileStream}, std::istreambuf_iterator<char>{});
if (fileStream.bad())
{
throw std::system_error(
errno,
std::generic_category(),
"Reading certificate " + path.string());
throw std::system_error(errno, std::generic_category(), "Reading certificate " + path.string());
}
return contents;
@@ -114,24 +100,21 @@ Settings
SettingsProvider::parseSettings() const
{
auto settings = Settings::defaultSettings();
if (auto const bundle =
config_.maybeValue<Settings::SecureConnectionBundle>(
"secure_connect_bundle");
bundle)
if (auto const bundle = config_.maybeValue<Settings::SecureConnectionBundle>("secure_connect_bundle"); bundle)
{
settings.connectionInfo = *bundle;
}
else
{
settings.connectionInfo = config_.valueOrThrow<Settings::ContactPoints>(
"Missing contact_points in Cassandra config");
settings.connectionInfo =
config_.valueOrThrow<Settings::ContactPoints>("Missing contact_points in Cassandra config");
}
settings.threads = config_.valueOr<uint32_t>("threads", settings.threads);
settings.maxWriteRequestsOutstanding = config_.valueOr<uint32_t>(
"max_write_requests_outstanding", settings.maxWriteRequestsOutstanding);
settings.maxReadRequestsOutstanding = config_.valueOr<uint32_t>(
"max_read_requests_outstanding", settings.maxReadRequestsOutstanding);
settings.maxWriteRequestsOutstanding =
config_.valueOr<uint32_t>("max_write_requests_outstanding", settings.maxWriteRequestsOutstanding);
settings.maxReadRequestsOutstanding =
config_.valueOr<uint32_t>("max_read_requests_outstanding", settings.maxReadRequestsOutstanding);
settings.certificate = parseOptionalCertificate();
settings.username = config_.maybeValue<std::string>("username");
settings.password = config_.maybeValue<std::string>("password");

35
src/backend/cassandra/impl/AsyncExecutor.h
View File

@@ -48,13 +48,10 @@ template <
typename StatementType,
typename HandleType = Handle,
SomeRetryPolicy RetryPolicyType = ExponentialBackoffRetryPolicy>
class AsyncExecutor
: public std::enable_shared_from_this<
AsyncExecutor<StatementType, HandleType, RetryPolicyType>>
class AsyncExecutor : public std::enable_shared_from_this<AsyncExecutor<StatementType, HandleType, RetryPolicyType>>
{
using FutureWithCallbackType = typename HandleType::FutureWithCallbackType;
using CallbackType =
std::function<void(typename HandleType::ResultOrErrorType)>;
using CallbackType = std::function<void(typename HandleType::ResultOrErrorType)>;
clio::Logger log_{"Backend"};
@@ -71,37 +68,24 @@ public:
* @brief Create a new instance of the AsyncExecutor and execute it.
*/
static void
run(boost::asio::io_context& ioc,
HandleType const& handle,
StatementType data,
CallbackType&& onComplete)
run(boost::asio::io_context& ioc, HandleType const& handle, StatementType data, CallbackType&& onComplete)
{
// this is a helper that allows us to use std::make_shared below
struct EnableMakeShared
: public AsyncExecutor<StatementType, HandleType, RetryPolicyType>
struct EnableMakeShared : public AsyncExecutor<StatementType, HandleType, RetryPolicyType>
{
EnableMakeShared(
boost::asio::io_context& ioc,
StatementType&& data,
CallbackType&& onComplete)
EnableMakeShared(boost::asio::io_context& ioc, StatementType&& data, CallbackType&& onComplete)
: AsyncExecutor(ioc, std::move(data), std::move(onComplete))
{
}
};
auto ptr = std::make_shared<EnableMakeShared>(
ioc, std::move(data), std::move(onComplete));
auto ptr = std::make_shared<EnableMakeShared>(ioc, std::move(data), std::move(onComplete));
ptr->execute(handle);
}
private:
AsyncExecutor(
boost::asio::io_context& ioc,
StatementType&& data,
CallbackType&& onComplete)
: data_{std::move(data)}
, retryPolicy_{ioc}
, onComplete_{std::move(onComplete)}
AsyncExecutor(boost::asio::io_context& ioc, StatementType&& data, CallbackType&& onComplete)
: data_{std::move(data)}, retryPolicy_{ioc}, onComplete_{std::move(onComplete)}
{
}
@@ -119,8 +103,7 @@ private:
else
{
if (retryPolicy_.shouldRetry(res.error()))
retryPolicy_.retry(
[self, &handle]() { self->execute(handle); });
retryPolicy_.retry([self, &handle]() { self->execute(handle); });
else
onComplete_(std::move(res)); // report error
}

10
src/backend/cassandra/impl/Batch.cpp
View File

@@ -26,9 +26,7 @@
#include <vector>
namespace {
static constexpr auto batchDeleter = [](CassBatch* ptr) {
cass_batch_free(ptr);
};
static constexpr auto batchDeleter = [](CassBatch* ptr) { cass_batch_free(ptr); };
};
namespace Backend::Cassandra::detail {
@@ -42,15 +40,13 @@ Batch::Batch(std::vector<Statement> const& statements)
for (auto const& statement : statements)
if (auto const res = add(statement); not res)
throw std::runtime_error(
"Failed to add statement to batch: " + res.error());
throw std::runtime_error("Failed to add statement to batch: " + res.error());
}
MaybeError
Batch::add(Statement const& statement)
{
if (auto const rc = cass_batch_add_statement(*this, statement);
rc != CASS_OK)
if (auto const rc = cass_batch_add_statement(*this, statement); rc != CASS_OK)
{
return Error{CassandraError{cass_error_desc(rc), rc}};
}

74
src/backend/cassandra/impl/Cluster.cpp
View File

@@ -26,9 +26,7 @@
#include <vector>
namespace {
static constexpr auto clusterDeleter = [](CassCluster* ptr) {
cass_cluster_free(ptr);
};
static constexpr auto clusterDeleter = [](CassCluster* ptr) { cass_cluster_free(ptr); };
template <class... Ts>
struct overloadSet : Ts...
@@ -43,28 +41,21 @@ overloadSet(Ts...) -> overloadSet<Ts...>;
namespace Backend::Cassandra::detail {
Cluster::Cluster(Settings const& settings)
: ManagedObject{cass_cluster_new(), clusterDeleter}
Cluster::Cluster(Settings const& settings) : ManagedObject{cass_cluster_new(), clusterDeleter}
{
using std::to_string;
cass_cluster_set_token_aware_routing(*this, cass_true);
if (auto const rc =
cass_cluster_set_protocol_version(*this, CASS_PROTOCOL_VERSION_V4);
rc != CASS_OK)
if (auto const rc = cass_cluster_set_protocol_version(*this, CASS_PROTOCOL_VERSION_V4); rc != CASS_OK)
{
throw std::runtime_error(
std::string{"Error setting cassandra protocol version to v4: "} +
cass_error_desc(rc));
throw std::runtime_error(std::string{"Error setting cassandra protocol version to v4: "} + cass_error_desc(rc));
}
if (auto const rc =
cass_cluster_set_num_threads_io(*this, settings.threads);
rc != CASS_OK)
if (auto const rc = cass_cluster_set_num_threads_io(*this, settings.threads); rc != CASS_OK)
{
throw std::runtime_error(
std::string{"Error setting cassandra io threads to "} +
to_string(settings.threads) + ": " + cass_error_desc(rc));
std::string{"Error setting cassandra io threads to "} + to_string(settings.threads) + ": " +
cass_error_desc(rc));
}
cass_log_set_level(settings.enableLog ? CASS_LOG_TRACE : CASS_LOG_DISABLED);
@@ -75,28 +66,21 @@ Cluster::Cluster(Settings const& settings)
// cass_cluster_set_max_concurrent_requests_threshold(*this, 10000);
// cass_cluster_set_queue_size_event(*this, 100000);
// cass_cluster_set_queue_size_io(*this, 100000);
// cass_cluster_set_write_bytes_high_water_mark(
// *this, 16 * 1024 * 1024); // 16mb
// cass_cluster_set_write_bytes_low_water_mark(
// *this, 8 * 1024 * 1024); // half of allowance
// cass_cluster_set_write_bytes_high_water_mark(*this, 16 * 1024 * 1024); // 16mb
// cass_cluster_set_write_bytes_low_water_mark(*this, 8 * 1024 * 1024); // half of allowance
// cass_cluster_set_pending_requests_high_water_mark(*this, 5000);
// cass_cluster_set_pending_requests_low_water_mark(*this, 2500); // half
// cass_cluster_set_max_requests_per_flush(*this, 1000);
// cass_cluster_set_max_concurrent_creation(*this, 8);
// cass_cluster_set_max_connections_per_host(*this, 6);
// cass_cluster_set_core_connections_per_host(*this, 4);
// cass_cluster_set_constant_speculative_execution_policy(*this, 1000,
// 1024);
// cass_cluster_set_constant_speculative_execution_policy(*this, 1000, 1024);
if (auto const rc = cass_cluster_set_queue_size_io(
*this,
settings.maxWriteRequestsOutstanding +
settings.maxReadRequestsOutstanding);
*this, settings.maxWriteRequestsOutstanding + settings.maxReadRequestsOutstanding);
rc != CASS_OK)
{
throw std::runtime_error(
std::string{"Could not set queue size for IO per host: "} +
cass_error_desc(rc));
throw std::runtime_error(std::string{"Could not set queue size for IO per host: "} + cass_error_desc(rc));
}
setupConnection(settings);
@@ -109,12 +93,8 @@ Cluster::setupConnection(Settings const& settings)
{
std::visit(
overloadSet{
[this](Settings::ContactPoints const& points) {
setupContactPoints(points);
},
[this](Settings::SecureConnectionBundle const& bundle) {
setupSecureBundle(bundle);
}},
[this](Settings::ContactPoints const& points) { setupContactPoints(points); },
[this](Settings::SecureConnectionBundle const& bundle) { setupSecureBundle(bundle); }},
settings.connectionInfo);
}
@@ -122,17 +102,13 @@ void
Cluster::setupContactPoints(Settings::ContactPoints const& points)
{
using std::to_string;
auto throwErrorIfNeeded =
[](CassError rc, std::string const label, std::string const value) {
if (rc != CASS_OK)
throw std::runtime_error(
"Cassandra: Error setting " + label + " [" + value +
"]: " + cass_error_desc(rc));
};
auto throwErrorIfNeeded = [](CassError rc, std::string const label, std::string const value) {
if (rc != CASS_OK)
throw std::runtime_error("Cassandra: Error setting " + label + " [" + value + "]: " + cass_error_desc(rc));
};
{
auto const rc =
cass_cluster_set_contact_points(*this, points.contactPoints.data());
auto const rc = cass_cluster_set_contact_points(*this, points.contactPoints.data());
throwErrorIfNeeded(rc, "contact_points", points.contactPoints);
}
@@ -146,12 +122,9 @@ Cluster::setupContactPoints(Settings::ContactPoints const& points)
void
Cluster::setupSecureBundle(Settings::SecureConnectionBundle const& bundle)
{
if (auto const rc = cass_cluster_set_cloud_secure_connection_bundle(
*this, bundle.bundle.data());
rc != CASS_OK)
if (auto const rc = cass_cluster_set_cloud_secure_connection_bundle(*this, bundle.bundle.data()); rc != CASS_OK)
{
throw std::runtime_error(
"Failed to connect using secure connection bundle" + bundle.bundle);
throw std::runtime_error("Failed to connect using secure connection bundle" + bundle.bundle);
}
}
@@ -171,10 +144,7 @@ Cluster::setupCredentials(Settings const& settings)
if (not settings.username || not settings.password)
return;
cass_cluster_set_credentials(
*this,
settings.username.value().c_str(),
settings.password.value().c_str());
cass_cluster_set_credentials(*this, settings.username.value().c_str(), settings.password.value().c_str());
}
} // namespace Backend::Cassandra::detail

9
src/backend/cassandra/impl/Cluster.h
View File

@@ -46,12 +46,9 @@ struct Settings
};
bool enableLog = false;
std::chrono::milliseconds connectionTimeout =
std::chrono::milliseconds{1000};
std::chrono::milliseconds requestTimeout =
std::chrono::milliseconds{0}; // no timeout at all
std::variant<ContactPoints, SecureConnectionBundle> connectionInfo =
ContactPoints{};
std::chrono::milliseconds connectionTimeout = std::chrono::milliseconds{1000};
std::chrono::milliseconds requestTimeout = std::chrono::milliseconds{0}; // no timeout at all
std::variant<ContactPoints, SecureConnectionBundle> connectionInfo = ContactPoints{};
uint32_t threads = std::thread::hardware_concurrency();
uint32_t maxWriteRequestsOutstanding = 10'000;
uint32_t maxReadRequestsOutstanding = 100'000;

82
src/backend/cassandra/impl/ExecutionStrategy.h
View File

@@ -78,8 +78,7 @@ public:
using CompletionTokenType = boost::asio::yield_context;
using FunctionType = void(boost::system::error_code);
using AsyncResultType =
boost::asio::async_result<CompletionTokenType, FunctionType>;
using AsyncResultType = boost::asio::async_result<CompletionTokenType, FunctionType>;
using HandlerType = typename AsyncResultType::completion_handler_type;
DefaultExecutionStrategy(Settings settings, HandleType const& handle)
@@ -89,10 +88,8 @@ public:
, handle_{std::cref(handle)}
, thread_{[this]() { ioc_.run(); }}
{
log_.info() << "Max write requests outstanding is "
<< maxWriteRequestsOutstanding_
<< "; Max read requests outstanding is "
<< maxReadRequestsOutstanding_;
log_.info() << "Max write requests outstanding is " << maxWriteRequestsOutstanding_
<< "; Max read requests outstanding is " << maxReadRequestsOutstanding_;
}
~DefaultExecutionStrategy()
@@ -136,8 +133,7 @@ public:
}
else
{
log_.warn()
<< "Cassandra sync write error, retrying: " << res.error();
log_.warn() << "Cassandra sync write error, retrying: " << res.error();
std::this_thread::sleep_for(std::chrono::milliseconds(5));
}
}
@@ -173,9 +169,7 @@ public:
// Note: lifetime is controlled by std::shared_from_this internally
AsyncExecutor<decltype(statement), HandleType>::run(
ioc_, handle_.get(), std::move(statement), [this](auto const&) {
decrementOutstandingRequestCount();
});
ioc_, handle_.get(), std::move(statement), [this](auto const&) { decrementOutstandingRequestCount(); });
}
/**
@@ -193,9 +187,7 @@ public:
// Note: lifetime is controlled by std::shared_from_this internally
AsyncExecutor<decltype(statements), HandleType>::run(
ioc_, handle_.get(), statements, [this](auto const&) {
decrementOutstandingRequestCount();
});
ioc_, handle_.get(), statements, [this](auto const&) { decrementOutstandingRequestCount(); });
}
/**
@@ -211,10 +203,7 @@ public:
*/
template <typename... Args>
[[maybe_unused]] ResultOrErrorType
read(
CompletionTokenType token,
PreparedStatementType const& preparedStatement,
Args&&... args)
read(CompletionTokenType token, PreparedStatementType const& preparedStatement, Args&&... args)
{
return read(token, preparedStatement.bind(std::forward<Args>(args)...));
}
@@ -230,9 +219,7 @@ public:
* @return ResultType or error wrapped in Expected
*/
[[maybe_unused]] ResultOrErrorType
read(
CompletionTokenType token,
std::vector<StatementType> const& statements)
read(CompletionTokenType token, std::vector<StatementType> const& statements)
{
auto handler = HandlerType{token};
auto result = AsyncResultType{handler};
@@ -242,14 +229,11 @@ public:
{
numReadRequestsOutstanding_ += statements.size();
auto const future = handle_.get().asyncExecute(
statements, [handler](auto&&) mutable {
boost::asio::post(
boost::asio::get_associated_executor(handler),
[handler]() mutable {
handler(boost::system::error_code{});
});
auto const future = handle_.get().asyncExecute(statements, [handler](auto&&) mutable {
boost::asio::post(boost::asio::get_associated_executor(handler), [handler]() mutable {
handler(boost::system::error_code{});
});
});
// suspend coroutine until completion handler is called
result.get();
@@ -264,8 +248,7 @@ public:
}
else
{
log_.error()
<< "Failed batch read in coroutine: " << res.error();
log_.error() << "Failed batch read in coroutine: " << res.error();
throwErrorIfNeeded(res.error());
}
}
@@ -292,14 +275,11 @@ public:
{
++numReadRequestsOutstanding_;
auto const future = handle_.get().asyncExecute(
statement, [handler](auto const&) mutable {
boost::asio::post(
boost::asio::get_associated_executor(handler),
[handler]() mutable {
handler(boost::system::error_code{});
});
auto const future = handle_.get().asyncExecute(statement, [handler](auto const&) mutable {
boost::asio::post(boost::asio::get_associated_executor(handler), [handler]() mutable {
handler(boost::system::error_code{});
});
});
// suspend coroutine until completion handler is called
result.get();
@@ -332,9 +312,7 @@ public:
* @return Vector of results
*/
std::vector<ResultType>
readEach(
CompletionTokenType token,
std::vector<StatementType> const& statements)
readEach(CompletionTokenType token, std::vector<StatementType> const& statements)
{
auto handler = HandlerType{token};
auto result = AsyncResultType{handler};
@@ -347,19 +325,16 @@ public:
futures.reserve(numOutstanding);
// used as the handler for each async statement individually
auto executionHandler =
[handler, &hadError, &numOutstanding](auto const& res) mutable {
if (not res)
hadError = true;
auto executionHandler = [handler, &hadError, &numOutstanding](auto const& res) mutable {
if (not res)
hadError = true;
// when all async operations complete unblock the result
if (--numOutstanding == 0)
boost::asio::post(
boost::asio::get_associated_executor(handler),
[handler]() mutable {
handler(boost::system::error_code{});
});
};
// when all async operations complete unblock the result
if (--numOutstanding == 0)
boost::asio::post(boost::asio::get_associated_executor(handler), [handler]() mutable {
handler(boost::system::error_code{});
});
};
std::transform(
std::cbegin(statements),
@@ -407,8 +382,7 @@ private:
{
log_.trace() << "Max outstanding requests reached. "
<< "Waiting for other requests to finish";
throttleCv_.wait(
lck, [this]() { return canAddWriteRequest(); });
throttleCv_.wait(lck, [this]() { return canAddWriteRequest(); });
}
}
++numWriteRequestsOutstanding_;

19
src/backend/cassandra/impl/Future.cpp
View File

@@ -25,15 +25,12 @@
#include <vector>
namespace {
static constexpr auto futureDeleter = [](CassFuture* ptr) {
cass_future_free(ptr);
};
static constexpr auto futureDeleter = [](CassFuture* ptr) { cass_future_free(ptr); };
} // namespace
namespace Backend::Cassandra::detail {
/* implicit */ Future::Future(CassFuture* ptr)
: ManagedObject{ptr, futureDeleter}
/* implicit */ Future::Future(CassFuture* ptr) : ManagedObject{ptr, futureDeleter}
{
}
@@ -62,9 +59,7 @@ Future::get() const
char const* message;
std::size_t len;
cass_future_error_message(*this, &message, &len);
return std::make_pair(
label + ": " + std::string{message, len},
cass_future_error_code(*this));
return std::make_pair(label + ": " + std::string{message, len}, cass_future_error_code(*this));
}("future::get()");
return Error{CassandraError{errMsg, code}};
}
@@ -85,9 +80,7 @@ invokeHelper(CassFuture* ptr, void* cbPtr)
char const* message;
std::size_t len;
cass_future_error_message(ptr, &message, &len);
return std::make_pair(
label + ": " + std::string{message, len},
cass_future_error_code(ptr));
return std::make_pair(label + ": " + std::string{message, len}, cass_future_error_code(ptr));
}("invokeHelper");
(*cb)(Error{CassandraError{errMsg, code}});
}
@@ -97,9 +90,7 @@ invokeHelper(CassFuture* ptr, void* cbPtr)
}
}
/* implicit */ FutureWithCallback::FutureWithCallback(
CassFuture* ptr,
fn_t&& cb)
/* implicit */ FutureWithCallback::FutureWithCallback(CassFuture* ptr, fn_t&& cb)
: Future{ptr}, cb_{std::make_unique<fn_t>(std::move(cb))}
{
// Instead of passing `this` as the userdata void*, we pass the address of

6
src/backend/cassandra/impl/ManagedObject.h
View File

@@ -31,12 +31,10 @@ protected:
public:
template <typename deleterCallable>
ManagedObject(Managed* rawPtr, deleterCallable deleter)
: ptr_{rawPtr, deleter}
ManagedObject(Managed* rawPtr, deleterCallable deleter) : ptr_{rawPtr, deleter}
{
if (rawPtr == nullptr)
throw std::runtime_error(
"Could not create DB object - got nullptr");
throw std::runtime_error("Could not create DB object - got nullptr");
}
ManagedObject(ManagedObject&&) = default;

14
src/backend/cassandra/impl/Result.cpp
View File

@@ -20,18 +20,13 @@
#include <backend/cassandra/impl/Result.h>
namespace {
static constexpr auto resultDeleter = [](CassResult const* ptr) {
cass_result_free(ptr);
};
static constexpr auto resultIteratorDeleter = [](CassIterator* ptr) {
cass_iterator_free(ptr);
};
static constexpr auto resultDeleter = [](CassResult const* ptr) { cass_result_free(ptr); };
static constexpr auto resultIteratorDeleter = [](CassIterator* ptr) { cass_iterator_free(ptr); };
} // namespace
namespace Backend::Cassandra::detail {
/* implicit */ Result::Result(CassResult const* ptr)
: ManagedObject{ptr, resultDeleter}
/* implicit */ Result::Result(CassResult const* ptr) : ManagedObject{ptr, resultDeleter}
{
}
@@ -48,8 +43,7 @@ Result::hasRows() const
}
/* implicit */ ResultIterator::ResultIterator(CassIterator* ptr)
: ManagedObject{ptr, resultIteratorDeleter}
, hasMore_{cass_iterator_next(ptr)}
: ManagedObject{ptr, resultIteratorDeleter}, hasMore_{cass_iterator_next(ptr)}
{
}

24
src/backend/cassandra/impl/Result.h
View File

@@ -59,8 +59,7 @@ extractColumn(CassRow const* row, std::size_t idx)
{
cass_byte_t const* buf;
std::size_t bufSize;
auto const rc =
cass_value_get_bytes(cass_row_get_column(row, idx), &buf, &bufSize);
auto const rc = cass_value_get_bytes(cass_row_get_column(row, idx), &buf, &bufSize);
throwErrorIfNeeded(rc, "Extract ripple::uint256");
output = ripple::uint256::fromVoid(buf);
}
@@ -68,8 +67,7 @@ extractColumn(CassRow const* row, std::size_t idx)
{
cass_byte_t const* buf;
std::size_t bufSize;
auto const rc =
cass_value_get_bytes(cass_row_get_column(row, idx), &buf, &bufSize);
auto const rc = cass_value_get_bytes(cass_row_get_column(row, idx), &buf, &bufSize);
throwErrorIfNeeded(rc, "Extract ripple::AccountID");
output = ripple::AccountID::fromVoid(buf);
}
@@ -77,8 +75,7 @@ extractColumn(CassRow const* row, std::size_t idx)
{
cass_byte_t const* buf;
std::size_t bufSize;
auto const rc =
cass_value_get_bytes(cass_row_get_column(row, idx), &buf, &bufSize);
auto const rc = cass_value_get_bytes(cass_row_get_column(row, idx), &buf, &bufSize);
throwErrorIfNeeded(rc, "Extract vector<unsigned char>");
output = uchar_vector_t{buf, buf + bufSize};
}
@@ -91,16 +88,14 @@ extractColumn(CassRow const* row, std::size_t idx)
{
char const* value;
std::size_t len;
auto const rc =
cass_value_get_string(cass_row_get_column(row, idx), &value, &len);
auto const rc = cass_value_get_string(cass_row_get_column(row, idx), &value, &len);
throwErrorIfNeeded(rc, "Extract string");
output = std::string{value, len};
}
else if constexpr (std::is_same_v<decayed_t, bool>)
{
cass_bool_t flag;
auto const rc =
cass_value_get_bool(cass_row_get_column(row, idx), &flag);
auto const rc = cass_value_get_bool(cass_row_get_column(row, idx), &flag);
throwErrorIfNeeded(rc, "Extract bool");
output = flag ? true : false;
}
@@ -108,8 +103,7 @@ extractColumn(CassRow const* row, std::size_t idx)
else if constexpr (std::is_convertible_v<decayed_t, int64_t>)
{
int64_t out;
auto const rc =
cass_value_get_int64(cass_row_get_column(row, idx), &out);
auto const rc = cass_value_get_int64(cass_row_get_column(row, idx), &out);
throwErrorIfNeeded(rc, "Extract int64");
output = static_cast<decayed_t>(out);
}
@@ -144,8 +138,7 @@ struct Result : public ManagedObject<CassResult const>
std::size_t idx = 0;
auto advanceId = [&idx]() { return idx++; };
return std::make_optional<std::tuple<RowTypes...>>(
{extractColumn<RowTypes>(row, advanceId())...});
return std::make_optional<std::tuple<RowTypes...>>({extractColumn<RowTypes>(row, advanceId())...});
}
template <typename RowType>
@@ -207,8 +200,7 @@ public:
using difference_type = std::size_t; // rows count
using value_type = std::tuple<Types...>;
/* implicit */ Iterator(ResultIterator iterator)
: iterator_{std::move(iterator)}
/* implicit */ Iterator(ResultIterator iterator) : iterator_{std::move(iterator)}
{
}

17
src/backend/cassandra/impl/RetryPolicy.h
View File

@@ -59,9 +59,8 @@ public:
shouldRetry([[maybe_unused]] CassandraError err)
{
auto const delay = calculateDelay(attempt_);
log_.error() << "Cassandra write error: " << err << ", current retries "
<< attempt_ << ", retrying in " << delay.count()
<< " milliseconds";
log_.error() << "Cassandra write error: " << err << ", current retries " << attempt_ << ", retrying in "
<< delay.count() << " milliseconds";
return true; // keep retrying forever
}
@@ -76,11 +75,10 @@ public:
retry(Fn&& fn)
{
timer_.expires_after(calculateDelay(attempt_++));
timer_.async_wait(
[fn = std::move(fn)]([[maybe_unused]] const auto& err) {
// todo: deal with cancellation (thru err)
fn();
});
timer_.async_wait([fn = std::move(fn)]([[maybe_unused]] const auto& err) {
// todo: deal with cancellation (thru err)
fn();
});
}
/**
@@ -89,8 +87,7 @@ public:
std::chrono::milliseconds
calculateDelay(uint32_t attempt)
{
return std::chrono::milliseconds{
lround(std::pow(2, std::min(10u, attempt)))};
return std::chrono::milliseconds{lround(std::pow(2, std::min(10u, attempt)))};
}
};

4
src/backend/cassandra/impl/Session.h
View File

@@ -27,9 +27,7 @@ namespace Backend::Cassandra::detail {
class Session : public ManagedObject<CassSession>
{
static constexpr auto deleter = [](CassSession* ptr) {
cass_session_free(ptr);
};
static constexpr auto deleter = [](CassSession* ptr) { cass_session_free(ptr); };
public:
Session() : ManagedObject{cass_session_new(), deleter}

10
src/backend/cassandra/impl/SslContext.cpp
View File

@@ -25,16 +25,12 @@ static constexpr auto contextDeleter = [](CassSsl* ptr) { cass_ssl_free(ptr); };
namespace Backend::Cassandra::detail {
SslContext::SslContext(std::string const& certificate)
: ManagedObject{cass_ssl_new(), contextDeleter}
SslContext::SslContext(std::string const& certificate) : ManagedObject{cass_ssl_new(), contextDeleter}
{
cass_ssl_set_verify_flags(*this, CASS_SSL_VERIFY_NONE);
if (auto const rc = cass_ssl_add_trusted_cert(*this, certificate.c_str());
rc != CASS_OK)
if (auto const rc = cass_ssl_add_trusted_cert(*this, certificate.c_str()); rc != CASS_OK)
{
throw std::runtime_error(
std::string{"Error setting Cassandra SSL Context: "} +
cass_error_desc(rc));
throw std::runtime_error(std::string{"Error setting Cassandra SSL Context: "} + cass_error_desc(rc));
}
}

31
src/backend/cassandra/impl/Statement.h
View File

@@ -37,9 +37,7 @@ namespace Backend::Cassandra::detail {
class Statement : public ManagedObject<CassStatement>
{
static constexpr auto deleter = [](CassStatement* ptr) {
cass_statement_free(ptr);
};
static constexpr auto deleter = [](CassStatement* ptr) { cass_statement_free(ptr); };
template <typename>
static constexpr bool unsupported_v = false;
@@ -53,9 +51,7 @@ public:
*/
template <typename... Args>
explicit Statement(std::string_view query, Args&&... args)
: ManagedObject{
cass_statement_new(query.data(), sizeof...(args)),
deleter}
: ManagedObject{cass_statement_new(query.data(), sizeof...(args)), deleter}
{
cass_statement_set_consistency(*this, CASS_CONSISTENCY_QUORUM);
cass_statement_set_is_idempotent(*this, cass_true);
@@ -85,13 +81,11 @@ public:
using std::to_string;
auto throwErrorIfNeeded = [idx](CassError rc, std::string_view label) {
if (rc != CASS_OK)
throw std::logic_error(fmt::format(
"[{}] at idx {}: {}", label, idx, cass_error_desc(rc)));
throw std::logic_error(fmt::format("[{}] at idx {}: {}", label, idx, cass_error_desc(rc)));
};
auto bindBytes = [this, idx](auto const* data, size_t size) {
return cass_statement_bind_bytes(
*this, idx, static_cast<cass_byte_t const*>(data), size);
return cass_statement_bind_bytes(*this, idx, static_cast<cass_byte_t const*>(data), size);
};
using decayed_t = std::decay_t<Type>;
@@ -116,21 +110,17 @@ public:
else if constexpr (std::is_convertible_v<decayed_t, std::string>)
{
// reinterpret_cast is needed here :'(
auto const rc = bindBytes(
reinterpret_cast<unsigned char const*>(value.data()),
value.size());
auto const rc = bindBytes(reinterpret_cast<unsigned char const*>(value.data()), value.size());
throwErrorIfNeeded(rc, "Bind string (as bytes)");
}
else if constexpr (std::is_same_v<decayed_t, uint_tuple_t>)
{
auto const rc =
cass_statement_bind_tuple(*this, idx, Tuple{std::move(value)});
auto const rc = cass_statement_bind_tuple(*this, idx, Tuple{std::move(value)});
throwErrorIfNeeded(rc, "Bind tuple<uint32, uint32>");
}
else if constexpr (std::is_same_v<decayed_t, bool>)
{
auto const rc = cass_statement_bind_bool(
*this, idx, value ? cass_true : cass_false);
auto const rc = cass_statement_bind_bool(*this, idx, value ? cass_true : cass_false);
throwErrorIfNeeded(rc, "Bind bool");
}
else if constexpr (std::is_same_v<decayed_t, Limit>)
@@ -154,13 +144,10 @@ public:
class PreparedStatement : public ManagedObject<CassPrepared const>
{
static constexpr auto deleter = [](CassPrepared const* ptr) {
cass_prepared_free(ptr);
};
static constexpr auto deleter = [](CassPrepared const* ptr) { cass_prepared_free(ptr); };
public:
/* implicit */ PreparedStatement(CassPrepared const* ptr)
: ManagedObject{ptr, deleter}
/* implicit */ PreparedStatement(CassPrepared const* ptr) : ManagedObject{ptr, deleter}
{
}

11
src/backend/cassandra/impl/Tuple.cpp
View File

@@ -20,12 +20,8 @@
#include <backend/cassandra/impl/Tuple.h>
namespace {
static constexpr auto tupleDeleter = [](CassTuple* ptr) {
cass_tuple_free(ptr);
};
static constexpr auto tupleIteratorDeleter = [](CassIterator* ptr) {
cass_iterator_free(ptr);
};
static constexpr auto tupleDeleter = [](CassTuple* ptr) { cass_tuple_free(ptr); };
static constexpr auto tupleIteratorDeleter = [](CassIterator* ptr) { cass_iterator_free(ptr); };
} // namespace
namespace Backend::Cassandra::detail {
@@ -34,8 +30,7 @@ namespace Backend::Cassandra::detail {
{
}
/* implicit */ TupleIterator::TupleIterator(CassIterator* ptr)
: ManagedObject{ptr, tupleIteratorDeleter}
/* implicit */ TupleIterator::TupleIterator(CassIterator* ptr) : ManagedObject{ptr, tupleIteratorDeleter}
{
}

24
src/backend/cassandra/impl/Tuple.h
View File

@@ -32,9 +32,7 @@ namespace Backend::Cassandra::detail {
class Tuple : public ManagedObject<CassTuple>
{
static constexpr auto deleter = [](CassTuple* ptr) {
cass_tuple_free(ptr);
};
static constexpr auto deleter = [](CassTuple* ptr) { cass_tuple_free(ptr); };
template <typename>
static constexpr bool unsupported_v = false;
@@ -44,12 +42,9 @@ public:
template <typename... Types>
explicit Tuple(std::tuple<Types...>&& value)
: ManagedObject{
cass_tuple_new(std::tuple_size<std::tuple<Types...>>{}),
deleter}
: ManagedObject{cass_tuple_new(std::tuple_size<std::tuple<Types...>>{}), deleter}
{
std::apply(
std::bind_front(&Tuple::bind<Types...>, this), std::move(value));
std::apply(std::bind_front(&Tuple::bind<Types...>, this), std::move(value));
}
template <typename... Args>
@@ -69,9 +64,7 @@ public:
if (rc != CASS_OK)
{
auto const tag = '[' + std::string{label} + ']';
throw std::logic_error(
tag + " at idx " + to_string(idx) + ": " +
cass_error_desc(rc));
throw std::logic_error(tag + " at idx " + to_string(idx) + ": " + cass_error_desc(rc));
}
};
@@ -79,8 +72,7 @@ public:
if constexpr (std::is_same_v<decayed_t, bool>)
{
auto const rc =
cass_tuple_set_bool(*this, idx, value ? cass_true : cass_false);
auto const rc = cass_tuple_set_bool(*this, idx, value ? cass_true : cass_false);
throwErrorIfNeeded(rc, "Bind bool");
}
// clio only uses bigint (int64_t) so we convert any incoming type
@@ -124,8 +116,7 @@ private:
Type output;
if (not cass_iterator_next(*this))
throw std::logic_error(
"Could not extract next value from tuple iterator");
throw std::logic_error("Could not extract next value from tuple iterator");
auto throwErrorIfNeeded = [](CassError rc, std::string_view label) {
if (rc != CASS_OK)
@@ -141,8 +132,7 @@ private:
if constexpr (std::is_convertible_v<decayed_t, int64_t>)
{
int64_t out;
auto const rc =
cass_value_get_int64(cass_iterator_get_value(*this), &out);
auto const rc = cass_value_get_int64(cass_iterator_get_value(*this), &out);
throwErrorIfNeeded(rc, "Extract int64 from tuple");
output = static_cast<decayed_t>(out);
}

19
src/config/Config.cpp
View File

@@ -62,8 +62,7 @@ Config::lookup(key_type key) const
if (not hasBrokenPath)
{
if (not cur.get().is_object())
throw detail::StoreException(
"Not an object at '" + subkey + "'");
throw detail::StoreException("Not an object at '" + subkey + "'");
if (not cur.get().as_object().contains(section))
hasBrokenPath = true;
else
@@ -91,11 +90,9 @@ Config::maybeArray(key_type key) const
array_type out;
out.reserve(arr.size());
std::transform(
std::begin(arr),
std::end(arr),
std::back_inserter(out),
[](auto&& element) { return Config{std::move(element)}; });
std::transform(std::begin(arr), std::end(arr), std::back_inserter(out), [](auto&& element) {
return Config{std::move(element)};
});
return std::make_optional<array_type>(std::move(out));
}
}
@@ -156,10 +153,7 @@ Config::array() const
out.reserve(arr.size());
std::transform(
std::cbegin(arr),
std::cend(arr),
std::back_inserter(out),
[](auto const& element) { return Config{element}; });
std::cbegin(arr), std::cend(arr), std::back_inserter(out), [](auto const& element) { return Config{element}; });
return out;
}
@@ -180,8 +174,7 @@ ConfigReader::open(std::filesystem::path path)
}
catch (std::exception const& e)
{
LogService::error() << "Could not read configuration file from '"
<< path.string() << "': " << e.what();
LogService::error() << "Could not read configuration file from '" << path.string() << "': " << e.what();
}
return Config{};

16
src/config/Config.h
View File

@@ -43,9 +43,7 @@ class Config final
public:
using key_type = std::string; /*! The type of key used */
using array_type = std::vector<Config>; /*! The type of array used */
using write_cursor_type = std::pair<
std::optional<std::reference_wrapper<boost::json::value>>,
key_type>;
using write_cursor_type = std::pair<std::optional<std::reference_wrapper<boost::json::value>>, key_type>;
/**
* @brief Construct a new Config object.
@@ -101,8 +99,7 @@ public:
{
auto maybe_element = lookup(key);
if (maybe_element)
return std::make_optional<Result>(
checkedAs<Result>(key, *maybe_element));
return std::make_optional<Result>(checkedAs<Result>(key, *maybe_element));
return std::nullopt;
}
@@ -367,9 +364,7 @@ private:
if (not value.is_number())
has_error = true;
}
else if constexpr (
std::is_convertible_v<Return, uint64_t> ||
std::is_convertible_v<Return, int64_t>)
else if constexpr (std::is_convertible_v<Return, uint64_t> || std::is_convertible_v<Return, int64_t>)
{
if (not value.is_int64() && not value.is_uint64())
has_error = true;
@@ -377,9 +372,8 @@ private:
if (has_error)
throw std::runtime_error(
"Type for key '" + key + "' is '" +
std::string{to_string(value.kind())} +
"' in JSON but requested '" + detail::typeName<Return>() + "'");
"Type for key '" + key + "' is '" + std::string{to_string(value.kind())} + "' in JSON but requested '" +
detail::typeName<Return>() + "'");
return value_to<Return>(value);
}

8
src/etl/ETLHelpers.h
View File

@@ -77,14 +77,10 @@ public:
/// @return true if sequence was validated, false otherwise
/// a return value of false means the datastructure has been stopped
bool
waitUntilValidatedByNetwork(
uint32_t sequence,
std::optional<uint32_t> maxWaitMs = {})
waitUntilValidatedByNetwork(uint32_t sequence, std::optional<uint32_t> maxWaitMs = {})
{
std::unique_lock lck(m_);
auto pred = [sequence, this]() -> bool {
return (max_ && sequence <= *max_);
};
auto pred = [sequence, this]() -> bool { return (max_ && sequence <= *max_); };
if (maxWaitMs)
cv_.wait_for(lck, std::chrono::milliseconds(*maxWaitMs));
else

379
src/etl/ETLSource.cpp
View File

@@ -43,15 +43,12 @@ ForwardCache::freshen()
{
log_.trace() << "Freshening ForwardCache";
auto numOutstanding =
std::make_shared<std::atomic_uint>(latestForwarded_.size());
auto numOutstanding = std::make_shared<std::atomic_uint>(latestForwarded_.size());
for (auto const& cacheEntry : latestForwarded_)
{
boost::asio::spawn(
strand_,
[this, numOutstanding, command = cacheEntry.first](
boost::asio::yield_context yield) {
strand_, [this, numOutstanding, command = cacheEntry.first](boost::asio::yield_context yield) {
boost::json::object request = {{"command", command}};
auto resp = source_.requestFromRippled(request, {}, yield);
@@ -78,12 +75,9 @@ std::optional<boost::json::object>
ForwardCache::get(boost::json::object const& request) const
{
std::optional<std::string> command = {};
if (request.contains("command") && !request.contains("method") &&
request.at("command").is_string())
if (request.contains("command") && !request.contains("method") && request.at("command").is_string())
command = request.at("command").as_string().c_str();
else if (
request.contains("method") && !request.contains("command") &&
request.at("method").is_string())
else if (request.contains("method") && !request.contains("command") && request.at("method").is_string())
command = request.at("method").as_string().c_str();
if (!command)
@@ -116,8 +110,7 @@ make_TimeoutOption()
}
else
{
return boost::beast::websocket::stream_base::timeout::suggested(
boost::beast::role_type::client);
return boost::beast::websocket::stream_base::timeout::suggested(boost::beast::role_type::client);
}
}
@@ -138,8 +131,7 @@ ETLSourceImpl<Derived>::reconnect(boost::beast::error_code ec)
// if we cannot connect to the transaction processing process
if (ec.category() == boost::asio::error::get_ssl_category())
{
err = std::string(" (") +
boost::lexical_cast<std::string>(ERR_GET_LIB(ec.value())) + "," +
err = std::string(" (") + boost::lexical_cast<std::string>(ERR_GET_LIB(ec.value())) + "," +
boost::lexical_cast<std::string>(ERR_GET_REASON(ec.value())) + ") ";
// ERR_PACK /* crypto/err/err.h */
char buf[128];
@@ -149,8 +141,7 @@ ETLSourceImpl<Derived>::reconnect(boost::beast::error_code ec)
std::cout << err << std::endl;
}
if (ec != boost::asio::error::operation_aborted &&
ec != boost::asio::error::connection_refused)
if (ec != boost::asio::error::operation_aborted && ec != boost::asio::error::connection_refused)
{
log_.error() << "error code = " << ec << " - " << toString();
}
@@ -184,30 +175,25 @@ PlainETLSource::close(bool startAgain)
// an assertion fails. Using closing_ makes sure async_close is only
// called once
closing_ = true;
derived().ws().async_close(
boost::beast::websocket::close_code::normal,
[this, startAgain](auto ec) {
if (ec)
{
log_.error()
<< " async_close : "
<< "error code = " << ec << " - " << toString();
}
closing_ = false;
if (startAgain)
{
ws_ = std::make_unique<boost::beast::websocket::stream<
boost::beast::tcp_stream>>(
boost::asio::make_strand(ioc_));
derived().ws().async_close(boost::beast::websocket::close_code::normal, [this, startAgain](auto ec) {
if (ec)
{
log_.error() << " async_close : "
<< "error code = " << ec << " - " << toString();
}
closing_ = false;
if (startAgain)
{
ws_ = std::make_unique<boost::beast::websocket::stream<boost::beast::tcp_stream>>(
boost::asio::make_strand(ioc_));
run();
}
});
run();
}
});
}
else if (startAgain)
{
ws_ = std::make_unique<
boost::beast::websocket::stream<boost::beast::tcp_stream>>(
ws_ = std::make_unique<boost::beast::websocket::stream<boost::beast::tcp_stream>>(
boost::asio::make_strand(ioc_));
run();
@@ -229,31 +215,26 @@ SslETLSource::close(bool startAgain)
// an assertion fails. Using closing_ makes sure async_close is only
// called once
closing_ = true;
derived().ws().async_close(
boost::beast::websocket::close_code::normal,
[this, startAgain](auto ec) {
if (ec)
{
log_.error()
<< " async_close : "
<< "error code = " << ec << " - " << toString();
}
closing_ = false;
if (startAgain)
{
ws_ = std::make_unique<boost::beast::websocket::stream<
boost::beast::ssl_stream<
boost::beast::tcp_stream>>>(
boost::asio::make_strand(ioc_), *sslCtx_);
derived().ws().async_close(boost::beast::websocket::close_code::normal, [this, startAgain](auto ec) {
if (ec)
{
log_.error() << " async_close : "
<< "error code = " << ec << " - " << toString();
}
closing_ = false;
if (startAgain)
{
ws_ = std::make_unique<
boost::beast::websocket::stream<boost::beast::ssl_stream<boost::beast::tcp_stream>>>(
boost::asio::make_strand(ioc_), *sslCtx_);
run();
}
});
run();
}
});
}
else if (startAgain)
{
ws_ = std::make_unique<boost::beast::websocket::stream<
boost::beast::ssl_stream<boost::beast::tcp_stream>>>(
ws_ = std::make_unique<boost::beast::websocket::stream<boost::beast::ssl_stream<boost::beast::tcp_stream>>>(
boost::asio::make_strand(ioc_), *sslCtx_);
run();
@@ -263,9 +244,7 @@ SslETLSource::close(bool startAgain)
template <class Derived>
void
ETLSourceImpl<Derived>::onResolve(
boost::beast::error_code ec,
boost::asio::ip::tcp::resolver::results_type results)
ETLSourceImpl<Derived>::onResolve(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type results)
{
log_.trace() << "ec = " << ec << " - " << toString();
if (ec)
@@ -275,12 +254,10 @@ ETLSourceImpl<Derived>::onResolve(
}
else
{
boost::beast::get_lowest_layer(derived().ws())
.expires_after(std::chrono::seconds(30));
boost::beast::get_lowest_layer(derived().ws())
.async_connect(results, [this](auto ec, auto ep) {
derived().onConnect(ec, ep);
});
boost::beast::get_lowest_layer(derived().ws()).expires_after(std::chrono::seconds(30));
boost::beast::get_lowest_layer(derived().ws()).async_connect(results, [this](auto ec, auto ep) {
derived().onConnect(ec, ep);
});
}
}
@@ -307,21 +284,18 @@ PlainETLSource::onConnect(
// Set a decorator to change the User-Agent of the handshake
derived().ws().set_option(
boost::beast::websocket::stream_base::decorator(
[](boost::beast::websocket::request_type& req) {
req.set(
boost::beast::http::field::user_agent, "clio-client");
boost::beast::websocket::stream_base::decorator([](boost::beast::websocket::request_type& req) {
req.set(boost::beast::http::field::user_agent, "clio-client");
req.set("X-User", "clio-client");
}));
req.set("X-User", "clio-client");
}));
// Update the host_ string. This will provide the value of the
// Host HTTP header during the WebSocket handshake.
// See https://tools.ietf.org/html/rfc7230#section-5.4
auto host = ip_ + ':' + std::to_string(endpoint.port());
// Perform the websocket handshake
derived().ws().async_handshake(
host, "/", [this](auto ec) { onHandshake(ec); });
derived().ws().async_handshake(host, "/", [this](auto ec) { onHandshake(ec); });
}
}
@@ -348,13 +322,11 @@ SslETLSource::onConnect(
// Set a decorator to change the User-Agent of the handshake
derived().ws().set_option(
boost::beast::websocket::stream_base::decorator(
[](boost::beast::websocket::request_type& req) {
req.set(
boost::beast::http::field::user_agent, "clio-client");
boost::beast::websocket::stream_base::decorator([](boost::beast::websocket::request_type& req) {
req.set(boost::beast::http::field::user_agent, "clio-client");
req.set("X-User", "clio-client");
}));
req.set("X-User", "clio-client");
}));
// Update the host_ string. This will provide the value of the
// Host HTTP header during the WebSocket handshake.
@@ -362,8 +334,7 @@ SslETLSource::onConnect(
auto host = ip_ + ':' + std::to_string(endpoint.port());
// Perform the websocket handshake
ws().next_layer().async_handshake(
boost::asio::ssl::stream_base::client,
[this, endpoint](auto ec) { onSslHandshake(ec, endpoint); });
boost::asio::ssl::stream_base::client, [this, endpoint](auto ec) { onSslHandshake(ec, endpoint); });
}
}
@@ -390,8 +361,7 @@ void
ETLSourceImpl<Derived>::onHandshake(boost::beast::error_code ec)
{
log_.trace() << "ec = " << ec << " - " << toString();
if (auto action = hooks_.onConnected(ec);
action == ETLSourceHooks::Action::STOP)
if (auto action = hooks_.onConnected(ec); action == ETLSourceHooks::Action::STOP)
return;
if (ec)
@@ -402,35 +372,26 @@ ETLSourceImpl<Derived>::onHandshake(boost::beast::error_code ec)
else
{
boost::json::object jv{
{"command", "subscribe"},
{"streams",
{"ledger", "manifests", "validations", "transactions_proposed"}}};
{"command", "subscribe"}, {"streams", {"ledger", "manifests", "validations", "transactions_proposed"}}};
std::string s = boost::json::serialize(jv);
log_.trace() << "Sending subscribe stream message";
derived().ws().set_option(
boost::beast::websocket::stream_base::decorator(
[](boost::beast::websocket::request_type& req) {
req.set(
boost::beast::http::field::user_agent,
std::string(BOOST_BEAST_VERSION_STRING) +
" clio-client");
boost::beast::websocket::stream_base::decorator([](boost::beast::websocket::request_type& req) {
req.set(
boost::beast::http::field::user_agent, std::string(BOOST_BEAST_VERSION_STRING) + " clio-client");
req.set("X-User", "coro-client");
}));
req.set("X-User", "coro-client");
}));
// Send the message
derived().ws().async_write(
boost::asio::buffer(s),
[this](auto ec, size_t size) { onWrite(ec, size); });
derived().ws().async_write(boost::asio::buffer(s), [this](auto ec, size_t size) { onWrite(ec, size); });
}
}
template <class Derived>
void
ETLSourceImpl<Derived>::onWrite(
boost::beast::error_code ec,
size_t bytesWritten)
ETLSourceImpl<Derived>::onWrite(boost::beast::error_code ec, size_t bytesWritten)
{
log_.trace() << "ec = " << ec << " - " << toString();
if (ec)
@@ -440,8 +401,7 @@ ETLSourceImpl<Derived>::onWrite(
}
else
{
derived().ws().async_read(
readBuffer_, [this](auto ec, size_t size) { onRead(ec, size); });
derived().ws().async_read(readBuffer_, [this](auto ec, size_t size) { onRead(ec, size); });
}
}
@@ -462,8 +422,7 @@ ETLSourceImpl<Derived>::onRead(boost::beast::error_code ec, size_t size)
swap(readBuffer_, buffer);
log_.trace() << "calling async_read - " << toString();
derived().ws().async_read(
readBuffer_, [this](auto ec, size_t size) { onRead(ec, size); });
derived().ws().async_read(readBuffer_, [this](auto ec, size_t size) { onRead(ec, size); });
}
}
@@ -477,9 +436,7 @@ ETLSourceImpl<Derived>::handleMessage()
connected_ = true;
try
{
std::string msg{
static_cast<char const*>(readBuffer_.data().data()),
readBuffer_.size()};
std::string msg{static_cast<char const*>(readBuffer_.data().data()), readBuffer_.size()};
log_.trace() << msg;
boost::json::value raw = boost::json::parse(msg);
log_.trace() << "parsed";
@@ -495,32 +452,25 @@ ETLSourceImpl<Derived>::handleMessage()
}
if (result.contains("validated_ledgers"))
{
boost::json::string const& validatedLedgers =
result["validated_ledgers"].as_string();
boost::json::string const& validatedLedgers = result["validated_ledgers"].as_string();
setValidatedRange(
{validatedLedgers.c_str(), validatedLedgers.size()});
setValidatedRange({validatedLedgers.c_str(), validatedLedgers.size()});
}
log_.info() << "Received a message on ledger "
<< " subscription stream. Message : " << response
<< " - " << toString();
<< " subscription stream. Message : " << response << " - " << toString();
}
else if (
response.contains("type") && response["type"] == "ledgerClosed")
else if (response.contains("type") && response["type"] == "ledgerClosed")
{
log_.info() << "Received a message on ledger "
<< " subscription stream. Message : " << response
<< " - " << toString();
<< " subscription stream. Message : " << response << " - " << toString();
if (response.contains("ledger_index"))
{
ledgerIndex = response["ledger_index"].as_int64();
}
if (response.contains("validated_ledgers"))
{
boost::json::string const& validatedLedgers =
response["validated_ledgers"].as_string();
setValidatedRange(
{validatedLedgers.c_str(), validatedLedgers.size()});
boost::json::string const& validatedLedgers = response["validated_ledgers"].as_string();
setValidatedRange({validatedLedgers.c_str(), validatedLedgers.size()});
}
}
else
@@ -532,15 +482,11 @@ ETLSourceImpl<Derived>::handleMessage()
forwardCache_.freshen();
subscriptions_->forwardProposedTransaction(response);
}
else if (
response.contains("type") &&
response["type"] == "validationReceived")
else if (response.contains("type") && response["type"] == "validationReceived")
{
subscriptions_->forwardValidation(response);
}
else if (
response.contains("type") &&
response["type"] == "manifestReceived")
else if (response.contains("type") && response["type"] == "manifestReceived")
{
subscriptions_->forwardManifest(response);
}
@@ -549,8 +495,7 @@ ETLSourceImpl<Derived>::handleMessage()
if (ledgerIndex != 0)
{
log_.trace() << "Pushing ledger sequence = " << ledgerIndex << " - "
<< toString();
log_.trace() << "Pushing ledger sequence = " << ledgerIndex << " - " << toString();
networkValidatedLedgers_->push(ledgerIndex);
}
return true;
@@ -578,10 +523,7 @@ class AsyncCallData
std::string lastKey_;
public:
AsyncCallData(
uint32_t seq,
ripple::uint256 const& marker,
std::optional<ripple::uint256> const& nextMarker)
AsyncCallData(uint32_t seq, ripple::uint256 const& marker, std::optional<ripple::uint256> const& nextMarker)
{
request_.mutable_ledger()->set_sequence(seq);
if (marker.isNonZero())
@@ -595,11 +537,9 @@ public:
unsigned char prefix = marker.data()[0];
log_.debug() << "Setting up AsyncCallData. marker = "
<< ripple::strHex(marker)
log_.debug() << "Setting up AsyncCallData. marker = " << ripple::strHex(marker)
<< " . prefix = " << ripple::strHex(std::string(1, prefix))
<< " . nextPrefix_ = "
<< ripple::strHex(std::string(1, nextPrefix_));
<< " . nextPrefix_ = " << ripple::strHex(std::string(1, nextPrefix_));
assert(nextPrefix_ > prefix || nextPrefix_ == 0x00);
@@ -629,8 +569,7 @@ public:
if (!status_.ok())
{
log_.error() << "AsyncCallData status_ not ok: "
<< " code = " << status_.error_code()
<< " message = " << status_.error_message();
<< " code = " << status_.error_code() << " message = " << status_.error_message();
return CallStatus::ERRORED;
}
if (!next_->is_unlimited())
@@ -679,37 +618,26 @@ public:
if (!cacheOnly)
{
if (lastKey_.size())
backend.writeSuccessor(
std::move(lastKey_),
request_.ledger().sequence(),
std::string{obj.key()});
backend.writeSuccessor(std::move(lastKey_), request_.ledger().sequence(), std::string{obj.key()});
lastKey_ = obj.key();
backend.writeNFTs(getNFTDataFromObj(
request_.ledger().sequence(), obj.key(), obj.data()));
backend.writeNFTs(getNFTDataFromObj(request_.ledger().sequence(), obj.key(), obj.data()));
backend.writeLedgerObject(
std::move(*obj.mutable_key()),
request_.ledger().sequence(),
std::move(*obj.mutable_data()));
std::move(*obj.mutable_key()), request_.ledger().sequence(), std::move(*obj.mutable_data()));
}
}
backend.cache().update(
cacheUpdates, request_.ledger().sequence(), cacheOnly);
log_.debug() << "Wrote " << numObjects
<< " objects. Got more: " << (more ? "YES" : "NO");
backend.cache().update(cacheUpdates, request_.ledger().sequence(), cacheOnly);
log_.debug() << "Wrote " << numObjects << " objects. Got more: " << (more ? "YES" : "NO");
return more ? CallStatus::MORE : CallStatus::DONE;
}
void
call(
std::unique_ptr<org::xrpl::rpc::v1::XRPLedgerAPIService::Stub>& stub,
grpc::CompletionQueue& cq)
call(std::unique_ptr<org::xrpl::rpc::v1::XRPLedgerAPIService::Stub>& stub, grpc::CompletionQueue& cq)
{
context_ = std::make_unique<grpc::ClientContext>();
std::unique_ptr<grpc::ClientAsyncResponseReader<
org::xrpl::rpc::v1::GetLedgerDataResponse>>
rpc(stub->PrepareAsyncGetLedgerData(context_.get(), request_, &cq));
std::unique_ptr<grpc::ClientAsyncResponseReader<org::xrpl::rpc::v1::GetLedgerDataResponse>> rpc(
stub->PrepareAsyncGetLedgerData(context_.get(), request_, &cq));
rpc->StartCall();
@@ -734,10 +662,7 @@ public:
template <class Derived>
bool
ETLSourceImpl<Derived>::loadInitialLedger(
uint32_t sequence,
uint32_t numMarkers,
bool cacheOnly)
ETLSourceImpl<Derived>::loadInitialLedger(uint32_t sequence, uint32_t numMarkers, bool cacheOnly)
{
if (!stub_)
return false;
@@ -759,8 +684,7 @@ ETLSourceImpl<Derived>::loadInitialLedger(
calls.emplace_back(sequence, markers[i], nextMarker);
}
log_.debug() << "Starting data download for ledger " << sequence
<< ". Using source = " << toString();
log_.debug() << "Starting data download for ledger " << sequence << ". Using source = " << toString();
for (auto& c : calls)
c.call(stub_, cq);
@@ -801,14 +725,12 @@ ETLSourceImpl<Derived>::loadInitialLedger(
}
if (backend_->cache().size() > progress)
{
log_.info() << "Downloaded " << backend_->cache().size()
<< " records from rippled";
log_.info() << "Downloaded " << backend_->cache().size() << " records from rippled";
progress += incr;
}
}
}
log_.info() << "Finished loadInitialLedger. cache size = "
<< backend_->cache().size();
log_.info() << "Finished loadInitialLedger. cache size = " << backend_->cache().size();
size_t numWrites = 0;
if (!abort)
{
@@ -818,27 +740,18 @@ ETLSourceImpl<Derived>::loadInitialLedger(
auto seconds = util::timed<std::chrono::seconds>([&]() {
for (auto& key : edgeKeys)
{
log_.debug()
<< "Writing edge key = " << ripple::strHex(key);
auto succ = backend_->cache().getSuccessor(
*ripple::uint256::fromVoidChecked(key), sequence);
log_.debug() << "Writing edge key = " << ripple::strHex(key);
auto succ = backend_->cache().getSuccessor(*ripple::uint256::fromVoidChecked(key), sequence);
if (succ)
backend_->writeSuccessor(
std::move(key),
sequence,
uint256ToString(succ->key));
backend_->writeSuccessor(std::move(key), sequence, uint256ToString(succ->key));
}
ripple::uint256 prev = Backend::firstKey;
while (auto cur =
backend_->cache().getSuccessor(prev, sequence))
while (auto cur = backend_->cache().getSuccessor(prev, sequence))
{
assert(cur);
if (prev == Backend::firstKey)
{
backend_->writeSuccessor(
uint256ToString(prev),
sequence,
uint256ToString(cur->key));
backend_->writeSuccessor(uint256ToString(prev), sequence, uint256ToString(cur->key));
}
if (isBookDir(cur->key, cur->blob))
@@ -847,40 +760,29 @@ ETLSourceImpl<Derived>::loadInitialLedger(
// make sure the base is not an actual object
if (!backend_->cache().get(cur->key, sequence))
{
auto succ =
backend_->cache().getSuccessor(base, sequence);
auto succ = backend_->cache().getSuccessor(base, sequence);
assert(succ);
if (succ->key == cur->key)
{
log_.debug() << "Writing book successor = "
<< ripple::strHex(base) << " - "
log_.debug() << "Writing book successor = " << ripple::strHex(base) << " - "
<< ripple::strHex(cur->key);
backend_->writeSuccessor(
uint256ToString(base),
sequence,
uint256ToString(cur->key));
backend_->writeSuccessor(uint256ToString(base), sequence, uint256ToString(cur->key));
}
}
++numWrites;
}
prev = std::move(cur->key);
if (numWrites % 100000 == 0 && numWrites != 0)
log_.info()
<< "Wrote " << numWrites << " book successors";
log_.info() << "Wrote " << numWrites << " book successors";
}
backend_->writeSuccessor(
uint256ToString(prev),
sequence,
uint256ToString(Backend::lastKey));
backend_->writeSuccessor(uint256ToString(prev), sequence, uint256ToString(Backend::lastKey));
++numWrites;
});
log_.info()
<< "Looping through cache and submitting all writes took "
<< seconds
<< " seconds. numWrites = " << std::to_string(numWrites);
log_.info() << "Looping through cache and submitting all writes took " << seconds
<< " seconds. numWrites = " << std::to_string(numWrites);
}
}
return !abort;
@@ -888,10 +790,7 @@ ETLSourceImpl<Derived>::loadInitialLedger(
template <class Derived>
std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>
ETLSourceImpl<Derived>::fetchLedger(
uint32_t ledgerSequence,
bool getObjects,
bool getObjectNeighbors)
ETLSourceImpl<Derived>::fetchLedger(uint32_t ledgerSequence, bool getObjects, bool getObjectNeighbors)
{
org::xrpl::rpc::v1::GetLedgerResponse response;
if (!stub_)
@@ -927,12 +826,7 @@ make_ETLSource(
ETLLoadBalancer& balancer)
{
auto src = std::make_unique<ProbingETLSource>(
config,
ioContext,
backend,
subscriptions,
networkValidatedLedgers,
balancer);
config, ioContext, backend, subscriptions, networkValidatedLedgers, balancer);
src->run();
@@ -953,8 +847,7 @@ ETLLoadBalancer::ETLLoadBalancer(
for (auto const& entry : config.array("etl_sources"))
{
std::unique_ptr<ETLSource> source = make_ETLSource(
entry, ioContext, backend, subscriptions, nwvl, *this);
std::unique_ptr<ETLSource> source = make_ETLSource(entry, ioContext, backend, subscriptions, nwvl, *this);
sources_.push_back(std::move(source));
log_.info() << "Added etl source - " << sources_.back()->toString();
@@ -966,13 +859,11 @@ ETLLoadBalancer::loadInitialLedger(uint32_t sequence, bool cacheOnly)
{
execute(
[this, &sequence, cacheOnly](auto& source) {
bool res =
source->loadInitialLedger(sequence, downloadRanges_, cacheOnly);
bool res = source->loadInitialLedger(sequence, downloadRanges_, cacheOnly);
if (!res)
{
log_.error() << "Failed to download initial ledger."
<< " Sequence = " << sequence
<< " source = " << source->toString();
<< " Sequence = " << sequence << " source = " << source->toString();
}
return res;
},
@@ -980,17 +871,12 @@ ETLLoadBalancer::loadInitialLedger(uint32_t sequence, bool cacheOnly)
}
std::optional<org::xrpl::rpc::v1::GetLedgerResponse>
ETLLoadBalancer::fetchLedger(
uint32_t ledgerSequence,
bool getObjects,
bool getObjectNeighbors)
ETLLoadBalancer::fetchLedger(uint32_t ledgerSequence, bool getObjects, bool getObjectNeighbors)
{
org::xrpl::rpc::v1::GetLedgerResponse response;
bool success = execute(
[&response, ledgerSequence, getObjects, getObjectNeighbors, log = log_](
auto& source) {
auto [status, data] = source->fetchLedger(
ledgerSequence, getObjects, getObjectNeighbors);
[&response, ledgerSequence, getObjects, getObjectNeighbors, log = log_](auto& source) {
auto [status, data] = source->fetchLedger(ledgerSequence, getObjects, getObjectNeighbors);
response = std::move(data);
if (status.ok() && response.validated())
{
@@ -1000,10 +886,8 @@ ETLLoadBalancer::fetchLedger(
}
else
{
log.warn() << "Could not fetch ledger " << ledgerSequence
<< ", Reply: " << response.DebugString()
<< ", error_code: " << status.error_code()
<< ", error_msg: " << status.error_message()
log.warn() << "Could not fetch ledger " << ledgerSequence << ", Reply: " << response.DebugString()
<< ", error_code: " << status.error_code() << ", error_msg: " << status.error_message()
<< ", source = " << source->toString();
return false;
}
@@ -1026,8 +910,7 @@ ETLLoadBalancer::forwardToRippled(
auto numAttempts = 0;
while (numAttempts < sources_.size())
{
if (auto res =
sources_[sourceIdx]->forwardToRippled(request, clientIp, yield))
if (auto res = sources_[sourceIdx]->forwardToRippled(request, clientIp, yield))
return res;
sourceIdx = (sourceIdx + 1) % sources_.size();
@@ -1100,14 +983,10 @@ ETLSourceImpl<Derived>::requestFromRippled(
// resources. See "secure_gateway" in
//
// https://github.com/ripple/rippled/blob/develop/cfg/rippled-example.cfg
ws->set_option(websocket::stream_base::decorator(
[&clientIp](websocket::request_type& req) {
req.set(
http::field::user_agent,
std::string(BOOST_BEAST_VERSION_STRING) +
" websocket-client-coro");
req.set(http::field::forwarded, "for=" + clientIp);
}));
ws->set_option(websocket::stream_base::decorator([&clientIp](websocket::request_type& req) {
req.set(http::field::user_agent, std::string(BOOST_BEAST_VERSION_STRING) + " websocket-client-coro");
req.set(http::field::forwarded, "for=" + clientIp);
}));
log_.trace() << "client ip: " << clientIp;
log_.trace() << "Performing websocket handshake";
@@ -1118,8 +997,7 @@ ETLSourceImpl<Derived>::requestFromRippled(
log_.trace() << "Sending request";
// Send the message
ws->async_write(
net::buffer(boost::json::serialize(request)), yield[ec]);
ws->async_write(net::buffer(boost::json::serialize(request)), yield[ec]);
if (ec)
return {};
@@ -1134,8 +1012,7 @@ ETLSourceImpl<Derived>::requestFromRippled(
if (!parsed.is_object())
{
log_.error() << "Error parsing response: "
<< std::string{begin, end};
log_.error() << "Error parsing response: " << std::string{begin, end};
return {};
}
log_.trace() << "Successfully forward request";
@@ -1164,8 +1041,8 @@ ETLLoadBalancer::execute(Func f, uint32_t ledgerSequence)
{
auto& source = sources_[sourceIdx];
log_.debug() << "Attempting to execute func. ledger sequence = "
<< ledgerSequence << " - source = " << source->toString();
log_.debug() << "Attempting to execute func. ledger sequence = " << ledgerSequence
<< " - source = " << source->toString();
// Originally, it was (source->hasLedger(ledgerSequence) || true)
/* Sometimes rippled has ledger but doesn't actually know. However,
but this does NOT happen in the normal case and is safe to remove
@@ -1175,30 +1052,26 @@ ETLLoadBalancer::execute(Func f, uint32_t ledgerSequence)
bool res = f(source);
if (res)
{
log_.debug() << "Successfully executed func at source = "
<< source->toString()
log_.debug() << "Successfully executed func at source = " << source->toString()
<< " - ledger sequence = " << ledgerSequence;
break;
}
else
{
log_.warn() << "Failed to execute func at source = "
<< source->toString()
log_.warn() << "Failed to execute func at source = " << source->toString()
<< " - ledger sequence = " << ledgerSequence;
}
}
else
{
log_.warn() << "Ledger not present at source = "
<< source->toString()
log_.warn() << "Ledger not present at source = " << source->toString()
<< " - ledger sequence = " << ledgerSequence;
}
sourceIdx = (sourceIdx + 1) % sources_.size();
numAttempts++;
if (numAttempts % sources_.size() == 0)
{
log_.info() << "Ledger sequence " << ledgerSequence
<< " is not yet available from any configured sources. "
log_.info() << "Ledger sequence " << ledgerSequence << " is not yet available from any configured sources. "
<< "Sleeping and trying again";
std::this_thread::sleep_for(std::chrono::seconds(2));
}

141
src/etl/ETLSource.h
View File

@@ -67,26 +67,20 @@ class ForwardCache
clear();
public:
ForwardCache(
clio::Config const& config,
boost::asio::io_context& ioc,
ETLSource const& source)
ForwardCache(clio::Config const& config, boost::asio::io_context& ioc, ETLSource const& source)
: strand_(ioc), timer_(strand_), source_(source)
{
if (config.contains("cache"))
{
auto commands =
config.arrayOrThrow("cache", "ETLSource cache must be array");
auto commands = config.arrayOrThrow("cache", "ETLSource cache must be array");
if (config.contains("cache_duration"))
duration_ = config.valueOrThrow<uint32_t>(
"cache_duration",
"ETLSource cache_duration must be a number");
duration_ =
config.valueOrThrow<uint32_t>("cache_duration", "ETLSource cache_duration must be a number");
for (auto const& command : commands)
{
auto key = command.valueOrThrow<std::string>(
"ETLSource forward command must be array of strings");
auto key = command.valueOrThrow<std::string>("ETLSource forward command must be array of strings");
latestForwarded_[key] = {};
}
}
@@ -128,22 +122,14 @@ public:
hasLedger(uint32_t sequence) const = 0;
virtual std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>
fetchLedger(
uint32_t ledgerSequence,
bool getObjects = true,
bool getObjectNeighbors = false) = 0;
fetchLedger(uint32_t ledgerSequence, bool getObjects = true, bool getObjectNeighbors = false) = 0;
virtual bool
loadInitialLedger(
uint32_t sequence,
std::uint32_t numMarkers,
bool cacheOnly = false) = 0;
loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, bool cacheOnly = false) = 0;
virtual std::optional<boost::json::object>
forwardToRippled(
boost::json::object const& request,
std::string const& clientIp,
boost::asio::yield_context& yield) const = 0;
forwardToRippled(boost::json::object const& request, std::string const& clientIp, boost::asio::yield_context& yield)
const = 0;
virtual boost::uuids::uuid
token() const = 0;
@@ -258,9 +244,7 @@ protected:
auto const host = ip_;
auto const port = wsPort_;
resolver_.async_resolve(host, port, [this](auto ec, auto results) {
onResolve(ec, results);
});
resolver_.async_resolve(host, port, [this](auto ec, auto results) { onResolve(ec, results); });
}
public:
@@ -327,21 +311,18 @@ public:
grpcPort_ = *value;
try
{
boost::asio::ip::tcp::endpoint endpoint{
boost::asio::ip::make_address(ip_), std::stoi(grpcPort_)};
boost::asio::ip::tcp::endpoint endpoint{boost::asio::ip::make_address(ip_), std::stoi(grpcPort_)};
std::stringstream ss;
ss << endpoint;
grpc::ChannelArguments chArgs;
chArgs.SetMaxReceiveMessageSize(-1);
stub_ = org::xrpl::rpc::v1::XRPLedgerAPIService::NewStub(
grpc::CreateCustomChannel(
ss.str(), grpc::InsecureChannelCredentials(), chArgs));
grpc::CreateCustomChannel(ss.str(), grpc::InsecureChannelCredentials(), chArgs));
log_.debug() << "Made stub for remote = " << toString();
}
catch (std::exception const& e)
{
log_.debug() << "Exception while creating stub = " << e.what()
<< " . Remote = " << toString();
log_.debug() << "Exception while creating stub = " << e.what() << " . Remote = " << toString();
}
}
}
@@ -397,9 +378,7 @@ public:
pairs.push_back(std::make_pair(min, max));
}
}
std::sort(pairs.begin(), pairs.end(), [](auto left, auto right) {
return left.first < right.first;
});
std::sort(pairs.begin(), pairs.end(), [](auto left, auto right) { return left.first < right.first; });
// we only hold the lock here, to avoid blocking while string processing
std::lock_guard lck(mtx_);
@@ -422,16 +401,13 @@ public:
/// and the prior one
/// @return the extracted data and the result status
std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>
fetchLedger(
uint32_t ledgerSequence,
bool getObjects = true,
bool getObjectNeighbors = false) override;
fetchLedger(uint32_t ledgerSequence, bool getObjects = true, bool getObjectNeighbors = false) override;
std::string
toString() const override
{
return "{validated_ledger: " + getValidatedRange() + ", ip: " + ip_ +
", web socket port: " + wsPort_ + ", grpc port: " + grpcPort_ + "}";
return "{validated_ledger: " + getValidatedRange() + ", ip: " + ip_ + ", web socket port: " + wsPort_ +
", grpc port: " + grpcPort_ + "}";
}
boost::json::object
@@ -446,8 +422,7 @@ public:
auto last = getLastMsgTime();
if (last.time_since_epoch().count() != 0)
res["last_msg_age_seconds"] = std::to_string(
std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now() - getLastMsgTime())
std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now() - getLastMsgTime())
.count());
return res;
}
@@ -457,10 +432,7 @@ public:
/// @param writeQueue queue to push downloaded ledger objects
/// @return true if the download was successful
bool
loadInitialLedger(
std::uint32_t ledgerSequence,
std::uint32_t numMarkers,
bool cacheOnly = false) override;
loadInitialLedger(std::uint32_t ledgerSequence, std::uint32_t numMarkers, bool cacheOnly = false) override;
/// Attempt to reconnect to the ETL source
void
@@ -484,16 +456,11 @@ public:
/// Callback
void
onResolve(
boost::beast::error_code ec,
boost::asio::ip::tcp::resolver::results_type results);
onResolve(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type results);
/// Callback
virtual void
onConnect(
boost::beast::error_code ec,
boost::asio::ip::tcp::resolver::results_type::endpoint_type
endpoint) = 0;
onConnect(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint) = 0;
/// Callback
void
@@ -513,16 +480,13 @@ public:
handleMessage();
std::optional<boost::json::object>
forwardToRippled(
boost::json::object const& request,
std::string const& clientIp,
boost::asio::yield_context& yield) const override;
forwardToRippled(boost::json::object const& request, std::string const& clientIp, boost::asio::yield_context& yield)
const override;
};
class PlainETLSource : public ETLSourceImpl<PlainETLSource>
{
std::unique_ptr<boost::beast::websocket::stream<boost::beast::tcp_stream>>
ws_;
std::unique_ptr<boost::beast::websocket::stream<boost::beast::tcp_stream>> ws_;
public:
PlainETLSource(
@@ -533,24 +497,14 @@ public:
std::shared_ptr<NetworkValidatedLedgers> nwvl,
ETLLoadBalancer& balancer,
ETLSourceHooks hooks)
: ETLSourceImpl(
config,
ioc,
backend,
subscriptions,
nwvl,
balancer,
std::move(hooks))
, ws_(std::make_unique<
boost::beast::websocket::stream<boost::beast::tcp_stream>>(
: ETLSourceImpl(config, ioc, backend, subscriptions, nwvl, balancer, std::move(hooks))
, ws_(std::make_unique<boost::beast::websocket::stream<boost::beast::tcp_stream>>(
boost::asio::make_strand(ioc)))
{
}
void
onConnect(
boost::beast::error_code ec,
boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint)
onConnect(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint)
override;
/// Close the websocket
@@ -569,9 +523,7 @@ class SslETLSource : public ETLSourceImpl<SslETLSource>
{
std::optional<std::reference_wrapper<boost::asio::ssl::context>> sslCtx_;
std::unique_ptr<boost::beast::websocket::stream<
boost::beast::ssl_stream<boost::beast::tcp_stream>>>
ws_;
std::unique_ptr<boost::beast::websocket::stream<boost::beast::ssl_stream<boost::beast::tcp_stream>>> ws_;
public:
SslETLSource(
@@ -583,40 +535,27 @@ public:
std::shared_ptr<NetworkValidatedLedgers> nwvl,
ETLLoadBalancer& balancer,
ETLSourceHooks hooks)
: ETLSourceImpl(
config,
ioc,
backend,
subscriptions,
nwvl,
balancer,
std::move(hooks))
: ETLSourceImpl(config, ioc, backend, subscriptions, nwvl, balancer, std::move(hooks))
, sslCtx_(sslCtx)
, ws_(std::make_unique<boost::beast::websocket::stream<
boost::beast::ssl_stream<boost::beast::tcp_stream>>>(
, ws_(std::make_unique<boost::beast::websocket::stream<boost::beast::ssl_stream<boost::beast::tcp_stream>>>(
boost::asio::make_strand(ioc_),
*sslCtx_))
{
}
void
onConnect(
boost::beast::error_code ec,
boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint)
onConnect(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint)
override;
void
onSslHandshake(
boost::beast::error_code ec,
boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint);
onSslHandshake(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint);
/// Close the websocket
/// @param startAgain whether to reconnect
void
close(bool startAgain);
boost::beast::websocket::stream<
boost::beast::ssl_stream<boost::beast::tcp_stream>>&
boost::beast::websocket::stream<boost::beast::ssl_stream<boost::beast::tcp_stream>>&
ws()
{
return *ws_;
@@ -652,8 +591,7 @@ public:
std::shared_ptr<SubscriptionManager> subscriptions,
std::shared_ptr<NetworkValidatedLedgers> validatedLedgers)
{
return std::make_shared<ETLLoadBalancer>(
config, ioc, backend, subscriptions, validatedLedgers);
return std::make_shared<ETLLoadBalancer>(config, ioc, backend, subscriptions, validatedLedgers);
}
~ETLLoadBalancer()
@@ -676,10 +614,7 @@ public:
/// was found in the database or the server is shutting down, the optional
/// will be empty
std::optional<org::xrpl::rpc::v1::GetLedgerResponse>
fetchLedger(
uint32_t ledgerSequence,
bool getObjects,
bool getObjectNeighbors);
fetchLedger(uint32_t ledgerSequence, bool getObjects, bool getObjectNeighbors);
/// Determine whether messages received on the transactions_proposed stream
/// should be forwarded to subscribing clients. The server subscribes to
@@ -720,10 +655,8 @@ public:
/// @param request JSON-RPC request
/// @return response received from rippled node
std::optional<boost::json::object>
forwardToRippled(
boost::json::object const& request,
std::string const& clientIp,
boost::asio::yield_context& yield) const;
forwardToRippled(boost::json::object const& request, std::string const& clientIp, boost::asio::yield_context& yield)
const;
private:
/// f is a function that takes an ETLSource as an argument and returns a

206
src/etl/NFTHelpers.cpp
View File

@@ -44,25 +44,18 @@ getNFTokenMintData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
for (ripple::STObject const& node : txMeta.getNodes())
{
if (node.getFieldU16(ripple::sfLedgerEntryType) !=
ripple::ltNFTOKEN_PAGE)
if (node.getFieldU16(ripple::sfLedgerEntryType) != ripple::ltNFTOKEN_PAGE)
continue;
if (!owner)
owner = ripple::AccountID::fromVoid(
node.getFieldH256(ripple::sfLedgerIndex).data());
owner = ripple::AccountID::fromVoid(node.getFieldH256(ripple::sfLedgerIndex).data());
if (node.getFName() == ripple::sfCreatedNode)
{
ripple::STArray const& toAddNFTs =
node.peekAtField(ripple::sfNewFields)
.downcast<ripple::STObject>()
.getFieldArray(ripple::sfNFTokens);
node.peekAtField(ripple::sfNewFields).downcast<ripple::STObject>().getFieldArray(ripple::sfNFTokens);
std::transform(
toAddNFTs.begin(),
toAddNFTs.end(),
std::back_inserter(finalIDs),
[](ripple::STObject const& nft) {
toAddNFTs.begin(), toAddNFTs.end(), std::back_inserter(finalIDs), [](ripple::STObject const& nft) {
return nft.getFieldH256(ripple::sfNFTokenID);
});
}
@@ -80,32 +73,23 @@ getNFTokenMintData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
// as rippled outputs all fields in final fields even if they were
// not changed.
ripple::STObject const& previousFields =
node.peekAtField(ripple::sfPreviousFields)
.downcast<ripple::STObject>();
node.peekAtField(ripple::sfPreviousFields).downcast<ripple::STObject>();
if (!previousFields.isFieldPresent(ripple::sfNFTokens))
continue;
ripple::STArray const& toAddNFTs =
previousFields.getFieldArray(ripple::sfNFTokens);
ripple::STArray const& toAddNFTs = previousFields.getFieldArray(ripple::sfNFTokens);
std::transform(
toAddNFTs.begin(),
toAddNFTs.end(),
std::back_inserter(prevIDs),
[](ripple::STObject const& nft) {
toAddNFTs.begin(), toAddNFTs.end(), std::back_inserter(prevIDs), [](ripple::STObject const& nft) {
return nft.getFieldH256(ripple::sfNFTokenID);
});
ripple::STArray const& toAddFinalNFTs =
node.peekAtField(ripple::sfFinalFields)
.downcast<ripple::STObject>()
.getFieldArray(ripple::sfNFTokens);
node.peekAtField(ripple::sfFinalFields).downcast<ripple::STObject>().getFieldArray(ripple::sfNFTokens);
std::transform(
toAddFinalNFTs.begin(),
toAddFinalNFTs.end(),
std::back_inserter(finalIDs),
[](ripple::STObject const& nft) {
return nft.getFieldH256(ripple::sfNFTokenID);
});
[](ripple::STObject const& nft) { return nft.getFieldH256(ripple::sfNFTokenID); });
}
}
@@ -120,13 +104,8 @@ getNFTokenMintData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
std::inserter(tokenIDResult, tokenIDResult.begin()));
if (tokenIDResult.size() == 1 && owner)
return {
{NFTTransactionsData(
tokenIDResult.front(), txMeta, sttx.getTransactionID())},
NFTsData(
tokenIDResult.front(),
*owner,
sttx.getFieldVL(ripple::sfURI),
txMeta)};
{NFTTransactionsData(tokenIDResult.front(), txMeta, sttx.getTransactionID())},
NFTsData(tokenIDResult.front(), *owner, sttx.getFieldVL(ripple::sfURI), txMeta)};
std::stringstream msg;
msg << " - unexpected NFTokenMint data in tx " << sttx.getTransactionID();
@@ -137,16 +116,14 @@ std::pair<std::vector<NFTTransactionsData>, std::optional<NFTsData>>
getNFTokenBurnData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
{
ripple::uint256 const tokenID = sttx.getFieldH256(ripple::sfNFTokenID);
std::vector<NFTTransactionsData> const txs = {
NFTTransactionsData(tokenID, txMeta, sttx.getTransactionID())};
std::vector<NFTTransactionsData> const txs = {NFTTransactionsData(tokenID, txMeta, sttx.getTransactionID())};
// Determine who owned the token when it was burned by finding an
// NFTokenPage that was deleted or modified that contains this
// tokenID.
for (ripple::STObject const& node : txMeta.getNodes())
{
if (node.getFieldU16(ripple::sfLedgerEntryType) !=
ripple::ltNFTOKEN_PAGE ||
if (node.getFieldU16(ripple::sfLedgerEntryType) != ripple::ltNFTOKEN_PAGE ||
node.getFName() == ripple::sfCreatedNode)
continue;
@@ -163,23 +140,19 @@ getNFTokenBurnData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
if (node.isFieldPresent(ripple::sfPreviousFields))
{
ripple::STObject const& previousFields =
node.peekAtField(ripple::sfPreviousFields)
.downcast<ripple::STObject>();
node.peekAtField(ripple::sfPreviousFields).downcast<ripple::STObject>();
if (previousFields.isFieldPresent(ripple::sfNFTokens))
prevNFTs = previousFields.getFieldArray(ripple::sfNFTokens);
}
else if (!prevNFTs && node.getFName() == ripple::sfDeletedNode)
prevNFTs = node.peekAtField(ripple::sfFinalFields)
.downcast<ripple::STObject>()
.getFieldArray(ripple::sfNFTokens);
prevNFTs =
node.peekAtField(ripple::sfFinalFields).downcast<ripple::STObject>().getFieldArray(ripple::sfNFTokens);
if (!prevNFTs)
continue;
auto const nft = std::find_if(
prevNFTs->begin(),
prevNFTs->end(),
[&tokenID](ripple::STObject const& candidate) {
auto const nft =
std::find_if(prevNFTs->begin(), prevNFTs->end(), [&tokenID](ripple::STObject const& candidate) {
return candidate.getFieldH256(ripple::sfNFTokenID) == tokenID;
});
if (nft != prevNFTs->end())
@@ -187,92 +160,74 @@ getNFTokenBurnData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
txs,
NFTsData(
tokenID,
ripple::AccountID::fromVoid(
node.getFieldH256(ripple::sfLedgerIndex).data()),
ripple::AccountID::fromVoid(node.getFieldH256(ripple::sfLedgerIndex).data()),
txMeta,
true));
}
std::stringstream msg;
msg << " - could not determine owner at burntime for tx "
<< sttx.getTransactionID();
msg << " - could not determine owner at burntime for tx " << sttx.getTransactionID();
throw std::runtime_error(msg.str());
}
std::pair<std::vector<NFTTransactionsData>, std::optional<NFTsData>>
getNFTokenAcceptOfferData(
ripple::TxMeta const& txMeta,
ripple::STTx const& sttx)
getNFTokenAcceptOfferData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
{
// If we have the buy offer from this tx, we can determine the owner
// more easily by just looking at the owner of the accepted NFTokenOffer
// object.
if (sttx.isFieldPresent(ripple::sfNFTokenBuyOffer))
{
auto const affectedBuyOffer = std::find_if(
txMeta.getNodes().begin(),
txMeta.getNodes().end(),
[&sttx](ripple::STObject const& node) {
return node.getFieldH256(ripple::sfLedgerIndex) ==
sttx.getFieldH256(ripple::sfNFTokenBuyOffer);
auto const affectedBuyOffer =
std::find_if(txMeta.getNodes().begin(), txMeta.getNodes().end(), [&sttx](ripple::STObject const& node) {
return node.getFieldH256(ripple::sfLedgerIndex) == sttx.getFieldH256(ripple::sfNFTokenBuyOffer);
});
if (affectedBuyOffer == txMeta.getNodes().end())
{
std::stringstream msg;
msg << " - unexpected NFTokenAcceptOffer data in tx "
<< sttx.getTransactionID();
msg << " - unexpected NFTokenAcceptOffer data in tx " << sttx.getTransactionID();
throw std::runtime_error(msg.str());
}
ripple::uint256 const tokenID =
affectedBuyOffer->peekAtField(ripple::sfFinalFields)
.downcast<ripple::STObject>()
.getFieldH256(ripple::sfNFTokenID);
ripple::uint256 const tokenID = affectedBuyOffer->peekAtField(ripple::sfFinalFields)
.downcast<ripple::STObject>()
.getFieldH256(ripple::sfNFTokenID);
ripple::AccountID const owner =
affectedBuyOffer->peekAtField(ripple::sfFinalFields)
.downcast<ripple::STObject>()
.getAccountID(ripple::sfOwner);
ripple::AccountID const owner = affectedBuyOffer->peekAtField(ripple::sfFinalFields)
.downcast<ripple::STObject>()
.getAccountID(ripple::sfOwner);
return {
{NFTTransactionsData(tokenID, txMeta, sttx.getTransactionID())},
NFTsData(tokenID, owner, txMeta, false)};
{NFTTransactionsData(tokenID, txMeta, sttx.getTransactionID())}, NFTsData(tokenID, owner, txMeta, false)};
}
// Otherwise we have to infer the new owner from the affected nodes.
auto const affectedSellOffer = std::find_if(
txMeta.getNodes().begin(),
txMeta.getNodes().end(),
[&sttx](ripple::STObject const& node) {
return node.getFieldH256(ripple::sfLedgerIndex) ==
sttx.getFieldH256(ripple::sfNFTokenSellOffer);
auto const affectedSellOffer =
std::find_if(txMeta.getNodes().begin(), txMeta.getNodes().end(), [&sttx](ripple::STObject const& node) {
return node.getFieldH256(ripple::sfLedgerIndex) == sttx.getFieldH256(ripple::sfNFTokenSellOffer);
});
if (affectedSellOffer == txMeta.getNodes().end())
{
std::stringstream msg;
msg << " - unexpected NFTokenAcceptOffer data in tx "
<< sttx.getTransactionID();
msg << " - unexpected NFTokenAcceptOffer data in tx " << sttx.getTransactionID();
throw std::runtime_error(msg.str());
}
ripple::uint256 const tokenID =
affectedSellOffer->peekAtField(ripple::sfFinalFields)
.downcast<ripple::STObject>()
.getFieldH256(ripple::sfNFTokenID);
ripple::uint256 const tokenID = affectedSellOffer->peekAtField(ripple::sfFinalFields)
.downcast<ripple::STObject>()
.getFieldH256(ripple::sfNFTokenID);
ripple::AccountID const seller =
affectedSellOffer->peekAtField(ripple::sfFinalFields)
.downcast<ripple::STObject>()
.getAccountID(ripple::sfOwner);
ripple::AccountID const seller = affectedSellOffer->peekAtField(ripple::sfFinalFields)
.downcast<ripple::STObject>()
.getAccountID(ripple::sfOwner);
for (ripple::STObject const& node : txMeta.getNodes())
{
if (node.getFieldU16(ripple::sfLedgerEntryType) !=
ripple::ltNFTOKEN_PAGE ||
if (node.getFieldU16(ripple::sfLedgerEntryType) != ripple::ltNFTOKEN_PAGE ||
node.getFName() == ripple::sfDeletedNode)
continue;
ripple::AccountID const nodeOwner = ripple::AccountID::fromVoid(
node.getFieldH256(ripple::sfLedgerIndex).data());
ripple::AccountID const nodeOwner =
ripple::AccountID::fromVoid(node.getFieldH256(ripple::sfLedgerIndex).data());
if (nodeOwner == seller)
continue;
@@ -286,12 +241,9 @@ getNFTokenAcceptOfferData(
.getFieldArray(ripple::sfNFTokens);
}();
auto const nft = std::find_if(
nfts.begin(),
nfts.end(),
[&tokenID](ripple::STObject const& candidate) {
return candidate.getFieldH256(ripple::sfNFTokenID) == tokenID;
});
auto const nft = std::find_if(nfts.begin(), nfts.end(), [&tokenID](ripple::STObject const& candidate) {
return candidate.getFieldH256(ripple::sfNFTokenID) == tokenID;
});
if (nft != nfts.end())
return {
{NFTTransactionsData(tokenID, txMeta, sttx.getTransactionID())},
@@ -299,8 +251,7 @@ getNFTokenAcceptOfferData(
}
std::stringstream msg;
msg << " - unexpected NFTokenAcceptOffer data in tx "
<< sttx.getTransactionID();
msg << " - unexpected NFTokenAcceptOffer data in tx " << sttx.getTransactionID();
throw std::runtime_error(msg.str());
}
@@ -309,40 +260,28 @@ getNFTokenAcceptOfferData(
// transaction using this feature. This transaction also never returns an
// NFTsData because it does not change the state of an NFT itself.
std::pair<std::vector<NFTTransactionsData>, std::optional<NFTsData>>
getNFTokenCancelOfferData(
ripple::TxMeta const& txMeta,
ripple::STTx const& sttx)
getNFTokenCancelOfferData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
{
std::vector<NFTTransactionsData> txs;
for (ripple::STObject const& node : txMeta.getNodes())
{
if (node.getFieldU16(ripple::sfLedgerEntryType) !=
ripple::ltNFTOKEN_OFFER)
if (node.getFieldU16(ripple::sfLedgerEntryType) != ripple::ltNFTOKEN_OFFER)
continue;
ripple::uint256 const tokenID = node.peekAtField(ripple::sfFinalFields)
.downcast<ripple::STObject>()
.getFieldH256(ripple::sfNFTokenID);
ripple::uint256 const tokenID =
node.peekAtField(ripple::sfFinalFields).downcast<ripple::STObject>().getFieldH256(ripple::sfNFTokenID);
txs.emplace_back(tokenID, txMeta, sttx.getTransactionID());
}
// Deduplicate any transactions based on tokenID/txIdx combo. Can't just
// use txIdx because in this case one tx can cancel offers for several
// NFTs.
std::sort(
txs.begin(),
txs.end(),
[](NFTTransactionsData const& a, NFTTransactionsData const& b) {
return a.tokenID < b.tokenID &&
a.transactionIndex < b.transactionIndex;
});
auto last = std::unique(
txs.begin(),
txs.end(),
[](NFTTransactionsData const& a, NFTTransactionsData const& b) {
return a.tokenID == b.tokenID &&
a.transactionIndex == b.transactionIndex;
});
std::sort(txs.begin(), txs.end(), [](NFTTransactionsData const& a, NFTTransactionsData const& b) {
return a.tokenID < b.tokenID && a.transactionIndex < b.transactionIndex;
});
auto last = std::unique(txs.begin(), txs.end(), [](NFTTransactionsData const& a, NFTTransactionsData const& b) {
return a.tokenID == b.tokenID && a.transactionIndex == b.transactionIndex;
});
txs.erase(last, txs.end());
return {txs, {}};
}
@@ -350,16 +289,9 @@ getNFTokenCancelOfferData(
// This transaction never returns an NFTokensData because it does not
// change the state of an NFT itself.
std::pair<std::vector<NFTTransactionsData>, std::optional<NFTsData>>
getNFTokenCreateOfferData(
ripple::TxMeta const& txMeta,
ripple::STTx const& sttx)
getNFTokenCreateOfferData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
{
return {
{NFTTransactionsData(
sttx.getFieldH256(ripple::sfNFTokenID),
txMeta,
sttx.getTransactionID())},
{}};
return {{NFTTransactionsData(sttx.getFieldH256(ripple::sfNFTokenID), txMeta, sttx.getTransactionID())}, {}};
}
std::pair<std::vector<NFTTransactionsData>, std::optional<NFTsData>>
@@ -391,26 +323,18 @@ getNFTDataFromTx(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
}
std::vector<NFTsData>
getNFTDataFromObj(
std::uint32_t const seq,
std::string const& key,
std::string const& blob)
getNFTDataFromObj(std::uint32_t const seq, std::string const& key, std::string const& blob)
{
std::vector<NFTsData> nfts;
ripple::STLedgerEntry const sle = ripple::STLedgerEntry(
ripple::SerialIter{blob.data(), blob.size()},
ripple::uint256::fromVoid(key.data()));
ripple::STLedgerEntry const sle =
ripple::STLedgerEntry(ripple::SerialIter{blob.data(), blob.size()}, ripple::uint256::fromVoid(key.data()));
if (sle.getFieldU16(ripple::sfLedgerEntryType) != ripple::ltNFTOKEN_PAGE)
return nfts;
auto const owner = ripple::AccountID::fromVoid(key.data());
for (ripple::STObject const& node : sle.getFieldArray(ripple::sfNFTokens))
nfts.emplace_back(
node.getFieldH256(ripple::sfNFTokenID),
seq,
owner,
node.getFieldVL(ripple::sfURI));
nfts.emplace_back(node.getFieldH256(ripple::sfNFTokenID), seq, owner, node.getFieldVL(ripple::sfURI));
return nfts;
}

5
src/etl/NFTHelpers.h
View File

@@ -30,7 +30,4 @@ getNFTDataFromTx(ripple::TxMeta const& txMeta, ripple::STTx const& sttx);
// Pulling from ledger object via loadInitialLedger
std::vector<NFTsData>
getNFTDataFromObj(
std::uint32_t const seq,
std::string const& key,
std::string const& blob);
getNFTDataFromObj(std::uint32_t const seq, std::string const& key, std::string const& blob);

45
src/etl/ProbingETLSource.cpp
View File

@@ -31,23 +31,9 @@ ProbingETLSource::ProbingETLSource(
ETLLoadBalancer& balancer,
boost::asio::ssl::context sslCtx)
: sslCtx_{std::move(sslCtx)}
, sslSrc_{make_shared<SslETLSource>(
config,
ioc,
std::ref(sslCtx_),
backend,
subscriptions,
nwvl,
balancer,
make_SSLHooks())}
, plainSrc_{make_shared<PlainETLSource>(
config,
ioc,
backend,
subscriptions,
nwvl,
balancer,
make_PlainHooks())}
, sslSrc_{make_shared<
SslETLSource>(config, ioc, std::ref(sslCtx_), backend, subscriptions, nwvl, balancer, make_SSLHooks())}
, plainSrc_{make_shared<PlainETLSource>(config, ioc, backend, subscriptions, nwvl, balancer, make_PlainHooks())}
{
}
@@ -107,8 +93,7 @@ std::string
ProbingETLSource::toString() const
{
if (!currentSrc_)
return "{probing... ws: " + plainSrc_->toString() +
", wss: " + sslSrc_->toString() + "}";
return "{probing... ws: " + plainSrc_->toString() + ", wss: " + sslSrc_->toString() + "}";
return currentSrc_->toString();
}
@@ -121,27 +106,19 @@ ProbingETLSource::token() const
}
bool
ProbingETLSource::loadInitialLedger(
std::uint32_t ledgerSequence,
std::uint32_t numMarkers,
bool cacheOnly)
ProbingETLSource::loadInitialLedger(std::uint32_t ledgerSequence, std::uint32_t numMarkers, bool cacheOnly)
{
if (!currentSrc_)
return false;
return currentSrc_->loadInitialLedger(
ledgerSequence, numMarkers, cacheOnly);
return currentSrc_->loadInitialLedger(ledgerSequence, numMarkers, cacheOnly);
}
std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>
ProbingETLSource::fetchLedger(
uint32_t ledgerSequence,
bool getObjects,
bool getObjectNeighbors)
ProbingETLSource::fetchLedger(uint32_t ledgerSequence, bool getObjects, bool getObjectNeighbors)
{
if (!currentSrc_)
return {};
return currentSrc_->fetchLedger(
ledgerSequence, getObjects, getObjectNeighbors);
return currentSrc_->fetchLedger(ledgerSequence, getObjects, getObjectNeighbors);
}
std::optional<boost::json::object>
@@ -179,8 +156,7 @@ ProbingETLSource::make_SSLHooks() noexcept
{
plainSrc_->pause();
currentSrc_ = sslSrc_;
log_.info() << "Selected WSS as the main source: "
<< currentSrc_->toString();
log_.info() << "Selected WSS as the main source: " << currentSrc_->toString();
}
return ETLSourceHooks::Action::PROCEED;
},
@@ -209,8 +185,7 @@ ProbingETLSource::make_PlainHooks() noexcept
{
sslSrc_->pause();
currentSrc_ = plainSrc_;
log_.info() << "Selected Plain WS as the main source: "
<< currentSrc_->toString();
log_.info() << "Selected Plain WS as the main source: " << currentSrc_->toString();
}
return ETLSourceHooks::Action::PROCEED;
},

19
src/etl/ProbingETLSource.h
View File

@@ -53,8 +53,7 @@ public:
std::shared_ptr<SubscriptionManager> subscriptions,
std::shared_ptr<NetworkValidatedLedgers> nwvl,
ETLLoadBalancer& balancer,
boost::asio::ssl::context sslCtx = boost::asio::ssl::context{
boost::asio::ssl::context::tlsv12});
boost::asio::ssl::context sslCtx = boost::asio::ssl::context{boost::asio::ssl::context::tlsv12});
~ProbingETLSource() = default;
@@ -80,22 +79,14 @@ public:
toString() const override;
bool
loadInitialLedger(
std::uint32_t ledgerSequence,
std::uint32_t numMarkers,
bool cacheOnly = false) override;
loadInitialLedger(std::uint32_t ledgerSequence, std::uint32_t numMarkers, bool cacheOnly = false) override;
std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>
fetchLedger(
uint32_t ledgerSequence,
bool getObjects = true,
bool getObjectNeighbors = false) override;
fetchLedger(uint32_t ledgerSequence, bool getObjects = true, bool getObjectNeighbors = false) override;
std::optional<boost::json::object>
forwardToRippled(
boost::json::object const& request,
std::string const& clientIp,
boost::asio::yield_context& yield) const override;
forwardToRippled(boost::json::object const& request, std::string const& clientIp, boost::asio::yield_context& yield)
const override;
boost::uuids::uuid
token() const override;

509
src/etl/ReportingETL.cpp
View File

@@ -47,23 +47,19 @@ std::string
toString(ripple::LedgerInfo const& info)
{
std::stringstream ss;
ss << "LedgerInfo { Sequence : " << info.seq
<< " Hash : " << strHex(info.hash) << " TxHash : " << strHex(info.txHash)
<< " AccountHash : " << strHex(info.accountHash)
ss << "LedgerInfo { Sequence : " << info.seq << " Hash : " << strHex(info.hash)
<< " TxHash : " << strHex(info.txHash) << " AccountHash : " << strHex(info.accountHash)
<< " ParentHash : " << strHex(info.parentHash) << " }";
return ss.str();
}
} // namespace clio::detail
FormattedTransactionsData
ReportingETL::insertTransactions(
ripple::LedgerInfo const& ledger,
org::xrpl::rpc::v1::GetLedgerResponse& data)
ReportingETL::insertTransactions(ripple::LedgerInfo const& ledger, org::xrpl::rpc::v1::GetLedgerResponse& data)
{
FormattedTransactionsData result;
for (auto& txn :
*(data.mutable_transactions_list()->mutable_transactions()))
for (auto& txn : *(data.mutable_transactions_list()->mutable_transactions()))
{
std::string* raw = txn.mutable_transaction_blob();
@@ -72,18 +68,15 @@ ReportingETL::insertTransactions(
log_.trace() << "Inserting transaction = " << sttx.getTransactionID();
ripple::TxMeta txMeta{
sttx.getTransactionID(), ledger.seq, txn.metadata_blob()};
ripple::TxMeta txMeta{sttx.getTransactionID(), ledger.seq, txn.metadata_blob()};
auto const [nftTxs, maybeNFT] = getNFTDataFromTx(txMeta, sttx);
result.nfTokenTxData.insert(
result.nfTokenTxData.end(), nftTxs.begin(), nftTxs.end());
result.nfTokenTxData.insert(result.nfTokenTxData.end(), nftTxs.begin(), nftTxs.end());
if (maybeNFT)
result.nfTokensData.push_back(*maybeNFT);
auto journal = ripple::debugLog();
result.accountTxData.emplace_back(
txMeta, sttx.getTransactionID(), journal);
result.accountTxData.emplace_back(txMeta, sttx.getTransactionID(), journal);
std::string keyStr{(const char*)sttx.getTransactionID().data(), 32};
backend_->writeTransaction(
std::move(keyStr),
@@ -96,18 +89,12 @@ ReportingETL::insertTransactions(
// Remove all but the last NFTsData for each id. unique removes all
// but the first of a group, so we want to reverse sort by transaction
// index
std::sort(
result.nfTokensData.begin(),
result.nfTokensData.end(),
[](NFTsData const& a, NFTsData const& b) {
return a.tokenID > b.tokenID &&
a.transactionIndex > b.transactionIndex;
});
std::sort(result.nfTokensData.begin(), result.nfTokensData.end(), [](NFTsData const& a, NFTsData const& b) {
return a.tokenID > b.tokenID && a.transactionIndex > b.transactionIndex;
});
// Now we can unique the NFTs by tokenID.
auto last = std::unique(
result.nfTokensData.begin(),
result.nfTokensData.end(),
[](NFTsData const& a, NFTsData const& b) {
auto last =
std::unique(result.nfTokensData.begin(), result.nfTokensData.end(), [](NFTsData const& a, NFTsData const& b) {
return a.tokenID == b.tokenID;
});
result.nfTokensData.erase(last, result.nfTokensData.end());
@@ -130,13 +117,11 @@ ReportingETL::loadInitialLedger(uint32_t startingSequence)
// fetch the ledger from the network. This function will not return until
// either the fetch is successful, or the server is being shutdown. This
// only fetches the ledger header and the transactions+metadata
std::optional<org::xrpl::rpc::v1::GetLedgerResponse> ledgerData{
fetchLedgerData(startingSequence)};
std::optional<org::xrpl::rpc::v1::GetLedgerResponse> ledgerData{fetchLedgerData(startingSequence)};
if (!ledgerData)
return {};
ripple::LedgerInfo lgrInfo =
deserializeHeader(ripple::makeSlice(ledgerData->ledger_header()));
ripple::LedgerInfo lgrInfo = deserializeHeader(ripple::makeSlice(ledgerData->ledger_header()));
log_.debug() << "Deserialized ledger header. " << detail::toString(lgrInfo);
@@ -145,12 +130,10 @@ ReportingETL::loadInitialLedger(uint32_t startingSequence)
log_.debug() << "Started writes";
backend_->writeLedger(
lgrInfo, std::move(*ledgerData->mutable_ledger_header()));
backend_->writeLedger(lgrInfo, std::move(*ledgerData->mutable_ledger_header()));
log_.debug() << "Wrote ledger";
FormattedTransactionsData insertTxResult =
insertTransactions(lgrInfo, *ledgerData);
FormattedTransactionsData insertTxResult = insertTransactions(lgrInfo, *ledgerData);
log_.debug() << "Inserted txns";
// download the full account state map. This function downloads full
@@ -164,11 +147,9 @@ ReportingETL::loadInitialLedger(uint32_t startingSequence)
if (!stopping_)
{
backend_->writeAccountTransactions(
std::move(insertTxResult.accountTxData));
backend_->writeAccountTransactions(std::move(insertTxResult.accountTxData));
backend_->writeNFTs(std::move(insertTxResult.nfTokensData));
backend_->writeNFTTransactions(
std::move(insertTxResult.nfTokenTxData));
backend_->writeNFTTransactions(std::move(insertTxResult.nfTokenTxData));
}
backend_->finishWrites(startingSequence);
});
@@ -185,10 +166,8 @@ ReportingETL::publishLedger(ripple::LedgerInfo const& lgrInfo)
{
log_.info() << "Updating cache";
std::vector<Backend::LedgerObject> diff =
Backend::synchronousAndRetryOnTimeout([&](auto yield) {
return backend_->fetchLedgerDiff(lgrInfo.seq, yield);
});
std::vector<Backend::LedgerObject> diff = Backend::synchronousAndRetryOnTimeout(
[&](auto yield) { return backend_->fetchLedgerDiff(lgrInfo.seq, yield); });
backend_->cache().update(diff, lgrInfo.seq);
backend_->updateRange(lgrInfo.seq);
@@ -201,22 +180,16 @@ ReportingETL::publishLedger(ripple::LedgerInfo const& lgrInfo)
if (age < 600)
{
std::optional<ripple::Fees> fees =
Backend::synchronousAndRetryOnTimeout([&](auto yield) {
return backend_->fetchFees(lgrInfo.seq, yield);
});
Backend::synchronousAndRetryOnTimeout([&](auto yield) { return backend_->fetchFees(lgrInfo.seq, yield); });
std::vector<Backend::TransactionAndMetadata> transactions =
Backend::synchronousAndRetryOnTimeout([&](auto yield) {
return backend_->fetchAllTransactionsInLedger(
lgrInfo.seq, yield);
});
std::vector<Backend::TransactionAndMetadata> transactions = Backend::synchronousAndRetryOnTimeout(
[&](auto yield) { return backend_->fetchAllTransactionsInLedger(lgrInfo.seq, yield); });
auto ledgerRange = backend_->fetchLedgerRange();
assert(ledgerRange);
assert(fees);
std::string range = std::to_string(ledgerRange->minSequence) + "-" +
std::to_string(ledgerRange->maxSequence);
std::string range = std::to_string(ledgerRange->minSequence) + "-" + std::to_string(ledgerRange->maxSequence);
subscriptions_->pubLedger(lgrInfo, *fees, range, transactions.size());
@@ -228,15 +201,12 @@ ReportingETL::publishLedger(ripple::LedgerInfo const& lgrInfo)
log_.info() << "Published ledger " << std::to_string(lgrInfo.seq);
}
else
log_.info() << "Skipping publishing ledger "
<< std::to_string(lgrInfo.seq);
log_.info() << "Skipping publishing ledger " << std::to_string(lgrInfo.seq);
setLastPublish();
}
bool
ReportingETL::publishLedger(
uint32_t ledgerSequence,
std::optional<uint32_t> maxAttempts)
ReportingETL::publishLedger(uint32_t ledgerSequence, std::optional<uint32_t> maxAttempts)
{
log_.info() << "Attempting to publish ledger = " << ledgerSequence;
size_t numAttempts = 0;
@@ -253,8 +223,7 @@ ReportingETL::publishLedger(
// second in between each attempt.
if (maxAttempts && numAttempts >= maxAttempts)
{
log_.debug() << "Failed to publish ledger after " << numAttempts
<< " attempts.";
log_.debug() << "Failed to publish ledger after " << numAttempts << " attempts.";
return false;
}
std::this_thread::sleep_for(std::chrono::seconds(1));
@@ -263,9 +232,8 @@ ReportingETL::publishLedger(
}
else
{
auto lgr = Backend::synchronousAndRetryOnTimeout([&](auto yield) {
return backend_->fetchLedgerBySequence(ledgerSequence, yield);
});
auto lgr = Backend::synchronousAndRetryOnTimeout(
[&](auto yield) { return backend_->fetchLedgerBySequence(ledgerSequence, yield); });
assert(lgr);
publishLedger(*lgr);
@@ -281,8 +249,7 @@ ReportingETL::fetchLedgerData(uint32_t seq)
{
log_.debug() << "Attempting to fetch ledger with sequence = " << seq;
std::optional<org::xrpl::rpc::v1::GetLedgerResponse> response =
loadBalancer_->fetchLedger(seq, false, false);
std::optional<org::xrpl::rpc::v1::GetLedgerResponse> response = loadBalancer_->fetchLedger(seq, false, false);
if (response)
log_.trace() << "GetLedger reply = " << response->DebugString();
return response;
@@ -293,12 +260,8 @@ ReportingETL::fetchLedgerDataAndDiff(uint32_t seq)
{
log_.debug() << "Attempting to fetch ledger with sequence = " << seq;
std::optional<org::xrpl::rpc::v1::GetLedgerResponse> response =
loadBalancer_->fetchLedger(
seq,
true,
!backend_->cache().isFull() ||
backend_->cache().latestLedgerSequence() >= seq);
std::optional<org::xrpl::rpc::v1::GetLedgerResponse> response = loadBalancer_->fetchLedger(
seq, true, !backend_->cache().isFull() || backend_->cache().latestLedgerSequence() >= seq);
if (response)
log_.trace() << "GetLedger reply = " << response->DebugString();
return response;
@@ -308,8 +271,7 @@ std::pair<ripple::LedgerInfo, bool>
ReportingETL::buildNextLedger(org::xrpl::rpc::v1::GetLedgerResponse& rawData)
{
log_.debug() << "Beginning ledger update";
ripple::LedgerInfo lgrInfo =
deserializeHeader(ripple::makeSlice(rawData.ledger_header()));
ripple::LedgerInfo lgrInfo = deserializeHeader(ripple::makeSlice(rawData.ledger_header()));
log_.debug() << "Deserialized ledger header. " << detail::toString(lgrInfo);
backend_->startWrites();
@@ -327,14 +289,10 @@ ReportingETL::buildNextLedger(org::xrpl::rpc::v1::GetLedgerResponse& rawData)
auto firstBook = std::move(*obj.mutable_first_book());
if (!firstBook.size())
firstBook = uint256ToString(Backend::lastKey);
log_.debug() << "writing book successor "
<< ripple::strHex(obj.book_base()) << " - "
log_.debug() << "writing book successor " << ripple::strHex(obj.book_base()) << " - "
<< ripple::strHex(firstBook);
backend_->writeSuccessor(
std::move(*obj.mutable_book_base()),
lgrInfo.seq,
std::move(firstBook));
backend_->writeSuccessor(std::move(*obj.mutable_book_base()), lgrInfo.seq, std::move(firstBook));
}
for (auto& obj : *(rawData.mutable_ledger_objects()->mutable_objects()))
{
@@ -347,32 +305,20 @@ ReportingETL::buildNextLedger(org::xrpl::rpc::v1::GetLedgerResponse& rawData)
if (!succPtr->size())
*succPtr = uint256ToString(Backend::lastKey);
if (obj.mod_type() ==
org::xrpl::rpc::v1::RawLedgerObject::DELETED)
if (obj.mod_type() == org::xrpl::rpc::v1::RawLedgerObject::DELETED)
{
log_.debug() << "Modifying successors for deleted object "
<< ripple::strHex(obj.key()) << " - "
<< ripple::strHex(*predPtr) << " - "
<< ripple::strHex(*succPtr);
log_.debug() << "Modifying successors for deleted object " << ripple::strHex(obj.key()) << " - "
<< ripple::strHex(*predPtr) << " - " << ripple::strHex(*succPtr);
backend_->writeSuccessor(
std::move(*predPtr), lgrInfo.seq, std::move(*succPtr));
backend_->writeSuccessor(std::move(*predPtr), lgrInfo.seq, std::move(*succPtr));
}
else
{
log_.debug() << "adding successor for new object "
<< ripple::strHex(obj.key()) << " - "
<< ripple::strHex(*predPtr) << " - "
<< ripple::strHex(*succPtr);
log_.debug() << "adding successor for new object " << ripple::strHex(obj.key()) << " - "
<< ripple::strHex(*predPtr) << " - " << ripple::strHex(*succPtr);
backend_->writeSuccessor(
std::move(*predPtr),
lgrInfo.seq,
std::string{obj.key()});
backend_->writeSuccessor(
std::string{obj.key()},
lgrInfo.seq,
std::move(*succPtr));
backend_->writeSuccessor(std::move(*predPtr), lgrInfo.seq, std::string{obj.key()});
backend_->writeSuccessor(std::string{obj.key()}, lgrInfo.seq, std::move(*succPtr));
}
}
else
@@ -388,17 +334,13 @@ ReportingETL::buildNextLedger(org::xrpl::rpc::v1::GetLedgerResponse& rawData)
{
auto key = ripple::uint256::fromVoidChecked(obj.key());
assert(key);
cacheUpdates.push_back(
{*key, {obj.mutable_data()->begin(), obj.mutable_data()->end()}});
log_.debug() << "key = " << ripple::strHex(*key)
<< " - mod type = " << obj.mod_type();
cacheUpdates.push_back({*key, {obj.mutable_data()->begin(), obj.mutable_data()->end()}});
log_.debug() << "key = " << ripple::strHex(*key) << " - mod type = " << obj.mod_type();
if (obj.mod_type() != org::xrpl::rpc::v1::RawLedgerObject::MODIFIED &&
!rawData.object_neighbors_included())
if (obj.mod_type() != org::xrpl::rpc::v1::RawLedgerObject::MODIFIED && !rawData.object_neighbors_included())
{
log_.debug() << "object neighbors not included. using cache";
if (!backend_->cache().isFull() ||
backend_->cache().latestLedgerSequence() != lgrInfo.seq - 1)
if (!backend_->cache().isFull() || backend_->cache().latestLedgerSequence() != lgrInfo.seq - 1)
throw std::runtime_error(
"Cache is not full, but object neighbors were not "
"included");
@@ -417,20 +359,15 @@ ReportingETL::buildNextLedger(org::xrpl::rpc::v1::GetLedgerResponse& rawData)
{
log_.debug() << "Is book dir. key = " << ripple::strHex(*key);
auto bookBase = getBookBase(*key);
auto oldFirstDir =
backend_->cache().getSuccessor(bookBase, lgrInfo.seq - 1);
auto oldFirstDir = backend_->cache().getSuccessor(bookBase, lgrInfo.seq - 1);
assert(oldFirstDir);
// We deleted the first directory, or we added a directory prior
// to the old first directory
if ((isDeleted && key == oldFirstDir->key) ||
(!isDeleted && key < oldFirstDir->key))
if ((isDeleted && key == oldFirstDir->key) || (!isDeleted && key < oldFirstDir->key))
{
log_.debug()
<< "Need to recalculate book base successor. base = "
<< ripple::strHex(bookBase)
<< " - key = " << ripple::strHex(*key)
<< " - isDeleted = " << isDeleted
<< " - seq = " << lgrInfo.seq;
log_.debug() << "Need to recalculate book base successor. base = " << ripple::strHex(bookBase)
<< " - key = " << ripple::strHex(*key) << " - isDeleted = " << isDeleted
<< " - seq = " << lgrInfo.seq;
bookSuccessorsToCalculate.insert(bookBase);
}
}
@@ -438,18 +375,14 @@ ReportingETL::buildNextLedger(org::xrpl::rpc::v1::GetLedgerResponse& rawData)
if (obj.mod_type() == org::xrpl::rpc::v1::RawLedgerObject::MODIFIED)
modified.insert(*key);
backend_->writeLedgerObject(
std::move(*obj.mutable_key()),
lgrInfo.seq,
std::move(*obj.mutable_data()));
backend_->writeLedgerObject(std::move(*obj.mutable_key()), lgrInfo.seq, std::move(*obj.mutable_data()));
}
backend_->cache().update(cacheUpdates, lgrInfo.seq);
// rippled didn't send successor information, so use our cache
if (!rawData.object_neighbors_included())
{
log_.debug() << "object neighbors not included. using cache";
if (!backend_->cache().isFull() ||
backend_->cache().latestLedgerSequence() != lgrInfo.seq)
if (!backend_->cache().isFull() || backend_->cache().latestLedgerSequence() != lgrInfo.seq)
throw std::runtime_error(
"Cache is not full, but object neighbors were not "
"included");
@@ -465,31 +398,18 @@ ReportingETL::buildNextLedger(org::xrpl::rpc::v1::GetLedgerResponse& rawData)
ub = {Backend::lastKey, {}};
if (obj.blob.size() == 0)
{
log_.debug() << "writing successor for deleted object "
<< ripple::strHex(obj.key) << " - "
<< ripple::strHex(lb->key) << " - "
<< ripple::strHex(ub->key);
log_.debug() << "writing successor for deleted object " << ripple::strHex(obj.key) << " - "
<< ripple::strHex(lb->key) << " - " << ripple::strHex(ub->key);
backend_->writeSuccessor(
uint256ToString(lb->key),
lgrInfo.seq,
uint256ToString(ub->key));
backend_->writeSuccessor(uint256ToString(lb->key), lgrInfo.seq, uint256ToString(ub->key));
}
else
{
backend_->writeSuccessor(
uint256ToString(lb->key),
lgrInfo.seq,
uint256ToString(obj.key));
backend_->writeSuccessor(
uint256ToString(obj.key),
lgrInfo.seq,
uint256ToString(ub->key));
backend_->writeSuccessor(uint256ToString(lb->key), lgrInfo.seq, uint256ToString(obj.key));
backend_->writeSuccessor(uint256ToString(obj.key), lgrInfo.seq, uint256ToString(ub->key));
log_.debug() << "writing successor for new object "
<< ripple::strHex(lb->key) << " - "
<< ripple::strHex(obj.key) << " - "
<< ripple::strHex(ub->key);
log_.debug() << "writing successor for new object " << ripple::strHex(lb->key) << " - "
<< ripple::strHex(obj.key) << " - " << ripple::strHex(ub->key);
}
}
for (auto const& base : bookSuccessorsToCalculate)
@@ -497,34 +417,24 @@ ReportingETL::buildNextLedger(org::xrpl::rpc::v1::GetLedgerResponse& rawData)
auto succ = backend_->cache().getSuccessor(base, lgrInfo.seq);
if (succ)
{
backend_->writeSuccessor(
uint256ToString(base),
lgrInfo.seq,
uint256ToString(succ->key));
backend_->writeSuccessor(uint256ToString(base), lgrInfo.seq, uint256ToString(succ->key));
log_.debug()
<< "Updating book successor " << ripple::strHex(base)
<< " - " << ripple::strHex(succ->key);
log_.debug() << "Updating book successor " << ripple::strHex(base) << " - "
<< ripple::strHex(succ->key);
}
else
{
backend_->writeSuccessor(
uint256ToString(base),
lgrInfo.seq,
uint256ToString(Backend::lastKey));
backend_->writeSuccessor(uint256ToString(base), lgrInfo.seq, uint256ToString(Backend::lastKey));
log_.debug()
<< "Updating book successor " << ripple::strHex(base)
<< " - " << ripple::strHex(Backend::lastKey);
log_.debug() << "Updating book successor " << ripple::strHex(base) << " - "
<< ripple::strHex(Backend::lastKey);
}
}
}
log_.debug()
<< "Inserted/modified/deleted all objects. Number of objects = "
<< rawData.ledger_objects().objects_size();
FormattedTransactionsData insertTxResult =
insertTransactions(lgrInfo, rawData);
log_.debug() << "Inserted/modified/deleted all objects. Number of objects = "
<< rawData.ledger_objects().objects_size();
FormattedTransactionsData insertTxResult = insertTransactions(lgrInfo, rawData);
log_.debug() << "Inserted all transactions. Number of transactions = "
<< rawData.transactions_list().transactions_size();
backend_->writeAccountTransactions(std::move(insertTxResult.accountTxData));
@@ -532,8 +442,8 @@ ReportingETL::buildNextLedger(org::xrpl::rpc::v1::GetLedgerResponse& rawData)
backend_->writeNFTTransactions(std::move(insertTxResult.nfTokenTxData));
log_.debug() << "wrote account_tx";
auto [success, duration] = util::timed<std::chrono::duration<double>>(
[&]() { return backend_->finishWrites(lgrInfo.seq); });
auto [success, duration] =
util::timed<std::chrono::duration<double>>([&]() { return backend_->finishWrites(lgrInfo.seq); });
log_.debug() << "Finished writes. took " << std::to_string(duration);
log_.debug() << "Finished ledger update. " << detail::toString(lgrInfo);
@@ -585,12 +495,10 @@ ReportingETL::runETLPipeline(uint32_t startSequence, int numExtractors)
std::optional<uint32_t> lastPublishedSequence;
uint32_t maxQueueSize = 1000 / numExtractors;
auto begin = std::chrono::system_clock::now();
using QueueType =
ThreadSafeQueue<std::optional<org::xrpl::rpc::v1::GetLedgerResponse>>;
using QueueType = ThreadSafeQueue<std::optional<org::xrpl::rpc::v1::GetLedgerResponse>>;
std::vector<std::shared_ptr<QueueType>> queues;
auto getNext = [&queues, &startSequence, &numExtractors](
uint32_t sequence) -> std::shared_ptr<QueueType> {
auto getNext = [&queues, &startSequence, &numExtractors](uint32_t sequence) -> std::shared_ptr<QueueType> {
return queues[(sequence - startSequence) % numExtractors];
};
std::vector<std::thread> extractors;
@@ -599,12 +507,7 @@ ReportingETL::runETLPipeline(uint32_t startSequence, int numExtractors)
auto transformQueue = std::make_shared<QueueType>(maxQueueSize);
queues.push_back(transformQueue);
extractors.emplace_back([this,
&startSequence,
&writeConflict,
transformQueue,
i,
numExtractors]() {
extractors.emplace_back([this, &startSequence, &writeConflict, transformQueue, i, numExtractors]() {
beast::setCurrentThreadName("rippled: ReportingETL extract");
uint32_t currentSequence = startSequence + i;
@@ -616,14 +519,11 @@ ReportingETL::runETLPipeline(uint32_t startSequence, int numExtractors)
// the entire server is shutting down. This can be detected in a
// variety of ways. See the comment at the top of the function
while ((!finishSequence_ || currentSequence <= *finishSequence_) &&
networkValidatedLedgers_->waitUntilValidatedByNetwork(
currentSequence) &&
!writeConflict && !isStopping())
networkValidatedLedgers_->waitUntilValidatedByNetwork(currentSequence) && !writeConflict &&
!isStopping())
{
auto [fetchResponse, time] =
util::timed<std::chrono::duration<double>>([&]() {
return fetchLedgerDataAndDiff(currentSequence);
});
auto [fetchResponse, time] = util::timed<std::chrono::duration<double>>(
[&]() { return fetchLedgerDataAndDiff(currentSequence); });
totalTime += time;
// if the fetch is unsuccessful, stop. fetchLedger only
@@ -637,16 +537,11 @@ ReportingETL::runETLPipeline(uint32_t startSequence, int numExtractors)
{
break;
}
auto tps =
fetchResponse->transactions_list().transactions_size() /
time;
auto tps = fetchResponse->transactions_list().transactions_size() / time;
log_.info() << "Extract phase time = " << time
<< " . Extract phase tps = " << tps
<< " . Avg extract time = "
<< totalTime / (currentSequence - startSequence + 1)
<< " . thread num = " << i
<< " . seq = " << currentSequence;
log_.info() << "Extract phase time = " << time << " . Extract phase tps = " << tps
<< " . Avg extract time = " << totalTime / (currentSequence - startSequence + 1)
<< " . thread num = " << i << " . seq = " << currentSequence;
transformQueue->push(std::move(fetchResponse));
currentSequence += numExtractors;
@@ -658,19 +553,13 @@ ReportingETL::runETLPipeline(uint32_t startSequence, int numExtractors)
});
}
std::thread transformer{[this,
&minSequence,
&writeConflict,
&startSequence,
&getNext,
&lastPublishedSequence]() {
std::thread transformer{[this, &minSequence, &writeConflict, &startSequence, &getNext, &lastPublishedSequence]() {
beast::setCurrentThreadName("rippled: ReportingETL transform");
uint32_t currentSequence = startSequence;
while (!writeConflict)
{
std::optional<org::xrpl::rpc::v1::GetLedgerResponse> fetchResponse{
getNext(currentSequence)->pop()};
std::optional<org::xrpl::rpc::v1::GetLedgerResponse> fetchResponse{getNext(currentSequence)->pop()};
++currentSequence;
// if fetchResponse is an empty optional, the extracter thread
// has stopped and the transformer should stop as well
@@ -681,8 +570,7 @@ ReportingETL::runETLPipeline(uint32_t startSequence, int numExtractors)
if (isStopping())
continue;
auto numTxns =
fetchResponse->transactions_list().transactions_size();
auto numTxns = fetchResponse->transactions_list().transactions_size();
auto numObjects = fetchResponse->ledger_objects().objects_size();
auto start = std::chrono::system_clock::now();
auto [lgrInfo, success] = buildNextLedger(*fetchResponse);
@@ -690,40 +578,31 @@ ReportingETL::runETLPipeline(uint32_t startSequence, int numExtractors)
auto duration = ((end - start).count()) / 1000000000.0;
if (success)
log_.info()
<< "Load phase of etl : "
<< "Successfully wrote ledger! Ledger info: "
<< detail::toString(lgrInfo) << ". txn count = " << numTxns
<< ". object count = " << numObjects
<< ". load time = " << duration
<< ". load txns per second = " << numTxns / duration
<< ". load objs per second = " << numObjects / duration;
log_.info() << "Load phase of etl : "
<< "Successfully wrote ledger! Ledger info: " << detail::toString(lgrInfo)
<< ". txn count = " << numTxns << ". object count = " << numObjects
<< ". load time = " << duration << ". load txns per second = " << numTxns / duration
<< ". load objs per second = " << numObjects / duration;
else
log_.error()
<< "Error writing ledger. " << detail::toString(lgrInfo);
log_.error() << "Error writing ledger. " << detail::toString(lgrInfo);
// success is false if the ledger was already written
if (success)
{
boost::asio::post(publishStrand_, [this, lgrInfo = lgrInfo]() {
publishLedger(lgrInfo);
});
boost::asio::post(publishStrand_, [this, lgrInfo = lgrInfo]() { publishLedger(lgrInfo); });
lastPublishedSequence = lgrInfo.seq;
}
writeConflict = !success;
// TODO move online delete logic to an admin RPC call
if (onlineDeleteInterval_ && !deleting_ &&
lgrInfo.seq - minSequence > *onlineDeleteInterval_)
if (onlineDeleteInterval_ && !deleting_ && lgrInfo.seq - minSequence > *onlineDeleteInterval_)
{
deleting_ = true;
ioContext_.post([this, &minSequence]() {
log_.info() << "Running online delete";
Backend::synchronous(
[&](boost::asio::yield_context& yield) {
backend_->doOnlineDelete(
*onlineDeleteInterval_, yield);
});
Backend::synchronous([&](boost::asio::yield_context& yield) {
backend_->doOnlineDelete(*onlineDeleteInterval_, yield);
});
log_.info() << "Finished online delete";
auto rng = backend_->fetchLedgerRange();
@@ -744,8 +623,7 @@ ReportingETL::runETLPipeline(uint32_t startSequence, int numExtractors)
for (auto& t : extractors)
t.join();
auto end = std::chrono::system_clock::now();
log_.debug() << "Extracted and wrote "
<< *lastPublishedSequence - startSequence << " in "
log_.debug() << "Extracted and wrote " << *lastPublishedSequence - startSequence << " in "
<< ((end - begin).count()) / 1000000000.0;
writing_ = false;
@@ -776,20 +654,16 @@ ReportingETL::monitor()
if (startSequence_)
{
log_.info() << "ledger sequence specified in config. "
<< "Will begin ETL process starting with ledger "
<< *startSequence_;
<< "Will begin ETL process starting with ledger " << *startSequence_;
ledger = loadInitialLedger(*startSequence_);
}
else
{
log_.info()
<< "Waiting for next ledger to be validated by network...";
std::optional<uint32_t> mostRecentValidated =
networkValidatedLedgers_->getMostRecent();
log_.info() << "Waiting for next ledger to be validated by network...";
std::optional<uint32_t> mostRecentValidated = networkValidatedLedgers_->getMostRecent();
if (mostRecentValidated)
{
log_.info() << "Ledger " << *mostRecentValidated
<< " has been validated. "
log_.info() << "Ledger " << *mostRecentValidated << " has been validated. "
<< "Downloading...";
ledger = loadInitialLedger(*mostRecentValidated);
}
@@ -805,8 +679,7 @@ ReportingETL::monitor()
rng = backend_->hardFetchLedgerRangeNoThrow();
else
{
log_.error()
<< "Failed to load initial ledger. Exiting monitor loop";
log_.error() << "Failed to load initial ledger. Exiting monitor loop";
return;
}
}
@@ -814,11 +687,9 @@ ReportingETL::monitor()
{
if (startSequence_)
{
log_.warn()
<< "start sequence specified but db is already populated";
log_.warn() << "start sequence specified but db is already populated";
}
log_.info()
<< "Database already populated. Picking up from the tip of history";
log_.info() << "Database already populated. Picking up from the tip of history";
loadCache(rng->maxSequence);
}
assert(rng);
@@ -828,17 +699,14 @@ ReportingETL::monitor()
<< "Starting monitor loop. sequence = " << nextSequence;
while (true)
{
if (auto rng = backend_->hardFetchLedgerRangeNoThrow();
rng && rng->maxSequence >= nextSequence)
if (auto rng = backend_->hardFetchLedgerRangeNoThrow(); rng && rng->maxSequence >= nextSequence)
{
publishLedger(nextSequence, {});
++nextSequence;
}
else if (networkValidatedLedgers_->waitUntilValidatedByNetwork(
nextSequence, 1000))
else if (networkValidatedLedgers_->waitUntilValidatedByNetwork(nextSequence, 1000))
{
log_.info() << "Ledger with sequence = " << nextSequence
<< " has been validated by the network. "
log_.info() << "Ledger with sequence = " << nextSequence << " has been validated by the network. "
<< "Attempting to find in database and publish";
// Attempt to take over responsibility of ETL writer after 10 failed
// attempts to publish the ledger. publishLedger() fails if the
@@ -850,12 +718,10 @@ ReportingETL::monitor()
bool success = publishLedger(nextSequence, timeoutSeconds);
if (!success)
{
log_.warn() << "Failed to publish ledger with sequence = "
<< nextSequence << " . Beginning ETL";
log_.warn() << "Failed to publish ledger with sequence = " << nextSequence << " . Beginning ETL";
// doContinousETLPipelined returns the most recent sequence
// published empty optional if no sequence was published
std::optional<uint32_t> lastPublished =
runETLPipeline(nextSequence, extractorThreads_);
std::optional<uint32_t> lastPublished = runETLPipeline(nextSequence, extractorThreads_);
log_.info() << "Aborting ETL. Falling back to publishing";
// if no ledger was published, don't increment nextSequence
if (lastPublished)
@@ -873,8 +739,7 @@ ReportingETL::loadCacheFromClioPeer(
std::string const& port,
boost::asio::yield_context& yield)
{
log_.info() << "Loading cache from peer. ip = " << ip
<< " . port = " << port;
log_.info() << "Loading cache from peer. ip = " << ip << " . port = " << port;
namespace beast = boost::beast; // from <boost/beast.hpp>
namespace http = beast::http; // from <boost/beast/http.hpp>
namespace websocket = beast::websocket; // from
@@ -887,8 +752,7 @@ ReportingETL::loadCacheFromClioPeer(
tcp::resolver resolver{ioContext_};
log_.trace() << "Creating websocket";
auto ws =
std::make_unique<websocket::stream<beast::tcp_stream>>(ioContext_);
auto ws = std::make_unique<websocket::stream<beast::tcp_stream>>(ioContext_);
// Look up the domain name
auto const results = resolver.async_resolve(ip, port, yield[ec]);
@@ -928,9 +792,7 @@ ReportingETL::loadCacheFromClioPeer(
do
{
// Send the message
ws->async_write(
net::buffer(boost::json::serialize(getRequest(marker))),
yield[ec]);
ws->async_write(net::buffer(boost::json::serialize(getRequest(marker))), yield[ec]);
if (ec)
{
log_.error() << "error writing = " << ec.message();
@@ -955,8 +817,7 @@ ReportingETL::loadCacheFromClioPeer(
}
log_.trace() << "Successfully parsed response " << parsed;
if (auto const& response = parsed.as_object();
response.contains("error"))
if (auto const& response = parsed.as_object(); response.contains("error"))
{
log_.error() << "Response contains error: " << response;
auto const& err = response.at("error");
@@ -965,15 +826,13 @@ ReportingETL::loadCacheFromClioPeer(
++numAttempts;
if (numAttempts >= 5)
{
log_.error()
<< " ledger not found at peer after 5 attempts. "
"peer = "
<< ip << " ledger = " << ledgerIndex
<< ". Check your config and the health of the peer";
log_.error() << " ledger not found at peer after 5 attempts. "
"peer = "
<< ip << " ledger = " << ledgerIndex
<< ". Check your config and the health of the peer";
return false;
}
log_.warn() << "Ledger not found. ledger = " << ledgerIndex
<< ". Sleeping and trying again";
log_.warn() << "Ledger not found. ledger = " << ledgerIndex << ". Sleeping and trying again";
std::this_thread::sleep_for(std::chrono::seconds(1));
continue;
}
@@ -982,8 +841,7 @@ ReportingETL::loadCacheFromClioPeer(
started = true;
auto const& response = parsed.as_object()["result"].as_object();
if (!response.contains("cache_full") ||
!response.at("cache_full").as_bool())
if (!response.contains("cache_full") || !response.at("cache_full").as_bool())
{
log_.error() << "cache not full for clio node. ip = " << ip;
return false;
@@ -1003,15 +861,12 @@ ReportingETL::loadCacheFromClioPeer(
Backend::LedgerObject stateObject = {};
if (!stateObject.key.parseHex(
obj.at("index").as_string().c_str()))
if (!stateObject.key.parseHex(obj.at("index").as_string().c_str()))
{
log_.error() << "failed to parse object id";
return false;
}
boost::algorithm::unhex(
obj.at("data").as_string().c_str(),
std::back_inserter(stateObject.blob));
boost::algorithm::unhex(obj.at("data").as_string().c_str(), std::back_inserter(stateObject.blob));
objects.push_back(std::move(stateObject));
}
backend_->cache().update(objects, ledgerIndex, true);
@@ -1020,16 +875,14 @@ ReportingETL::loadCacheFromClioPeer(
log_.debug() << "At marker " << *marker;
} while (marker || !started);
log_.info() << "Finished downloading ledger from clio node. ip = "
<< ip;
log_.info() << "Finished downloading ledger from clio node. ip = " << ip;
backend_->cache().setFull();
return true;
}
catch (std::exception const& e)
{
log_.error() << "Encountered exception : " << e.what()
<< " - ip = " << ip;
log_.error() << "Encountered exception : " << e.what() << " - ip = " << ip;
return false;
}
}
@@ -1059,18 +912,16 @@ ReportingETL::loadCache(uint32_t seq)
if (clioPeers.size() > 0)
{
boost::asio::spawn(
ioContext_, [this, seq](boost::asio::yield_context yield) {
for (auto const& peer : clioPeers)
{
// returns true on success
if (loadCacheFromClioPeer(
seq, peer.ip, std::to_string(peer.port), yield))
return;
}
// if we couldn't successfully load from any peers, load from db
loadCacheFromDb(seq);
});
boost::asio::spawn(ioContext_, [this, seq](boost::asio::yield_context yield) {
for (auto const& peer : clioPeers)
{
// returns true on success
if (loadCacheFromClioPeer(seq, peer.ip, std::to_string(peer.port), yield))
return;
}
// if we couldn't successfully load from any peers, load from db
loadCacheFromDb(seq);
});
return;
}
else
@@ -1078,14 +929,11 @@ ReportingETL::loadCache(uint32_t seq)
loadCacheFromDb(seq);
}
// If loading synchronously, poll cache until full
while (cacheLoadStyle_ == CacheLoadStyle::SYNC &&
!backend_->cache().isFull())
while (cacheLoadStyle_ == CacheLoadStyle::SYNC && !backend_->cache().isFull())
{
log_.debug() << "Cache not full. Cache size = "
<< backend_->cache().size() << ". Sleeping ...";
log_.debug() << "Cache not full. Cache size = " << backend_->cache().size() << ". Sleeping ...";
std::this_thread::sleep_for(std::chrono::seconds(10));
log_.info() << "Cache is full. Cache size = "
<< backend_->cache().size();
log_.info() << "Cache is full. Cache size = " << backend_->cache().size();
}
}
@@ -1101,9 +949,7 @@ ReportingETL::loadCacheFromDb(uint32_t seq)
}
loading = true;
std::vector<Backend::LedgerObject> diff;
auto append = [](auto&& a, auto&& b) {
a.insert(std::end(a), std::begin(b), std::end(b));
};
auto append = [](auto&& a, auto&& b) { a.insert(std::end(a), std::begin(b), std::end(b)); };
for (size_t i = 0; i < numCacheDiffs_; ++i)
{
@@ -1113,15 +959,9 @@ ReportingETL::loadCacheFromDb(uint32_t seq)
}
std::sort(diff.begin(), diff.end(), [](auto a, auto b) {
return a.key < b.key ||
(a.key == b.key && a.blob.size() < b.blob.size());
return a.key < b.key || (a.key == b.key && a.blob.size() < b.blob.size());
});
diff.erase(
std::unique(
diff.begin(),
diff.end(),
[](auto a, auto b) { return a.key == b.key; }),
diff.end());
diff.erase(std::unique(diff.begin(), diff.end(), [](auto a, auto b) { return a.key == b.key; }), diff.end());
std::vector<std::optional<ripple::uint256>> cursors;
cursors.push_back({});
for (auto& obj : diff)
@@ -1142,8 +982,7 @@ ReportingETL::loadCacheFromDb(uint32_t seq)
cacheDownloader_ = std::thread{[this, seq, cursors]() {
auto startTime = std::chrono::system_clock::now();
auto markers = std::make_shared<std::atomic_int>(0);
auto numRemaining =
std::make_shared<std::atomic_int>(cursors.size() - 1);
auto numRemaining = std::make_shared<std::atomic_int>(cursors.size() - 1);
for (size_t i = 0; i < cursors.size() - 1; ++i)
{
std::optional<ripple::uint256> start = cursors[i];
@@ -1152,33 +991,23 @@ ReportingETL::loadCacheFromDb(uint32_t seq)
++(*markers);
boost::asio::spawn(
ioContext_,
[this, seq, start, end, numRemaining, startTime, markers](
boost::asio::yield_context yield) {
[this, seq, start, end, numRemaining, startTime, markers](boost::asio::yield_context yield) {
std::optional<ripple::uint256> cursor = start;
std::string cursorStr = cursor.has_value()
? ripple::strHex(cursor.value())
: ripple::strHex(Backend::firstKey);
log_.debug() << "Starting a cursor: " << cursorStr
<< " markers = " << *markers;
std::string cursorStr =
cursor.has_value() ? ripple::strHex(cursor.value()) : ripple::strHex(Backend::firstKey);
log_.debug() << "Starting a cursor: " << cursorStr << " markers = " << *markers;
while (!stopping_)
{
auto res = Backend::retryOnTimeout([this,
seq,
&cursor,
&yield]() {
return backend_->fetchLedgerPage(
cursor, seq, cachePageFetchSize_, false, yield);
auto res = Backend::retryOnTimeout([this, seq, &cursor, &yield]() {
return backend_->fetchLedgerPage(cursor, seq, cachePageFetchSize_, false, yield);
});
backend_->cache().update(res.objects, seq, true);
if (!res.cursor || (end && *(res.cursor) > *end))
break;
log_.trace()
<< "Loading cache. cache size = "
<< backend_->cache().size() << " - cursor = "
<< ripple::strHex(res.cursor.value())
<< " start = " << cursorStr
<< " markers = " << *markers;
log_.trace() << "Loading cache. cache size = " << backend_->cache().size()
<< " - cursor = " << ripple::strHex(res.cursor.value()) << " start = " << cursorStr
<< " markers = " << *markers;
cursor = std::move(res.cursor);
}
@@ -1187,19 +1016,15 @@ ReportingETL::loadCacheFromDb(uint32_t seq)
if (--(*numRemaining) == 0)
{
auto endTime = std::chrono::system_clock::now();
auto duration =
std::chrono::duration_cast<std::chrono::seconds>(
endTime - startTime);
log_.info() << "Finished loading cache. cache size = "
<< backend_->cache().size() << ". Took "
auto duration = std::chrono::duration_cast<std::chrono::seconds>(endTime - startTime);
log_.info() << "Finished loading cache. cache size = " << backend_->cache().size() << ". Took "
<< duration.count() << " seconds";
backend_->cache().setFull();
}
else
{
log_.info() << "Finished a cursor. num remaining = "
<< *numRemaining << " start = " << cursorStr
<< " markers = " << *markers;
log_.info() << "Finished a cursor. num remaining = " << *numRemaining
<< " start = " << cursorStr << " markers = " << *markers;
}
});
}
@@ -1223,8 +1048,7 @@ ReportingETL::monitorReadOnly()
latestSequence++;
while (true)
{
if (auto rng = backend_->hardFetchLedgerRangeNoThrow();
rng && rng->maxSequence >= latestSequence)
if (auto rng = backend_->hardFetchLedgerRangeNoThrow(); rng && rng->maxSequence >= latestSequence)
{
publishLedger(latestSequence, {});
latestSequence = latestSequence + 1;
@@ -1233,8 +1057,7 @@ ReportingETL::monitorReadOnly()
// second passes, whichever occurs first. Even if we don't hear
// from rippled, if ledgers are being written to the db, we
// publish them
networkValidatedLedgers_->waitUntilValidatedByNetwork(
latestSequence, 1000);
networkValidatedLedgers_->waitUntilValidatedByNetwork(latestSequence, 1000);
}
}
@@ -1274,16 +1097,14 @@ ReportingETL::ReportingETL(
if (*interval > max)
{
std::stringstream msg;
msg << "online_delete cannot be greater than "
<< std::to_string(max);
msg << "online_delete cannot be greater than " << std::to_string(max);
throw std::runtime_error(msg.str());
}
if (*interval > 0)
onlineDeleteInterval_ = *interval;
}
extractorThreads_ =
config.valueOr<uint32_t>("extractor_threads", extractorThreads_);
extractorThreads_ = config.valueOr<uint32_t>("extractor_threads", extractorThreads_);
txnThreshold_ = config.valueOr<size_t>("txn_threshold", txnThreshold_);
if (config.contains("cache"))
{
@@ -1299,10 +1120,8 @@ ReportingETL::ReportingETL(
}
numCacheDiffs_ = cache.valueOr<size_t>("num_diffs", numCacheDiffs_);
numCacheMarkers_ =
cache.valueOr<size_t>("num_markers", numCacheMarkers_);
cachePageFetchSize_ =
cache.valueOr<size_t>("page_fetch_size", cachePageFetchSize_);
numCacheMarkers_ = cache.valueOr<size_t>("num_markers", numCacheMarkers_);
cachePageFetchSize_ = cache.valueOr<size_t>("page_fetch_size", cachePageFetchSize_);
if (auto peers = cache.maybeArray("peers"); peers)
{
@@ -1314,13 +1133,9 @@ ReportingETL::ReportingETL(
// todo: use emplace_back when clang is ready
clioPeers.push_back({ip, port});
}
unsigned seed =
std::chrono::system_clock::now().time_since_epoch().count();
unsigned seed = std::chrono::system_clock::now().time_since_epoch().count();
std::shuffle(
clioPeers.begin(),
clioPeers.end(),
std::default_random_engine(seed));
std::shuffle(clioPeers.begin(), clioPeers.end(), std::default_random_engine(seed));
}
}
}

16
src/etl/ReportingETL.h
View File

@@ -265,9 +265,7 @@ private:
/// (mostly transaction hashes, corresponding nodestore hashes and affected
/// accounts)
FormattedTransactionsData
insertTransactions(
ripple::LedgerInfo const& ledger,
org::xrpl::rpc::v1::GetLedgerResponse& data);
insertTransactions(ripple::LedgerInfo const& ledger, org::xrpl::rpc::v1::GetLedgerResponse& data);
// TODO update this documentation
/// Build the next ledger using the previous ledger and the extracted data.
@@ -341,8 +339,7 @@ public:
std::shared_ptr<ETLLoadBalancer> balancer,
std::shared_ptr<NetworkValidatedLedgers> ledgers)
{
auto etl = std::make_shared<ReportingETL>(
config, ioc, backend, subscriptions, balancer, ledgers);
auto etl = std::make_shared<ReportingETL>(config, ioc, backend, subscriptions, balancer, ledgers);
etl->run();
@@ -373,8 +370,7 @@ public:
result["read_only"] = readOnly_;
auto last = getLastPublish();
if (last.time_since_epoch().count() != 0)
result["last_publish_age_seconds"] =
std::to_string(lastPublishAgeSeconds());
result["last_publish_age_seconds"] = std::to_string(lastPublishAgeSeconds());
return result;
}
@@ -388,8 +384,7 @@ public:
std::uint32_t
lastPublishAgeSeconds() const
{
return std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now() - getLastPublish())
return std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now() - getLastPublish())
.count();
}
@@ -397,8 +392,7 @@ public:
lastCloseAgeSeconds() const
{
std::shared_lock lck(closeTimeMtx_);
auto now = std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
auto now = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch())
.count();
auto closeTime = lastCloseTime_.time_since_epoch().count();
if (now < (rippleEpochStart + closeTime))

45
src/log/Logger.cpp
View File

@@ -57,8 +57,7 @@ tag_invoke(boost::json::value_to_tag<Severity>, boost::json::value const& value)
return Severity::DBG;
else if (boost::iequals(logLevel, "info"))
return Severity::NFO;
else if (
boost::iequals(logLevel, "warning") || boost::iequals(logLevel, "warn"))
else if (boost::iequals(logLevel, "warning") || boost::iequals(logLevel, "warn"))
return Severity::WRN;
else if (boost::iequals(logLevel, "error"))
return Severity::ERR;
@@ -82,8 +81,7 @@ LogService::init(Config const& config)
auto const defaultFormat =
"%TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% "
"%Message%";
std::string format =
config.valueOr<std::string>("log_format", defaultFormat);
std::string format = config.valueOr<std::string>("log_format", defaultFormat);
if (config.valueOr("log_to_console", false))
{
@@ -96,14 +94,9 @@ LogService::init(Config const& config)
boost::filesystem::path dirPath{logDir.value()};
if (!boost::filesystem::exists(dirPath))
boost::filesystem::create_directories(dirPath);
auto const rotationSize =
config.valueOr<uint64_t>("log_rotation_size", 2048u) * 1024u *
1024u;
auto const rotationPeriod =
config.valueOr<uint32_t>("log_rotation_hour_interval", 12u);
auto const dirSize =
config.valueOr<uint64_t>("log_directory_max_size", 50u * 1024u) *
1024u * 1024u;
auto const rotationSize = config.valueOr<uint64_t>("log_rotation_size", 2048u) * 1024u * 1024u;
auto const rotationPeriod = config.valueOr<uint32_t>("log_rotation_hour_interval", 12u);
auto const dirSize = config.valueOr<uint64_t>("log_directory_max_size", 50u * 1024u) * 1024u * 1024u;
auto fileSink = boost::log::add_file_log(
keywords::file_name = dirPath / "clio.log",
keywords::target_file_name = dirPath / "clio_%Y-%m-%d_%H-%M-%S.log",
@@ -112,11 +105,9 @@ LogService::init(Config const& config)
keywords::open_mode = std::ios_base::app,
keywords::rotation_size = rotationSize,
keywords::time_based_rotation =
sinks::file::rotation_at_time_interval(
boost::posix_time::hours(rotationPeriod)));
sinks::file::rotation_at_time_interval(boost::posix_time::hours(rotationPeriod)));
fileSink->locked_backend()->set_file_collector(
sinks::file::make_collector(
keywords::target = dirPath, keywords::max_size = dirSize));
sinks::file::make_collector(keywords::target = dirPath, keywords::max_size = dirSize));
fileSink->locked_backend()->scan_for_files();
}
@@ -134,26 +125,19 @@ LogService::init(Config const& config)
};
auto core = boost::log::core::get();
auto min_severity = boost::log::expressions::channel_severity_filter(
log_channel, log_severity);
auto min_severity = boost::log::expressions::channel_severity_filter(log_channel, log_severity);
for (auto const& channel : channels)
min_severity[channel] = defaultSeverity;
min_severity["Alert"] =
Severity::WRN; // Channel for alerts, always warning severity
min_severity["Alert"] = Severity::WRN; // Channel for alerts, always warning severity
for (auto const overrides = config.arrayOr("log_channels", {});
auto const& cfg : overrides)
for (auto const overrides = config.arrayOr("log_channels", {}); auto const& cfg : overrides)
{
auto name = cfg.valueOrThrow<std::string>(
"channel", "Channel name is required");
auto name = cfg.valueOrThrow<std::string>("channel", "Channel name is required");
if (not std::count(std::begin(channels), std::end(channels), name))
throw std::runtime_error(
"Can't override settings for log channel " + name +
": invalid channel");
throw std::runtime_error("Can't override settings for log channel " + name + ": invalid channel");
min_severity[name] =
cfg.valueOr<Severity>("log_level", defaultSeverity);
min_severity[name] = cfg.valueOr<Severity>("log_level", defaultSeverity);
}
core->set_filter(min_severity);
@@ -202,8 +186,7 @@ Logger::Pump::pretty_path(source_location_t const& loc, size_t max_depth) const
if (idx == std::string::npos || idx == 0)
break;
}
return file_path.substr(idx == std::string::npos ? 0 : idx + 1) + ':' +
std::to_string(loc.line());
return file_path.substr(idx == std::string::npos ? 0 : idx + 1) + ':' + std::to_string(loc.line());
}
} // namespace clio

22
src/log/Logger.h
View File

@@ -68,8 +68,7 @@ class SourceLocation
std::size_t line_;
public:
SourceLocation(std::string_view file, std::size_t line)
: file_{file}, line_{line}
SourceLocation(std::string_view file, std::size_t line) : file_{file}, line_{line}
{
}
std::string_view
@@ -84,8 +83,7 @@ public:
}
};
using source_location_t = SourceLocation;
#define CURRENT_SRC_LOCATION \
source_location_t(__builtin_FILE(), __builtin_LINE())
#define CURRENT_SRC_LOCATION source_location_t(__builtin_FILE(), __builtin_LINE())
#endif
/**
@@ -121,9 +119,7 @@ operator<<(std::ostream& stream, Severity sev);
* @throws std::runtime_error Thrown if severity is not in the right format
*/
Severity
tag_invoke(
boost::json::value_to_tag<Severity>,
boost::json::value const& value);
tag_invoke(boost::json::value_to_tag<Severity>, boost::json::value const& value);
/**
* @brief A simple thread-safe logger for the channel specified
@@ -135,8 +131,7 @@ tag_invoke(
*/
class Logger final
{
using logger_t =
boost::log::sources::severity_channel_logger_mt<Severity, std::string>;
using logger_t = boost::log::sources::severity_channel_logger_mt<Severity, std::string>;
mutable logger_t logger_;
friend class LogService; // to expose the Pump interface
@@ -146,8 +141,7 @@ class Logger final
*/
class Pump final
{
using pump_opt_t =
std::optional<boost::log::aux::record_pump<logger_t>>;
using pump_opt_t = std::optional<boost::log::aux::record_pump<logger_t>>;
boost::log::record rec_;
pump_opt_t pump_ = std::nullopt;
@@ -160,8 +154,7 @@ class Logger final
if (rec_)
{
pump_.emplace(boost::log::aux::make_record_pump(logger, rec_));
pump_->stream() << boost::log::add_value(
"SourceLocation", pretty_path(loc));
pump_->stream() << boost::log::add_value("SourceLocation", pretty_path(loc));
}
}
@@ -205,8 +198,7 @@ public:
*
* @param channel The channel this logger will report into.
*/
Logger(std::string channel)
: logger_{boost::log::keywords::channel = channel}
Logger(std::string channel) : logger_{boost::log::keywords::channel = channel}
{
}
Logger(Logger const&) = default;

38
src/main/main.cpp
View File

@@ -78,12 +78,7 @@ parseCli(int argc, char* argv[])
positional.add("conf", 1);
po::variables_map parsed;
po::store(
po::command_line_parser(argc, argv)
.options(description)
.positional(positional)
.run(),
parsed);
po::store(po::command_line_parser(argc, argv).options(description).positional(positional).run(), parsed);
po::notify(parsed);
if (parsed.count("version"))
@@ -94,9 +89,7 @@ parseCli(int argc, char* argv[])
if (parsed.count("help"))
{
std::cout << "Clio server " << Build::getClioFullVersionString()
<< "\n\n"
<< description;
std::cout << "Clio server " << Build::getClioFullVersionString() << "\n\n" << description;
std::exit(EXIT_SUCCESS);
}
@@ -137,15 +130,11 @@ parseCerts(Config const& config)
ssl::context ctx{ssl::context::tlsv12};
ctx.set_options(
boost::asio::ssl::context::default_workarounds |
boost::asio::ssl::context::no_sslv2);
ctx.set_options(boost::asio::ssl::context::default_workarounds | boost::asio::ssl::context::no_sslv2);
ctx.use_certificate_chain(boost::asio::buffer(cert.data(), cert.size()));
ctx.use_private_key(
boost::asio::buffer(key.data(), key.size()),
boost::asio::ssl::context::file_format::pem);
ctx.use_private_key(boost::asio::buffer(key.data(), key.size()), boost::asio::ssl::context::file_format::pem);
return ctx;
}
@@ -175,8 +164,7 @@ try
auto const config = ConfigReader::open(configPath);
if (!config)
{
std::cerr << "Couldnt parse config '" << configPath << "'."
<< std::endl;
std::cerr << "Couldnt parse config '" << configPath << "'." << std::endl;
return EXIT_FAILURE;
}
@@ -184,9 +172,7 @@ try
LogService::info() << "Clio version: " << Build::getClioFullVersionString();
auto ctx = parseCerts(config);
auto ctxRef = ctx
? std::optional<std::reference_wrapper<ssl::context>>{ctx.value()}
: std::nullopt;
auto ctxRef = ctx ? std::optional<std::reference_wrapper<ssl::context>>{ctx.value()} : std::nullopt;
auto const threads = config.valueOr("io_threads", 2);
if (threads <= 0)
@@ -208,8 +194,7 @@ try
auto backend = Backend::make_Backend(ioc, config);
// Manages clients subscribed to streams
auto subscriptions =
SubscriptionManager::make_SubscriptionManager(config, backend);
auto subscriptions = SubscriptionManager::make_SubscriptionManager(config, backend);
// Tracks which ledgers have been validated by the
// network
@@ -220,17 +205,14 @@ try
// The server uses the balancer to forward RPCs to a rippled node.
// The balancer itself publishes to streams (transactions_proposed and
// accounts_proposed)
auto balancer = ETLLoadBalancer::make_ETLLoadBalancer(
config, ioc, backend, subscriptions, ledgers);
auto balancer = ETLLoadBalancer::make_ETLLoadBalancer(config, ioc, backend, subscriptions, ledgers);
// ETL is responsible for writing and publishing to streams. In read-only
// mode, ETL only publishes
auto etl = ReportingETL::make_ReportingETL(
config, ioc, backend, subscriptions, balancer, ledgers);
auto etl = ReportingETL::make_ReportingETL(config, ioc, backend, subscriptions, balancer, ledgers);
// The server handles incoming RPCs
auto httpServer = Server::make_HttpServer(
config, ioc, ctxRef, backend, subscriptions, balancer, etl, dosGuard);
auto httpServer = Server::make_HttpServer(config, ioc, ctxRef, backend, subscriptions, balancer, etl, dosGuard);
// Blocks until stopped.
// When stopped, shared_ptrs fall out of scope

4
src/rpc/Counters.cpp
View File

@@ -52,9 +52,7 @@ Counters::rpcErrored(std::string const& method)
}
void
Counters::rpcComplete(
std::string const& method,
std::chrono::microseconds const& rpcDuration)
Counters::rpcComplete(std::string const& method, std::chrono::microseconds const& rpcDuration)
{
if (!validHandler(method))
return;

4
src/rpc/Counters.h
View File

@@ -59,9 +59,7 @@ public:
rpcErrored(std::string const& method);
void
rpcComplete(
std::string const& method,
std::chrono::microseconds const& rpcDuration);
rpcComplete(std::string const& method, std::chrono::microseconds const& rpcDuration);
void
rpcForwarded(std::string const& method);

47
src/rpc/Errors.cpp
View File

@@ -50,8 +50,7 @@ getWarningInfo(WarningCode code)
{warnRPC_RATE_LIMIT, "You are about to be rate limited"}};
auto matchByCode = [code](auto const& info) { return info.code == code; };
if (auto it = find_if(begin(infos), end(infos), matchByCode);
it != end(infos))
if (auto it = find_if(begin(infos), end(infos), matchByCode); it != end(infos))
return *it;
throw(out_of_range("Invalid WarningCode"));
@@ -71,31 +70,21 @@ ClioErrorInfo const&
getErrorInfo(ClioError code)
{
constexpr static ClioErrorInfo infos[]{
{ClioError::rpcMALFORMED_CURRENCY,
"malformedCurrency",
"Malformed currency."},
{ClioError::rpcMALFORMED_REQUEST,
"malformedRequest",
"Malformed request."},
{ClioError::rpcMALFORMED_CURRENCY, "malformedCurrency", "Malformed currency."},
{ClioError::rpcMALFORMED_REQUEST, "malformedRequest", "Malformed request."},
{ClioError::rpcMALFORMED_OWNER, "malformedOwner", "Malformed owner."},
{ClioError::rpcMALFORMED_ADDRESS,
"malformedAddress",
"Malformed address."},
{ClioError::rpcMALFORMED_ADDRESS, "malformedAddress", "Malformed address."},
};
auto matchByCode = [code](auto const& info) { return info.code == code; };
if (auto it = find_if(begin(infos), end(infos), matchByCode);
it != end(infos))
if (auto it = find_if(begin(infos), end(infos), matchByCode); it != end(infos))
return *it;
throw(out_of_range("Invalid error code"));
}
boost::json::object
makeError(
RippledError err,
optional<string_view> customError,
optional<string_view> customMessage)
makeError(RippledError err, optional<string_view> customError, optional<string_view> customMessage)
{
boost::json::object json;
auto const& info = ripple::RPC::get_error_info(err);
@@ -109,10 +98,7 @@ makeError(
}
boost::json::object
makeError(
ClioError err,
optional<string_view> customError,
optional<string_view> customMessage)
makeError(ClioError err, optional<string_view> customError, optional<string_view> customMessage)
{
boost::json::object json;
auto const& info = getErrorInfo(err);
@@ -128,31 +114,20 @@ makeError(
boost::json::object
makeError(Status const& status)
{
auto wrapOptional = [](string_view const& str) {
return str.empty() ? nullopt : make_optional(str);
};
auto wrapOptional = [](string_view const& str) { return str.empty() ? nullopt : make_optional(str); };
auto res = visit(
overloadSet{
[&status, &wrapOptional](RippledError err) {
if (err == ripple::rpcUNKNOWN)
{
return boost::json::object{
{"error", status.message},
{"type", "response"},
{"status", "error"}};
return boost::json::object{{"error", status.message}, {"type", "response"}, {"status", "error"}};
}
return makeError(
err,
wrapOptional(status.error),
wrapOptional(status.message));
return makeError(err, wrapOptional(status.error), wrapOptional(status.message));
},
[&status, &wrapOptional](ClioError err) {
return makeError(
err,
wrapOptional(status.error),
wrapOptional(status.message));
return makeError(err, wrapOptional(status.error), wrapOptional(status.message));
},
},
status.code);

22
src/rpc/Errors.h
View File

@@ -75,24 +75,20 @@ struct Status
Status() = default;
/* implicit */ Status(CombinedError code) : code(code){};
Status(CombinedError code, boost::json::object&& extraInfo)
: code(code), extraInfo(std::move(extraInfo)){};
Status(CombinedError code, boost::json::object&& extraInfo) : code(code), extraInfo(std::move(extraInfo)){};
// HACK. Some rippled handlers explicitly specify errors.
// This means that we have to be able to duplicate this
// functionality.
explicit Status(std::string const& message)
: code(ripple::rpcUNKNOWN), message(message)
explicit Status(std::string const& message) : code(ripple::rpcUNKNOWN), message(message)
{
}
Status(CombinedError code, std::string message)
: code(code), message(message)
Status(CombinedError code, std::string message) : code(code), message(message)
{
}
Status(CombinedError code, std::string error, std::string message)
: code(code), error(error), message(message)
Status(CombinedError code, std::string error, std::string message) : code(code), error(error), message(message)
{
}
@@ -138,12 +134,7 @@ struct Status
/**
* @brief Warning codes that can be returned by clio.
*/
enum WarningCode {
warnUNKNOWN = -1,
warnRPC_CLIO = 2001,
warnRPC_OUTDATED = 2002,
warnRPC_RATE_LIMIT = 2003
};
enum WarningCode { warnUNKNOWN = -1, warnRPC_CLIO = 2001, warnRPC_OUTDATED = 2002, warnRPC_RATE_LIMIT = 2003 };
/**
* @brief Holds information about a clio warning.
@@ -151,8 +142,7 @@ enum WarningCode {
struct WarningInfo
{
constexpr WarningInfo() = default;
constexpr WarningInfo(WarningCode code, char const* message)
: code(code), message(message)
constexpr WarningInfo(WarningCode code, char const* message) : code(code), message(message)
{
}

35
src/rpc/RPC.cpp
View File

@@ -95,19 +95,7 @@ make_WsContext(
string command = commandValue.as_string().c_str();
return make_optional<Context>(
yc,
command,
1,
request,
backend,
subscriptions,
balancer,
etl,
session,
tagFactory,
range,
counters,
clientIp);
yc, command, 1, request, backend, subscriptions, balancer, etl, session, tagFactory, range, counters, clientIp);
}
optional<Context>
@@ -267,8 +255,7 @@ isClioOnly(string const& method)
bool
shouldSuppressValidatedFlag(RPC::Context const& context)
{
return boost::iequals(context.method, "subscribe") ||
boost::iequals(context.method, "unsubscribe");
return boost::iequals(context.method, "subscribe") || boost::iequals(context.method, "unsubscribe");
}
Status
@@ -278,8 +265,7 @@ getLimit(RPC::Context const& context, uint32_t& limit)
return Status{RippledError::rpcUNKNOWN_COMMAND};
if (!handlerTable.getLimitRange(context.method))
return Status{
RippledError::rpcINVALID_PARAMS, "rpcDoesNotRequireLimit"};
return Status{RippledError::rpcINVALID_PARAMS, "rpcDoesNotRequireLimit"};
auto [lo, def, hi] = *handlerTable.getLimitRange(context.method);
@@ -317,8 +303,7 @@ shouldForwardToRippled(Context const& ctx)
if (specifiesCurrentOrClosedLedger(request))
return true;
if (ctx.method == "account_info" && request.contains("queue") &&
request.at("queue").as_bool())
if (ctx.method == "account_info" && request.contains("queue") && request.at("queue").as_bool())
return true;
return false;
@@ -332,8 +317,7 @@ buildResponse(Context const& ctx)
boost::json::object toForward = ctx.params;
toForward["command"] = ctx.method;
auto res =
ctx.balancer->forwardToRippled(toForward, ctx.clientIp, ctx.yield);
auto res = ctx.balancer->forwardToRippled(toForward, ctx.clientIp, ctx.yield);
ctx.counters.rpcForwarded(ctx.method);
@@ -359,14 +343,11 @@ buildResponse(Context const& ctx)
try
{
gPerfLog.debug() << ctx.tag() << " start executing rpc `" << ctx.method
<< '`';
gPerfLog.debug() << ctx.tag() << " start executing rpc `" << ctx.method << '`';
auto v = (*method)(ctx);
gPerfLog.debug() << ctx.tag() << " finish executing rpc `" << ctx.method
<< '`';
gPerfLog.debug() << ctx.tag() << " finish executing rpc `" << ctx.method << '`';
if (auto object = get_if<boost::json::object>(&v);
object && not shouldSuppressValidatedFlag(ctx))
if (auto object = get_if<boost::json::object>(&v); object && not shouldSuppressValidatedFlag(ctx))
{
(*object)[JS(validated)] = true;
}

7
src/rpc/RPC.h
View File

@@ -150,11 +150,10 @@ logDuration(Context const& ctx, T const& dur)
{
static clio::Logger log{"RPC"};
std::stringstream ss;
ss << ctx.tag() << "Request processing duration = "
<< std::chrono::duration_cast<std::chrono::milliseconds>(dur).count()
ss << ctx.tag()
<< "Request processing duration = " << std::chrono::duration_cast<std::chrono::milliseconds>(dur).count()
<< " milliseconds. request = " << ctx.params;
auto seconds =
std::chrono::duration_cast<std::chrono::seconds>(dur).count();
auto seconds = std::chrono::duration_cast<std::chrono::seconds>(dur).count();
if (seconds > 10)
log.error() << ss.str();
else if (seconds > 1)

368
src/rpc/RPCHelpers.cpp
View File

@@ -47,10 +47,7 @@ getBool(boost::json::object const& request, std::string const& field)
}
bool
getBool(
boost::json::object const& request,
std::string const& field,
bool dfault)
getBool(boost::json::object const& request, std::string const& field, bool dfault)
{
if (auto res = getBool(request, field))
return *res;
@@ -81,10 +78,7 @@ getUInt(boost::json::object const& request, std::string const& field)
}
std::uint32_t
getUInt(
boost::json::object const& request,
std::string const& field,
std::uint32_t const dfault)
getUInt(boost::json::object const& request, std::string const& field, std::uint32_t const dfault)
{
if (auto res = getUInt(request, field))
return *res;
@@ -156,10 +150,7 @@ getRequiredString(boost::json::object const& request, std::string const& field)
}
std::string
getString(
boost::json::object const& request,
std::string const& field,
std::string dfault)
getString(boost::json::object const& request, std::string const& field, std::string dfault)
{
if (auto res = getString(request, field))
return *res;
@@ -192,25 +183,21 @@ getAccount(
if (!request.contains(field))
{
if (required)
return Status{
RippledError::rpcINVALID_PARAMS, field.to_string() + "Missing"};
return Status{RippledError::rpcINVALID_PARAMS, field.to_string() + "Missing"};
return {};
}
if (!request.at(field).is_string())
return Status{
RippledError::rpcINVALID_PARAMS, field.to_string() + "NotString"};
return Status{RippledError::rpcINVALID_PARAMS, field.to_string() + "NotString"};
if (auto a = accountFromStringStrict(request.at(field).as_string().c_str());
a)
if (auto a = accountFromStringStrict(request.at(field).as_string().c_str()); a)
{
account = a.value();
return {};
}
return Status{
RippledError::rpcACT_MALFORMED, field.to_string() + "Malformed"};
return Status{RippledError::rpcACT_MALFORMED, field.to_string() + "Malformed"};
}
Status
@@ -226,18 +213,15 @@ getOptionalAccount(
}
if (!request.at(field).is_string())
return Status{
RippledError::rpcINVALID_PARAMS, field.to_string() + "NotString"};
return Status{RippledError::rpcINVALID_PARAMS, field.to_string() + "NotString"};
if (auto a = accountFromStringStrict(request.at(field).as_string().c_str());
a)
if (auto a = accountFromStringStrict(request.at(field).as_string().c_str()); a)
{
account = a.value();
return {};
}
return Status{
RippledError::rpcINVALID_PARAMS, field.to_string() + "Malformed"};
return Status{RippledError::rpcINVALID_PARAMS, field.to_string() + "Malformed"};
}
Status
@@ -247,10 +231,7 @@ getAccount(boost::json::object const& request, ripple::AccountID& accountId)
}
Status
getAccount(
boost::json::object const& request,
ripple::AccountID& destAccount,
boost::string_view const& field)
getAccount(boost::json::object const& request, ripple::AccountID& destAccount, boost::string_view const& field)
{
return getAccount(request, destAccount, field, false);
}
@@ -320,8 +301,7 @@ canHaveDeliveredAmount(
std::shared_ptr<ripple::TxMeta const> const& meta)
{
ripple::TxType const tt{txn->getTxnType()};
if (tt != ripple::ttPAYMENT && tt != ripple::ttCHECK_CASH &&
tt != ripple::ttACCOUNT_DELETE)
if (tt != ripple::ttPAYMENT && tt != ripple::ttCHECK_CASH && tt != ripple::ttACCOUNT_DELETE)
return false;
/*
@@ -343,13 +323,11 @@ accountFromStringStrict(std::string const& account)
std::optional<ripple::PublicKey> publicKey = {};
if (blob && ripple::publicKeyType(ripple::makeSlice(*blob)))
{
publicKey =
ripple::PublicKey(ripple::Slice{blob->data(), blob->size()});
publicKey = ripple::PublicKey(ripple::Slice{blob->data(), blob->size()});
}
else
{
publicKey = ripple::parseBase58<ripple::PublicKey>(
ripple::TokenType::AccountPublic, account);
publicKey = ripple::parseBase58<ripple::PublicKey>(ripple::TokenType::AccountPublic, account);
}
std::optional<ripple::AccountID> result;
@@ -363,26 +341,19 @@ accountFromStringStrict(std::string const& account)
else
return {};
}
std::pair<
std::shared_ptr<ripple::STTx const>,
std::shared_ptr<ripple::STObject const>>
std::pair<std::shared_ptr<ripple::STTx const>, std::shared_ptr<ripple::STObject const>>
deserializeTxPlusMeta(Backend::TransactionAndMetadata const& blobs)
{
try
{
std::pair<
std::shared_ptr<ripple::STTx const>,
std::shared_ptr<ripple::STObject const>>
result;
std::pair<std::shared_ptr<ripple::STTx const>, std::shared_ptr<ripple::STObject const>> result;
{
ripple::SerialIter s{
blobs.transaction.data(), blobs.transaction.size()};
ripple::SerialIter s{blobs.transaction.data(), blobs.transaction.size()};
result.first = std::make_shared<ripple::STTx const>(s);
}
{
ripple::SerialIter s{blobs.metadata.data(), blobs.metadata.size()};
result.second =
std::make_shared<ripple::STObject const>(s, ripple::sfMetadata);
result.second = std::make_shared<ripple::STObject const>(s, ripple::sfMetadata);
}
return result;
}
@@ -390,34 +361,21 @@ deserializeTxPlusMeta(Backend::TransactionAndMetadata const& blobs)
{
std::stringstream txn;
std::stringstream meta;
std::copy(
blobs.transaction.begin(),
blobs.transaction.end(),
std::ostream_iterator<unsigned char>(txn));
std::copy(
blobs.metadata.begin(),
blobs.metadata.end(),
std::ostream_iterator<unsigned char>(meta));
gLog.error() << "Failed to deserialize transaction. txn = " << txn.str()
<< " - meta = " << meta.str() << " txn length = "
<< std::to_string(blobs.transaction.size())
<< " meta length = "
<< std::to_string(blobs.metadata.size());
std::copy(blobs.transaction.begin(), blobs.transaction.end(), std::ostream_iterator<unsigned char>(txn));
std::copy(blobs.metadata.begin(), blobs.metadata.end(), std::ostream_iterator<unsigned char>(meta));
gLog.error() << "Failed to deserialize transaction. txn = " << txn.str() << " - meta = " << meta.str()
<< " txn length = " << std::to_string(blobs.transaction.size())
<< " meta length = " << std::to_string(blobs.metadata.size());
throw e;
}
}
std::pair<
std::shared_ptr<ripple::STTx const>,
std::shared_ptr<ripple::TxMeta const>>
deserializeTxPlusMeta(
Backend::TransactionAndMetadata const& blobs,
std::uint32_t seq)
std::pair<std::shared_ptr<ripple::STTx const>, std::shared_ptr<ripple::TxMeta const>>
deserializeTxPlusMeta(Backend::TransactionAndMetadata const& blobs, std::uint32_t seq)
{
auto [tx, meta] = deserializeTxPlusMeta(blobs);
std::shared_ptr<ripple::TxMeta> m =
std::make_shared<ripple::TxMeta>(tx->getTransactionID(), seq, *meta);
std::shared_ptr<ripple::TxMeta> m = std::make_shared<ripple::TxMeta>(tx->getTransactionID(), seq, *meta);
return {tx, m};
}
@@ -425,8 +383,7 @@ deserializeTxPlusMeta(
boost::json::object
toJson(ripple::STBase const& obj)
{
boost::json::value value = boost::json::parse(
obj.getJson(ripple::JsonOptions::none).toStyledString());
boost::json::value value = boost::json::parse(obj.getJson(ripple::JsonOptions::none).toStyledString());
return value.as_object();
}
@@ -451,8 +408,7 @@ insertDeliveredAmount(
if (canHaveDeliveredAmount(txn, meta))
{
if (auto amt = getDeliveredAmount(txn, meta, meta->getLgrSeq(), date))
metaJson["delivered_amount"] =
toBoostJson(amt->getJson(ripple::JsonOptions::include_date));
metaJson["delivered_amount"] = toBoostJson(amt->getJson(ripple::JsonOptions::include_date));
else
metaJson["delivered_amount"] = "unavailable";
return true;
@@ -463,8 +419,7 @@ insertDeliveredAmount(
boost::json::object
toJson(ripple::TxMeta const& meta)
{
boost::json::value value = boost::json::parse(
meta.getJson(ripple::JsonOptions::none).toStyledString());
boost::json::value value = boost::json::parse(meta.getJson(ripple::JsonOptions::none).toStyledString());
return value.as_object();
}
@@ -480,8 +435,7 @@ toBoostJson(Json::Value const& value)
boost::json::object
toJson(ripple::SLE const& sle)
{
boost::json::value value = boost::json::parse(
sle.getJson(ripple::JsonOptions::none).toStyledString());
boost::json::value value = boost::json::parse(sle.getJson(ripple::JsonOptions::none).toStyledString());
if (sle.getType() == ripple::ltACCOUNT_ROOT)
{
if (sle.isFieldPresent(ripple::sfEmailHash))
@@ -489,8 +443,7 @@ toJson(ripple::SLE const& sle)
auto const& hash = sle.getFieldH128(ripple::sfEmailHash);
std::string md5 = strHex(hash);
boost::algorithm::to_lower(md5);
value.as_object()["urlgravatar"] =
str(boost::format("http://www.gravatar.com/avatar/%s") % md5);
value.as_object()["urlgravatar"] = str(boost::format("http://www.gravatar.com/avatar/%s") % md5);
}
}
return value.as_object();
@@ -509,8 +462,7 @@ toJson(ripple::LedgerInfo const& lgrInfo)
header["close_flags"] = lgrInfo.closeFlags;
// Always show fields that contribute to the ledger hash
header["parent_close_time"] =
lgrInfo.parentCloseTime.time_since_epoch().count();
header["parent_close_time"] = lgrInfo.parentCloseTime.time_since_epoch().count();
header["close_time"] = lgrInfo.closeTime.time_since_epoch().count();
header["close_time_resolution"] = lgrInfo.closeTimeResolution.count();
return header;
@@ -534,20 +486,16 @@ parseStringAsUInt(std::string const& value)
std::variant<Status, ripple::LedgerInfo>
ledgerInfoFromRequest(Context const& ctx)
{
auto hashValue = ctx.params.contains("ledger_hash")
? ctx.params.at("ledger_hash")
: nullptr;
auto hashValue = ctx.params.contains("ledger_hash") ? ctx.params.at("ledger_hash") : nullptr;
if (!hashValue.is_null())
{
if (!hashValue.is_string())
return Status{
RippledError::rpcINVALID_PARAMS, "ledgerHashNotString"};
return Status{RippledError::rpcINVALID_PARAMS, "ledgerHashNotString"};
ripple::uint256 ledgerHash;
if (!ledgerHash.parseHex(hashValue.as_string().c_str()))
return Status{
RippledError::rpcINVALID_PARAMS, "ledgerHashMalformed"};
return Status{RippledError::rpcINVALID_PARAMS, "ledgerHashMalformed"};
auto lgrInfo = ctx.backend->fetchLedgerByHash(ledgerHash, ctx.yield);
@@ -557,9 +505,7 @@ ledgerInfoFromRequest(Context const& ctx)
return *lgrInfo;
}
auto indexValue = ctx.params.contains("ledger_index")
? ctx.params.at("ledger_index")
: nullptr;
auto indexValue = ctx.params.contains("ledger_index") ? ctx.params.at("ledger_index") : nullptr;
std::optional<std::uint32_t> ledgerSequence = {};
if (!indexValue.is_null())
@@ -583,8 +529,7 @@ ledgerInfoFromRequest(Context const& ctx)
if (!ledgerSequence)
return Status{RippledError::rpcINVALID_PARAMS, "ledgerIndexMalformed"};
auto lgrInfo =
ctx.backend->fetchLedgerBySequence(*ledgerSequence, ctx.yield);
auto lgrInfo = ctx.backend->fetchLedgerBySequence(*ledgerSequence, ctx.yield);
if (!lgrInfo || lgrInfo->seq > ctx.range.maxSequence)
return Status{RippledError::rpcLGR_NOT_FOUND, "ledgerNotFound"};
@@ -602,8 +547,7 @@ getLedgerInfoFromHashOrSeq(
uint32_t maxSeq)
{
std::optional<ripple::LedgerInfo> lgrInfo;
auto const err =
RPC::Status{RPC::RippledError::rpcLGR_NOT_FOUND, "ledgerNotFound"};
auto const err = RPC::Status{RPC::RippledError::rpcLGR_NOT_FOUND, "ledgerNotFound"};
if (ledgerHash)
{
// invoke uint256's constructor to parse the hex string , instead of
@@ -652,8 +596,7 @@ getStartHint(ripple::SLE const& sle, ripple::AccountID const& accountID)
{
if (sle.getFieldAmount(ripple::sfLowLimit).getIssuer() == accountID)
return sle.getFieldU64(ripple::sfLowNode);
else if (
sle.getFieldAmount(ripple::sfHighLimit).getIssuer() == accountID)
else if (sle.getFieldAmount(ripple::sfHighLimit).getIssuer() == accountID)
return sle.getFieldU64(ripple::sfHighNode);
}
@@ -673,8 +616,7 @@ traverseOwnedNodes(
boost::asio::yield_context& yield,
std::function<void(ripple::SLE&&)> atOwnedNode)
{
if (!backend.fetchLedgerObject(
ripple::keylet::account(accountID).key, sequence, yield))
if (!backend.fetchLedgerObject(ripple::keylet::account(accountID).key, sequence, yield))
return Status{RippledError::rpcACT_NOT_FOUND};
auto const maybeCursor = parseAccountCursor(jsonCursor);
@@ -752,8 +694,7 @@ traverseOwnedNodes(
if (hexMarker.isNonZero())
{
auto const hintIndex = ripple::keylet::page(rootIndex, startHint);
auto hintDir =
backend.fetchLedgerObject(hintIndex.key, sequence, yield);
auto hintDir = backend.fetchLedgerObject(hintIndex.key, sequence, yield);
if (!hintDir)
return Status(ripple::rpcINVALID_PARAMS, "Invalid marker");
@@ -762,8 +703,7 @@ traverseOwnedNodes(
ripple::SLE sle{it, hintIndex.key};
if (auto const& indexes = sle.getFieldV256(ripple::sfIndexes);
std::find(std::begin(indexes), std::end(indexes), hexMarker) ==
std::end(indexes))
std::find(std::begin(indexes), std::end(indexes), hexMarker) == std::end(indexes))
{
// result in empty dataset
return AccountCursor({beast::zero, 0});
@@ -773,12 +713,10 @@ traverseOwnedNodes(
bool found = false;
for (;;)
{
auto const ownerDir =
backend.fetchLedgerObject(currentIndex.key, sequence, yield);
auto const ownerDir = backend.fetchLedgerObject(currentIndex.key, sequence, yield);
if (!ownerDir)
return Status(
ripple::rpcINVALID_PARAMS, "Owner directory not found");
return Status(ripple::rpcINVALID_PARAMS, "Owner directory not found");
ripple::SerialIter it{ownerDir->data(), ownerDir->size()};
ripple::SLE sle{it, currentIndex.key};
@@ -819,8 +757,7 @@ traverseOwnedNodes(
{
for (;;)
{
auto const ownerDir =
backend.fetchLedgerObject(currentIndex.key, sequence, yield);
auto const ownerDir = backend.fetchLedgerObject(currentIndex.key, sequence, yield);
if (!ownerDir)
break;
@@ -853,16 +790,11 @@ traverseOwnedNodes(
auto end = std::chrono::system_clock::now();
gLog.debug() << "Time loading owned directories: "
<< std::chrono::duration_cast<std::chrono::milliseconds>(
end - start)
.count()
<< " milliseconds";
<< std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count() << " milliseconds";
auto [objects, timeDiff] = util::timed(
[&]() { return backend.fetchLedgerObjects(keys, sequence, yield); });
auto [objects, timeDiff] = util::timed([&]() { return backend.fetchLedgerObjects(keys, sequence, yield); });
gLog.debug() << "Time loading owned entries: " << timeDiff
<< " milliseconds";
gLog.debug() << "Time loading owned entries: " << timeDiff << " milliseconds";
for (auto i = 0; i < objects.size(); ++i)
{
@@ -877,17 +809,11 @@ traverseOwnedNodes(
}
std::shared_ptr<ripple::SLE const>
read(
ripple::Keylet const& keylet,
ripple::LedgerInfo const& lgrInfo,
Context const& context)
read(ripple::Keylet const& keylet, ripple::LedgerInfo const& lgrInfo, Context const& context)
{
if (auto const blob = context.backend->fetchLedgerObject(
keylet.key, lgrInfo.seq, context.yield);
blob)
if (auto const blob = context.backend->fetchLedgerObject(keylet.key, lgrInfo.seq, context.yield); blob)
{
return std::make_shared<ripple::SLE const>(
ripple::SerialIter{blob->data(), blob->size()}, keylet.key);
return std::make_shared<ripple::SLE const>(ripple::SerialIter{blob->data(), blob->size()}, keylet.key);
}
return nullptr;
@@ -902,11 +828,9 @@ parseRippleLibSeed(boost::json::value const& value)
if (!value.is_string())
return {};
auto const result = ripple::decodeBase58Token(
value.as_string().c_str(), ripple::TokenType::None);
auto const result = ripple::decodeBase58Token(value.as_string().c_str(), ripple::TokenType::None);
if (result.size() == 18 &&
static_cast<std::uint8_t>(result[0]) == std::uint8_t(0xE1) &&
if (result.size() == 18 && static_cast<std::uint8_t>(result[0]) == std::uint8_t(0xE1) &&
static_cast<std::uint8_t>(result[1]) == std::uint8_t(0x4B))
return ripple::Seed(ripple::makeSlice(result.substr(2)));
@@ -920,8 +844,7 @@ keypairFromRequst(boost::json::object const& request)
// All of the secret types we allow, but only one at a time.
// The array should be constexpr, but that makes Visual Studio unhappy.
static std::string const secretTypes[]{
"passphrase", "secret", "seed", "seed_hex"};
static std::string const secretTypes[]{"passphrase", "secret", "seed", "seed_hex"};
// Identify which secret type is in use.
std::string secretType = "";
@@ -958,13 +881,10 @@ keypairFromRequst(boost::json::object const& request)
keyType = ripple::keyTypeFromString(key_type);
if (!keyType)
return Status{
RippledError::rpcINVALID_PARAMS, "invalidFieldKeyType"};
return Status{RippledError::rpcINVALID_PARAMS, "invalidFieldKeyType"};
if (secretType == "secret")
return Status{
RippledError::rpcINVALID_PARAMS,
"The secret field is not allowed if key_type is used."};
return Status{RippledError::rpcINVALID_PARAMS, "The secret field is not allowed if key_type is used."};
}
// ripple-lib encodes seed used to generate an Ed25519 wallet in a
@@ -978,11 +898,8 @@ keypairFromRequst(boost::json::object const& request)
{
// If the user passed in an Ed25519 seed but *explicitly*
// requested another key type, return an error.
if (keyType.value_or(ripple::KeyType::ed25519) !=
ripple::KeyType::ed25519)
return Status{
RippledError::rpcINVALID_PARAMS,
"Specified seed is for an Ed25519 wallet."};
if (keyType.value_or(ripple::KeyType::ed25519) != ripple::KeyType::ed25519)
return Status{RippledError::rpcINVALID_PARAMS, "Specified seed is for an Ed25519 wallet."};
keyType = ripple::KeyType::ed25519;
}
@@ -996,9 +913,7 @@ keypairFromRequst(boost::json::object const& request)
if (has_key_type)
{
if (!request.at(secretType).is_string())
return Status{
RippledError::rpcINVALID_PARAMS,
"secret value must be string"};
return Status{RippledError::rpcINVALID_PARAMS, "secret value must be string"};
std::string key = request.at(secretType).as_string().c_str();
@@ -1016,9 +931,7 @@ keypairFromRequst(boost::json::object const& request)
else
{
if (!request.at("secret").is_string())
return Status{
RippledError::rpcINVALID_PARAMS,
"field secret should be a string"};
return Status{RippledError::rpcINVALID_PARAMS, "field secret should be a string"};
std::string secret = request.at("secret").as_string().c_str();
seed = ripple::parseGenericSeed(secret);
@@ -1026,15 +939,10 @@ keypairFromRequst(boost::json::object const& request)
}
if (!seed)
return Status{
RippledError::rpcBAD_SEED,
"Bad Seed: invalid field message secretType"};
return Status{RippledError::rpcBAD_SEED, "Bad Seed: invalid field message secretType"};
if (keyType != ripple::KeyType::secp256k1 &&
keyType != ripple::KeyType::ed25519)
return Status{
RippledError::rpcINVALID_PARAMS,
"keypairForSignature: invalid key type"};
if (keyType != ripple::KeyType::secp256k1 && keyType != ripple::KeyType::ed25519)
return Status{RippledError::rpcINVALID_PARAMS, "keypairForSignature: invalid key type"};
return generateKeyPair(*keyType, *seed);
}
@@ -1120,8 +1028,7 @@ isFrozen(
ripple::SerialIter issuerIt{blob->data(), blob->size()};
ripple::SLE issuerLine{issuerIt, key};
auto frozen =
(issuer > account) ? ripple::lsfHighFreeze : ripple::lsfLowFreeze;
auto frozen = (issuer > account) ? ripple::lsfHighFreeze : ripple::lsfLowFreeze;
if (issuerLine.isFlag(frozen))
return true;
@@ -1148,8 +1055,7 @@ xrpLiquid(
std::uint32_t const ownerCount = sle.getFieldU32(ripple::sfOwnerCount);
auto const reserve =
backend.fetchFees(sequence, yield)->accountReserve(ownerCount);
auto const reserve = backend.fetchFees(sequence, yield)->accountReserve(ownerCount);
auto const balance = sle.getFieldAmount(ripple::sfBalance);
@@ -1174,14 +1080,7 @@ accountFunds(
}
else
{
return accountHolds(
backend,
sequence,
id,
amount.getCurrency(),
amount.getIssuer(),
true,
yield);
return accountHolds(backend, sequence, id, amount.getCurrency(), amount.getIssuer(), true, yield);
}
}
@@ -1213,8 +1112,7 @@ accountHolds(
ripple::SerialIter it{blob->data(), blob->size()};
ripple::SLE sle{it, key};
if (zeroIfFrozen &&
isFrozen(backend, sequence, account, currency, issuer, yield))
if (zeroIfFrozen && isFrozen(backend, sequence, account, currency, issuer, yield))
{
amount.clear(ripple::Issue(currency, issuer));
}
@@ -1267,8 +1165,7 @@ postProcessOrderBook(
std::map<ripple::AccountID, ripple::STAmount> umBalance;
bool globalFreeze =
isGlobalFrozen(backend, ledgerSequence, book.out.account, yield) ||
bool globalFreeze = isGlobalFrozen(backend, ledgerSequence, book.out.account, yield) ||
isGlobalFrozen(backend, ledgerSequence, book.in.account, yield);
auto rate = transferRate(backend, ledgerSequence, book.out.account, yield);
@@ -1279,8 +1176,7 @@ postProcessOrderBook(
{
ripple::SerialIter it{obj.blob.data(), obj.blob.size()};
ripple::SLE offer{it, obj.key};
ripple::uint256 bookDir =
offer.getFieldH256(ripple::sfBookDirectory);
ripple::uint256 bookDir = offer.getFieldH256(ripple::sfBookDirectory);
auto const uOfferOwnerID = offer.getAccountID(ripple::sfAccount);
auto const& saTakerGets = offer.getFieldAmount(ripple::sfTakerGets);
@@ -1313,13 +1209,7 @@ postProcessOrderBook(
else
{
saOwnerFunds = accountHolds(
backend,
ledgerSequence,
uOfferOwnerID,
book.out.currency,
book.out.account,
true,
yield);
backend, ledgerSequence, uOfferOwnerID, book.out.currency, book.out.account, true, yield);
if (saOwnerFunds < beast::zero)
saOwnerFunds.clear();
@@ -1331,8 +1221,7 @@ postProcessOrderBook(
ripple::STAmount saTakerGetsFunded;
ripple::STAmount saOwnerFundsLimit = saOwnerFunds;
ripple::Rate offerRate = ripple::parityRate;
ripple::STAmount dirRate =
ripple::amountFromQuality(getQuality(bookDir));
ripple::STAmount dirRate = ripple::amountFromQuality(getQuality(bookDir));
if (rate != ripple::parityRate
// Have a tranfer fee.
@@ -1354,21 +1243,15 @@ postProcessOrderBook(
else
{
saTakerGetsFunded = saOwnerFundsLimit;
offerJson["taker_gets_funded"] = toBoostJson(
saTakerGetsFunded.getJson(ripple::JsonOptions::none));
offerJson["taker_pays_funded"] = toBoostJson(
std::min(
saTakerPays,
ripple::multiply(
saTakerGetsFunded, dirRate, saTakerPays.issue()))
.getJson(ripple::JsonOptions::none));
offerJson["taker_gets_funded"] = toBoostJson(saTakerGetsFunded.getJson(ripple::JsonOptions::none));
offerJson["taker_pays_funded"] =
toBoostJson(std::min(saTakerPays, ripple::multiply(saTakerGetsFunded, dirRate, saTakerPays.issue()))
.getJson(ripple::JsonOptions::none));
}
ripple::STAmount saOwnerPays = (ripple::parityRate == offerRate)
? saTakerGetsFunded
: std::min(
saOwnerFunds,
ripple::multiply(saTakerGetsFunded, offerRate));
: std::min(saOwnerFunds, ripple::multiply(saTakerGetsFunded, offerRate));
umBalance[uOfferOwnerID] = saOwnerFunds - saOwnerPays;
@@ -1389,11 +1272,7 @@ postProcessOrderBook(
// get book via currency type
std::variant<Status, ripple::Book>
parseBook(
ripple::Currency pays,
ripple::AccountID payIssuer,
ripple::Currency gets,
ripple::AccountID getIssuer)
parseBook(ripple::Currency pays, ripple::AccountID payIssuer, ripple::Currency gets, ripple::AccountID getIssuer)
{
if (isXRP(pays) && !isXRP(payIssuer))
return Status{
@@ -1415,8 +1294,7 @@ parseBook(
if (!ripple::isXRP(gets) && ripple::isXRP(getIssuer))
return Status{
RippledError::rpcDST_ISR_MALFORMED,
"Invalid field 'taker_gets.issuer', expected non-XRP issuer."};
RippledError::rpcDST_ISR_MALFORMED, "Invalid field 'taker_gets.issuer', expected non-XRP issuer."};
if (pays == gets && payIssuer == getIssuer)
return Status{RippledError::rpcBAD_MARKET, "badMarket"};
@@ -1428,22 +1306,16 @@ std::variant<Status, ripple::Book>
parseBook(boost::json::object const& request)
{
if (!request.contains("taker_pays"))
return Status{
RippledError::rpcINVALID_PARAMS, "Missing field 'taker_pays'"};
return Status{RippledError::rpcINVALID_PARAMS, "Missing field 'taker_pays'"};
if (!request.contains("taker_gets"))
return Status{
RippledError::rpcINVALID_PARAMS, "Missing field 'taker_gets'"};
return Status{RippledError::rpcINVALID_PARAMS, "Missing field 'taker_gets'"};
if (!request.at("taker_pays").is_object())
return Status{
RippledError::rpcINVALID_PARAMS,
"Field 'taker_pays' is not an object"};
return Status{RippledError::rpcINVALID_PARAMS, "Field 'taker_pays' is not an object"};
if (!request.at("taker_gets").is_object())
return Status{
RippledError::rpcINVALID_PARAMS,
"Field 'taker_gets' is not an object"};
return Status{RippledError::rpcINVALID_PARAMS, "Field 'taker_gets' is not an object"};
auto taker_pays = request.at("taker_pays").as_object();
if (!taker_pays.contains("currency"))
@@ -1462,24 +1334,20 @@ parseBook(boost::json::object const& request)
};
ripple::Currency pay_currency;
if (!ripple::to_currency(
pay_currency, taker_pays.at("currency").as_string().c_str()))
if (!ripple::to_currency(pay_currency, taker_pays.at("currency").as_string().c_str()))
return Status{RippledError::rpcSRC_CUR_MALFORMED};
ripple::Currency get_currency;
if (!ripple::to_currency(
get_currency, taker_gets["currency"].as_string().c_str()))
if (!ripple::to_currency(get_currency, taker_gets["currency"].as_string().c_str()))
return Status{RippledError::rpcDST_AMT_MALFORMED};
ripple::AccountID pay_issuer;
if (taker_pays.contains("issuer"))
{
if (!taker_pays.at("issuer").is_string())
return Status{
RippledError::rpcINVALID_PARAMS, "takerPaysIssuerNotString"};
return Status{RippledError::rpcINVALID_PARAMS, "takerPaysIssuerNotString"};
if (!ripple::to_issuer(
pay_issuer, taker_pays.at("issuer").as_string().c_str()))
if (!ripple::to_issuer(pay_issuer, taker_pays.at("issuer").as_string().c_str()))
return Status{RippledError::rpcSRC_ISR_MALFORMED};
if (pay_issuer == ripple::noAccount())
@@ -1503,23 +1371,17 @@ parseBook(boost::json::object const& request)
"issuer."};
if ((!isXRP(pay_currency)) && (!taker_pays.contains("issuer")))
return Status{
RippledError::rpcSRC_ISR_MALFORMED, "Missing non-XRP issuer."};
return Status{RippledError::rpcSRC_ISR_MALFORMED, "Missing non-XRP issuer."};
ripple::AccountID get_issuer;
if (taker_gets.contains("issuer"))
{
if (!taker_gets["issuer"].is_string())
return Status{
RippledError::rpcINVALID_PARAMS,
"taker_gets.issuer should be string"};
return Status{RippledError::rpcINVALID_PARAMS, "taker_gets.issuer should be string"};
if (!ripple::to_issuer(
get_issuer, taker_gets.at("issuer").as_string().c_str()))
return Status{
RippledError::rpcDST_ISR_MALFORMED,
"Invalid field 'taker_gets.issuer', bad issuer."};
if (!ripple::to_issuer(get_issuer, taker_gets.at("issuer").as_string().c_str()))
return Status{RippledError::rpcDST_ISR_MALFORMED, "Invalid field 'taker_gets.issuer', bad issuer."};
if (get_issuer == ripple::noAccount())
return Status{
@@ -1540,8 +1402,7 @@ parseBook(boost::json::object const& request)
if (!ripple::isXRP(get_currency) && ripple::isXRP(get_issuer))
return Status{
RippledError::rpcDST_ISR_MALFORMED,
"Invalid field 'taker_gets.issuer', expected non-XRP issuer."};
RippledError::rpcDST_ISR_MALFORMED, "Invalid field 'taker_gets.issuer', expected non-XRP issuer."};
if (pay_currency == get_currency && pay_issuer == get_issuer)
return Status{RippledError::rpcBAD_MARKET, "badMarket"};
@@ -1623,27 +1484,22 @@ traverseTransactions(
if (obj.contains(JS(seq)))
{
if (!obj.at(JS(seq)).is_int64())
return Status{
RippledError::rpcINVALID_PARAMS, "transactionIndexNotInt"};
return Status{RippledError::rpcINVALID_PARAMS, "transactionIndexNotInt"};
transactionIndex =
boost::json::value_to<std::uint32_t>(obj.at(JS(seq)));
transactionIndex = boost::json::value_to<std::uint32_t>(obj.at(JS(seq)));
}
std::optional<std::uint32_t> ledgerIndex = {};
if (obj.contains(JS(ledger)))
{
if (!obj.at(JS(ledger)).is_int64())
return Status{
RippledError::rpcINVALID_PARAMS, "ledgerIndexNotInt"};
return Status{RippledError::rpcINVALID_PARAMS, "ledgerIndexNotInt"};
ledgerIndex =
boost::json::value_to<std::uint32_t>(obj.at(JS(ledger)));
ledgerIndex = boost::json::value_to<std::uint32_t>(obj.at(JS(ledger)));
}
if (!transactionIndex || !ledgerIndex)
return Status{
RippledError::rpcINVALID_PARAMS, "missingLedgerOrSeq"};
return Status{RippledError::rpcINVALID_PARAMS, "missingLedgerOrSeq"};
cursor = {*ledgerIndex, *transactionIndex};
}
@@ -1657,19 +1513,15 @@ traverseTransactions(
{
if (!request.at(JS(ledger_index_min)).is_int64())
{
return Status{
RippledError::rpcINVALID_PARAMS, "ledgerSeqMinNotNumber"};
return Status{RippledError::rpcINVALID_PARAMS, "ledgerSeqMinNotNumber"};
}
min = request.at(JS(ledger_index_min)).as_int64();
if (*min != -1)
{
if (context.range.maxSequence < *min ||
context.range.minSequence > *min)
return Status{
RippledError::rpcLGR_IDX_MALFORMED,
"ledgerSeqMinOutOfRange"};
if (context.range.maxSequence < *min || context.range.minSequence > *min)
return Status{RippledError::rpcLGR_IDX_MALFORMED, "ledgerSeqMinOutOfRange"};
else
minIndex = static_cast<uint32_t>(*min);
}
@@ -1682,16 +1534,14 @@ traverseTransactions(
{
if (!request.at(JS(ledger_index_max)).is_int64())
{
return Status{
RippledError::rpcINVALID_PARAMS, "ledgerSeqMaxNotNumber"};
return Status{RippledError::rpcINVALID_PARAMS, "ledgerSeqMaxNotNumber"};
}
max = request.at(JS(ledger_index_max)).as_int64();
if (*max != -1)
{
if (context.range.maxSequence < *max ||
context.range.minSequence > *max)
if (context.range.maxSequence < *max || context.range.minSequence > *max)
return Status{RippledError::rpcLGR_IDXS_INVALID};
else
maxIndex = static_cast<uint32_t>(*max);
@@ -1711,11 +1561,8 @@ traverseTransactions(
if (request.contains(JS(ledger_index)) || request.contains(JS(ledger_hash)))
{
if (request.contains(JS(ledger_index_max)) ||
request.contains(JS(ledger_index_min)))
return Status{
RippledError::rpcINVALID_PARAMS,
"containsLedgerSpecifierAndRange"};
if (request.contains(JS(ledger_index_max)) || request.contains(JS(ledger_index_min)))
return Status{RippledError::rpcINVALID_PARAMS, "containsLedgerSpecifierAndRange"};
auto v = ledgerInfoFromRequest(context);
if (auto status = std::get_if<Status>(&v); status)
@@ -1740,8 +1587,7 @@ traverseTransactions(
response[JS(limit)] = limit;
boost::json::array txns;
auto [blobs, retCursor] = transactionFetcher(
context.backend, limit, forward, cursor, context.yield);
auto [blobs, retCursor] = transactionFetcher(context.backend, limit, forward, cursor, context.yield);
auto timeDiff = util::timed([&, &retCursor = retCursor, &blobs = blobs]() {
if (retCursor)
{
@@ -1761,8 +1607,7 @@ traverseTransactions(
}
else if (txnPlusMeta.ledgerSequence > maxIndex && !forward)
{
gLog.debug()
<< "Skipping over transactions from incomplete ledger";
gLog.debug() << "Skipping over transactions from incomplete ledger";
continue;
}
@@ -1773,8 +1618,7 @@ traverseTransactions(
auto [txn, meta] = toExpandedJson(txnPlusMeta);
obj[JS(meta)] = meta;
obj[JS(tx)] = txn;
obj[JS(tx)].as_object()[JS(ledger_index)] =
txnPlusMeta.ledgerSequence;
obj[JS(tx)].as_object()[JS(ledger_index)] = txnPlusMeta.ledgerSequence;
obj[JS(tx)].as_object()[JS(date)] = txnPlusMeta.date;
}
else

47
src/rpc/RPCHelpers.h
View File

@@ -53,18 +53,12 @@ std::optional<AccountCursor>
parseAccountCursor(std::optional<std::string> jsonCursor);
// TODO this function should probably be in a different file and namespace
std::pair<
std::shared_ptr<ripple::STTx const>,
std::shared_ptr<ripple::STObject const>>
std::pair<std::shared_ptr<ripple::STTx const>, std::shared_ptr<ripple::STObject const>>
deserializeTxPlusMeta(Backend::TransactionAndMetadata const& blobs);
// TODO this function should probably be in a different file and namespace
std::pair<
std::shared_ptr<ripple::STTx const>,
std::shared_ptr<ripple::TxMeta const>>
deserializeTxPlusMeta(
Backend::TransactionAndMetadata const& blobs,
std::uint32_t seq);
std::pair<std::shared_ptr<ripple::STTx const>, std::shared_ptr<ripple::TxMeta const>>
deserializeTxPlusMeta(Backend::TransactionAndMetadata const& blobs, std::uint32_t seq);
std::pair<boost::json::object, boost::json::object>
toExpandedJson(Backend::TransactionAndMetadata const& blobs);
@@ -145,10 +139,7 @@ ngTraverseOwnedNodes(
std::function<void(ripple::SLE&&)> atOwnedNode);
std::shared_ptr<ripple::SLE const>
read(
ripple::Keylet const& keylet,
ripple::LedgerInfo const& lgrInfo,
Context const& context);
read(ripple::Keylet const& keylet, ripple::LedgerInfo const& lgrInfo, Context const& context);
std::variant<Status, std::pair<ripple::PublicKey, ripple::SecretKey>>
keypairFromRequst(boost::json::object const& request);
@@ -217,11 +208,7 @@ postProcessOrderBook(
boost::asio::yield_context& yield);
std::variant<Status, ripple::Book>
parseBook(
ripple::Currency pays,
ripple::AccountID payIssuer,
ripple::Currency gets,
ripple::AccountID getIssuer);
parseBook(ripple::Currency pays, ripple::AccountID payIssuer, ripple::Currency gets, ripple::AccountID getIssuer);
std::variant<Status, ripple::Book>
parseBook(boost::json::object const& request);
@@ -233,10 +220,7 @@ std::optional<std::uint32_t>
getUInt(boost::json::object const& request, std::string const& field);
std::uint32_t
getUInt(
boost::json::object const& request,
std::string const& field,
std::uint32_t dfault);
getUInt(boost::json::object const& request, std::string const& field, std::uint32_t dfault);
std::uint32_t
getRequiredUInt(boost::json::object const& request, std::string const& field);
@@ -245,10 +229,7 @@ std::optional<bool>
getBool(boost::json::object const& request, std::string const& field);
bool
getBool(
boost::json::object const& request,
std::string const& field,
bool dfault);
getBool(boost::json::object const& request, std::string const& field, bool dfault);
bool
getRequiredBool(boost::json::object const& request, std::string const& field);
@@ -260,10 +241,7 @@ std::string
getRequiredString(boost::json::object const& request, std::string const& field);
std::string
getString(
boost::json::object const& request,
std::string const& field,
std::string dfault);
getString(boost::json::object const& request, std::string const& field, std::string dfault);
Status
getHexMarker(boost::json::object const& request, ripple::uint256& marker);
@@ -272,10 +250,7 @@ Status
getAccount(boost::json::object const& request, ripple::AccountID& accountId);
Status
getAccount(
boost::json::object const& request,
ripple::AccountID& destAccount,
boost::string_view const& field);
getAccount(boost::json::object const& request, ripple::AccountID& destAccount, boost::string_view const& field);
Status
getOptionalAccount(
@@ -308,8 +283,6 @@ traverseTransactions(
boost::asio::yield_context& yield)> transactionFetcher);
[[nodiscard]] boost::json::object const
computeBookChanges(
ripple::LedgerInfo const& lgrInfo,
std::vector<Backend::TransactionAndMetadata> const& transactions);
computeBookChanges(ripple::LedgerInfo const& lgrInfo, std::vector<Backend::TransactionAndMetadata> const& transactions);
} // namespace RPC

31
src/rpc/WorkQueue.h
View File

@@ -50,31 +50,24 @@ public:
{
if (curSize_ >= maxSize_ && !isWhiteListed)
{
log_.warn() << "Queue is full. rejecting job. current size = "
<< curSize_ << " max size = " << maxSize_;
log_.warn() << "Queue is full. rejecting job. current size = " << curSize_ << " max size = " << maxSize_;
return false;
}
++curSize_;
auto start = std::chrono::system_clock::now();
// Each time we enqueue a job, we want to post a symmetrical job that
// will dequeue and run the job at the front of the job queue.
boost::asio::spawn(
ioc_,
[this, f = std::move(f), start](boost::asio::yield_context yield) {
auto run = std::chrono::system_clock::now();
auto wait =
std::chrono::duration_cast<std::chrono::microseconds>(
run - start)
.count();
// increment queued_ here, in the same place we implement
// durationUs_
++queued_;
durationUs_ += wait;
log_.info() << "WorkQueue wait time = " << wait
<< " queue size = " << curSize_;
f(yield);
--curSize_;
});
boost::asio::spawn(ioc_, [this, f = std::move(f), start](boost::asio::yield_context yield) {
auto run = std::chrono::system_clock::now();
auto wait = std::chrono::duration_cast<std::chrono::microseconds>(run - start).count();
// increment queued_ here, in the same place we implement
// durationUs_
++queued_;
durationUs_ += wait;
log_.info() << "WorkQueue wait time = " << wait << " queue size = " << curSize_;
f(yield);
--curSize_;
});
return true;
}

10
src/rpc/common/AnyHandler.h
View File

@@ -44,12 +44,9 @@ public:
* @param handler The handler to wrap. Required to fulfil the @ref Handler
* concept.
*/
template <
Handler HandlerType,
typename ProcessingStrategy = detail::DefaultProcessor<HandlerType>>
template <Handler HandlerType, typename ProcessingStrategy = detail::DefaultProcessor<HandlerType>>
/* implicit */ AnyHandler(HandlerType&& handler)
: pimpl_{std::make_unique<Model<HandlerType, ProcessingStrategy>>(
std::forward<HandlerType>(handler))}
: pimpl_{std::make_unique<Model<HandlerType, ProcessingStrategy>>(std::forward<HandlerType>(handler))}
{
}
@@ -125,8 +122,7 @@ private:
}
[[nodiscard]] ReturnType
process(boost::json::value const& value, Context const& ctx)
const override
process(boost::json::value const& value, Context const& ctx) const override
{
return processor(handler, value, &ctx);
}

4
src/rpc/common/Specs.h
View File

@@ -43,9 +43,7 @@ struct FieldSpec final
*/
template <Requirement... Requirements>
FieldSpec(std::string const& key, Requirements&&... requirements)
: validator_{detail::makeFieldValidator<Requirements...>(
key,
std::forward<Requirements>(requirements)...)}
: validator_{detail::makeFieldValidator<Requirements...>(key, std::forward<Requirements>(requirements)...)}
{
}

5
src/rpc/common/Types.h
View File

@@ -68,10 +68,7 @@ struct Context
};
inline void
tag_invoke(
boost::json::value_from_tag,
boost::json::value& jv,
VoidOutput const&)
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, VoidOutput const&)
{
jv = boost::json::object{};
}

93
src/rpc/common/Validators.cpp
View File

@@ -52,16 +52,14 @@ Section::verify(boost::json::value const& value, std::string_view key) const
Required::verify(boost::json::value const& value, std::string_view key) const
{
if (not value.is_object() or not value.as_object().contains(key.data()))
return Error{RPC::Status{
RPC::RippledError::rpcINVALID_PARAMS,
"Required field '" + std::string{key} + "' missing"}};
return Error{
RPC::Status{RPC::RippledError::rpcINVALID_PARAMS, "Required field '" + std::string{key} + "' missing"}};
return {};
}
[[nodiscard]] MaybeError
ValidateArrayAt::verify(boost::json::value const& value, std::string_view key)
const
ValidateArrayAt::verify(boost::json::value const& value, std::string_view key) const
{
if (not value.is_object() or not value.as_object().contains(key.data()))
return {}; // ignore. field does not exist, let 'required' fail
@@ -83,8 +81,7 @@ ValidateArrayAt::verify(boost::json::value const& value, std::string_view key)
}
[[nodiscard]] MaybeError
CustomValidator::verify(boost::json::value const& value, std::string_view key)
const
CustomValidator::verify(boost::json::value const& value, std::string_view key) const
{
if (not value.is_object() or not value.as_object().contains(key.data()))
return {}; // ignore. field does not exist, let 'required' fail
@@ -101,118 +98,96 @@ checkIsU32Numeric(std::string_view sv)
return ec == std::errc();
}
CustomValidator Uint256HexStringValidator = CustomValidator{
[](boost::json::value const& value, std::string_view key) -> MaybeError {
CustomValidator Uint256HexStringValidator =
CustomValidator{[](boost::json::value const& value, std::string_view key) -> MaybeError {
if (!value.is_string())
{
return Error{RPC::Status{
RPC::RippledError::rpcINVALID_PARAMS,
std::string(key) + "NotString"}};
return Error{RPC::Status{RPC::RippledError::rpcINVALID_PARAMS, std::string(key) + "NotString"}};
}
ripple::uint256 ledgerHash;
if (!ledgerHash.parseHex(value.as_string().c_str()))
return Error{RPC::Status{
RPC::RippledError::rpcINVALID_PARAMS,
std::string(key) + "Malformed"}};
return Error{RPC::Status{RPC::RippledError::rpcINVALID_PARAMS, std::string(key) + "Malformed"}};
return MaybeError{};
}};
CustomValidator LedgerIndexValidator = CustomValidator{
[](boost::json::value const& value, std::string_view key) -> MaybeError {
auto err = Error{RPC::Status{
RPC::RippledError::rpcINVALID_PARAMS, "ledgerIndexMalformed"}};
CustomValidator LedgerIndexValidator =
CustomValidator{[](boost::json::value const& value, std::string_view key) -> MaybeError {
auto err = Error{RPC::Status{RPC::RippledError::rpcINVALID_PARAMS, "ledgerIndexMalformed"}};
if (!value.is_string() && !(value.is_uint64() || value.is_int64()))
{
return err;
}
if (value.is_string() && value.as_string() != "validated" &&
!checkIsU32Numeric(value.as_string().c_str()))
if (value.is_string() && value.as_string() != "validated" && !checkIsU32Numeric(value.as_string().c_str()))
{
return err;
}
return MaybeError{};
}};
CustomValidator AccountValidator = CustomValidator{
[](boost::json::value const& value, std::string_view key) -> MaybeError {
CustomValidator AccountValidator =
CustomValidator{[](boost::json::value const& value, std::string_view key) -> MaybeError {
if (!value.is_string())
{
return Error{RPC::Status{
RPC::RippledError::rpcINVALID_PARAMS,
std::string(key) + "NotString"}};
return Error{RPC::Status{RPC::RippledError::rpcINVALID_PARAMS, std::string(key) + "NotString"}};
}
// TODO: we are using accountFromStringStrict from RPCHelpers, after we
// remove all old handler, this function can be moved to here
if (!RPC::accountFromStringStrict(value.as_string().c_str()))
{
return Error{RPC::Status{
RPC::RippledError::rpcINVALID_PARAMS,
std::string(key) + "Malformed"}};
return Error{RPC::Status{RPC::RippledError::rpcINVALID_PARAMS, std::string(key) + "Malformed"}};
}
return MaybeError{};
}};
CustomValidator AccountBase58Validator = CustomValidator{
[](boost::json::value const& value, std::string_view key) -> MaybeError {
CustomValidator AccountBase58Validator =
CustomValidator{[](boost::json::value const& value, std::string_view key) -> MaybeError {
if (!value.is_string())
{
return Error{RPC::Status{
RPC::RippledError::rpcINVALID_PARAMS,
std::string(key) + "NotString"}};
return Error{RPC::Status{RPC::RippledError::rpcINVALID_PARAMS, std::string(key) + "NotString"}};
}
auto const account =
ripple::parseBase58<ripple::AccountID>(value.as_string().c_str());
auto const account = ripple::parseBase58<ripple::AccountID>(value.as_string().c_str());
if (!account || account->isZero())
return Error{RPC::Status{RPC::ClioError::rpcMALFORMED_ADDRESS}};
return MaybeError{};
}};
CustomValidator AccountMarkerValidator = CustomValidator{
[](boost::json::value const& value, std::string_view key) -> MaybeError {
CustomValidator AccountMarkerValidator =
CustomValidator{[](boost::json::value const& value, std::string_view key) -> MaybeError {
if (!value.is_string())
{
return Error{RPC::Status{
RPC::RippledError::rpcINVALID_PARAMS,
std::string(key) + "NotString"}};
return Error{RPC::Status{RPC::RippledError::rpcINVALID_PARAMS, std::string(key) + "NotString"}};
}
// TODO: we are using parseAccountCursor from RPCHelpers, after we
// remove all old handler, this function can be moved to here
if (!RPC::parseAccountCursor(value.as_string().c_str()))
{
// align with the current error message
return Error{RPC::Status{
RPC::RippledError::rpcINVALID_PARAMS, "Malformed cursor"}};
return Error{RPC::Status{RPC::RippledError::rpcINVALID_PARAMS, "Malformed cursor"}};
}
return MaybeError{};
}};
CustomValidator CurrencyValidator = CustomValidator{
[](boost::json::value const& value, std::string_view key) -> MaybeError {
CustomValidator CurrencyValidator =
CustomValidator{[](boost::json::value const& value, std::string_view key) -> MaybeError {
if (!value.is_string())
{
return Error{RPC::Status{
RPC::RippledError::rpcINVALID_PARAMS,
std::string(key) + "NotString"}};
return Error{RPC::Status{RPC::RippledError::rpcINVALID_PARAMS, std::string(key) + "NotString"}};
}
ripple::Currency currency;
if (!ripple::to_currency(currency, value.as_string().c_str()))
return Error{RPC::Status{
RPC::ClioError::rpcMALFORMED_CURRENCY, "malformedCurrency"}};
return Error{RPC::Status{RPC::ClioError::rpcMALFORMED_CURRENCY, "malformedCurrency"}};
return MaybeError{};
}};
CustomValidator IssuerValidator = CustomValidator{
[](boost::json::value const& value, std::string_view key) -> MaybeError {
CustomValidator IssuerValidator =
CustomValidator{[](boost::json::value const& value, std::string_view key) -> MaybeError {
if (!value.is_string())
return Error{RPC::Status{
RPC::RippledError::rpcINVALID_PARAMS,
std::string(key) + "NotString"}};
return Error{RPC::Status{RPC::RippledError::rpcINVALID_PARAMS, std::string(key) + "NotString"}};
ripple::AccountID issuer;
if (!ripple::to_issuer(issuer, value.as_string().c_str()))
return Error{RPC::Status{
// TODO: need to align with the error
RPC::RippledError::rpcINVALID_PARAMS,
fmt::format("Invalid field '{}', bad issuer.", key)}};
return Error{RPC::Status{// TODO: need to align with the error
RPC::RippledError::rpcINVALID_PARAMS,
fmt::format("Invalid field '{}', bad issuer.", key)}};
if (issuer == ripple::noAccount())
return Error{RPC::Status{

27
src/rpc/common/Validators.h
View File

@@ -46,8 +46,7 @@ template <typename Expected>
if (not value.is_string())
hasError = true;
}
else if constexpr (
std::is_same_v<Expected, double> or std::is_same_v<Expected, float>)
else if constexpr (std::is_same_v<Expected, double> or std::is_same_v<Expected, float>)
{
if (not value.is_double())
hasError = true;
@@ -62,9 +61,7 @@ template <typename Expected>
if (not value.is_object())
hasError = true;
}
else if constexpr (
std::is_convertible_v<Expected, uint64_t> or
std::is_convertible_v<Expected, int64_t>)
else if constexpr (std::is_convertible_v<Expected, uint64_t> or std::is_convertible_v<Expected, int64_t>)
{
if (not value.is_int64() && not value.is_uint64())
hasError = true;
@@ -266,8 +263,7 @@ public:
using boost::json::value_to;
auto const res = value_to<Type>(value.as_object().at(key.data()));
if (std::find(std::begin(options_), std::end(options_), res) ==
std::end(options_))
if (std::find(std::begin(options_), std::end(options_), res) == std::end(options_))
return Error{RPC::Status{RPC::RippledError::rpcINVALID_PARAMS}};
return {};
@@ -297,8 +293,7 @@ public:
* @param idx The index inside the array to validate
* @param specs The specifications to validate against
*/
ValidateArrayAt(std::size_t idx, std::initializer_list<FieldSpec> specs)
: idx_{idx}, specs_{specs}
ValidateArrayAt(std::size_t idx, std::initializer_list<FieldSpec> specs) : idx_{idx}, specs_{specs}
{
}
@@ -330,12 +325,11 @@ public:
IfType(Requirements&&... requirements)
{
validator_ = [... r = std::forward<Requirements>(requirements)](
boost::json::value const& j,
std::string_view key) -> MaybeError {
// clang-format off
boost::json::value const& j, std::string_view key) -> MaybeError {
std::optional<RPC::Status> firstFailure = std::nullopt;
// the check logic is the same as fieldspec
// clang-format off
([&j, &key, &firstFailure, req = &r]() {
if (firstFailure)
return;
@@ -373,8 +367,7 @@ public:
}
private:
std::function<MaybeError(boost::json::value const&, std::string_view)>
validator_;
std::function<MaybeError(boost::json::value const&, std::string_view)> validator_;
};
/**
@@ -392,8 +385,7 @@ public:
* @brief Constructs a validator that calls the given validator "req" and
* return customized error "err"
*/
WithCustomError(Requirement req, RPC::Status err)
: requirement{std::move(req)}, error{err}
WithCustomError(Requirement req, RPC::Status err) : requirement{std::move(req)}, error{err}
{
}
@@ -412,8 +404,7 @@ public:
*/
class CustomValidator final
{
std::function<MaybeError(boost::json::value const&, std::string_view)>
validator_;
std::function<MaybeError(boost::json::value const&, std::string_view)> validator_;
public:
/**

9
src/rpc/common/impl/Factories.h
View File

@@ -32,14 +32,13 @@ template <Requirement... Requirements>
[[nodiscard]] auto
makeFieldValidator(std::string const& key, Requirements&&... requirements)
{
return [key, ... r = std::forward<Requirements>(requirements)](
boost::json::value const& j) -> MaybeError {
// clang-format off
return [key, ... r = std::forward<Requirements>(requirements)](boost::json::value const& j) -> MaybeError {
std::optional<RPC::Status> firstFailure = std::nullopt;
// This expands in order of Requirements and stops evaluating after
// first failure which is stored in `firstFailure` and can be checked
// This expands in order of Requirements and stops evaluating after
// first failure which is stored in `firstFailure` and can be checked
// later on to see whether the verification failed as a whole or not.
// clang-format off
([&j, &key, &firstFailure, req = &r]() {
if (firstFailure)
return; // already failed earlier - skip

5
src/rpc/common/impl/Processors.h
View File

@@ -31,10 +31,7 @@ template <Handler HandlerType>
struct DefaultProcessor final
{
[[nodiscard]] ReturnType
operator()(
HandlerType const& handler,
boost::json::value const& value,
Context const* ctx = nullptr) const
operator()(HandlerType const& handler, boost::json::value const& value, Context const* ctx = nullptr) const
{
using boost::json::value_from;
using boost::json::value_to;

27
src/rpc/handlers/AccountChannels.cpp
View File

@@ -38,8 +38,7 @@ addChannel(boost::json::array& jsonLines, ripple::SLE const& line)
boost::json::object jDst;
jDst[JS(channel_id)] = ripple::to_string(line.key());
jDst[JS(account)] = ripple::to_string(line.getAccountID(ripple::sfAccount));
jDst[JS(destination_account)] =
ripple::to_string(line.getAccountID(ripple::sfDestination));
jDst[JS(destination_account)] = ripple::to_string(line.getAccountID(ripple::sfDestination));
jDst[JS(amount)] = line[ripple::sfAmount].getText();
jDst[JS(balance)] = line[ripple::sfBalance].getText();
if (publicKeyType(line[ripple::sfPublicKey]))
@@ -77,16 +76,14 @@ doAccountChannels(Context const& context)
if (auto const status = getAccount(request, accountID); status)
return status;
auto rawAcct = context.backend->fetchLedgerObject(
ripple::keylet::account(accountID).key, lgrInfo.seq, context.yield);
auto rawAcct =
context.backend->fetchLedgerObject(ripple::keylet::account(accountID).key, lgrInfo.seq, context.yield);
if (!rawAcct)
return Status{RippledError::rpcACT_NOT_FOUND, "accountNotFound"};
ripple::AccountID destAccount;
if (auto const status =
getAccount(request, destAccount, JS(destination_account));
status)
if (auto const status = getAccount(request, destAccount, JS(destination_account)); status)
return status;
std::uint32_t limit;
@@ -108,10 +105,8 @@ doAccountChannels(Context const& context)
boost::json::array& jsonChannels = response.at(JS(channels)).as_array();
auto const addToResponse = [&](ripple::SLE&& sle) {
if (sle.getType() == ripple::ltPAYCHAN &&
sle.getAccountID(ripple::sfAccount) == accountID &&
(!destAccount ||
destAccount == sle.getAccountID(ripple::sfDestination)))
if (sle.getType() == ripple::ltPAYCHAN && sle.getAccountID(ripple::sfAccount) == accountID &&
(!destAccount || destAccount == sle.getAccountID(ripple::sfDestination)))
{
addChannel(jsonChannels, sle);
}
@@ -119,14 +114,8 @@ doAccountChannels(Context const& context)
return true;
};
auto next = traverseOwnedNodes(
*context.backend,
accountID,
lgrInfo.seq,
limit,
marker,
context.yield,
addToResponse);
auto next =
traverseOwnedNodes(*context.backend, accountID, lgrInfo.seq, limit, marker, context.yield, addToResponse);
response[JS(ledger_hash)] = ripple::strHex(lgrInfo.hash);
response[JS(ledger_index)] = lgrInfo.seq;

10
src/rpc/handlers/AccountCurrencies.cpp
View File

@@ -47,8 +47,8 @@ doAccountCurrencies(Context const& context)
if (auto const status = getAccount(request, accountID); status)
return status;
auto rawAcct = context.backend->fetchLedgerObject(
ripple::keylet::account(accountID).key, lgrInfo.seq, context.yield);
auto rawAcct =
context.backend->fetchLedgerObject(ripple::keylet::account(accountID).key, lgrInfo.seq, context.yield);
if (!rawAcct)
return Status{RippledError::rpcACT_NOT_FOUND, "accountNotFound"};
@@ -88,10 +88,8 @@ doAccountCurrencies(Context const& context)
response[JS(ledger_hash)] = ripple::strHex(lgrInfo.hash);
response[JS(ledger_index)] = lgrInfo.seq;
response[JS(receive_currencies)] =
boost::json::value(boost::json::array_kind);
boost::json::array& jsonReceive =
response.at(JS(receive_currencies)).as_array();
response[JS(receive_currencies)] = boost::json::value(boost::json::array_kind);
boost::json::array& jsonReceive = response.at(JS(receive_currencies)).as_array();
for (auto const& currency : receive)
jsonReceive.push_back(currency.c_str());

16
src/rpc/handlers/AccountInfo.cpp
View File

@@ -76,8 +76,7 @@ doAccountInfo(Context const& context)
if (!dbResponse)
return Status{RippledError::rpcACT_NOT_FOUND};
ripple::STLedgerEntry sle{
ripple::SerialIter{dbResponse->data(), dbResponse->size()}, key.key};
ripple::STLedgerEntry sle{ripple::SerialIter{dbResponse->data(), dbResponse->size()}, key.key};
if (!key.check(sle))
return Status{RippledError::rpcDB_DESERIALIZATION};
@@ -87,8 +86,7 @@ doAccountInfo(Context const& context)
response[JS(ledger_index)] = lgrInfo.seq;
// Return SignerList(s) if that is requested.
if (request.contains(JS(signer_lists)) &&
request.at(JS(signer_lists)).as_bool())
if (request.contains(JS(signer_lists)) && request.at(JS(signer_lists)).as_bool())
{
// We put the SignerList in an array because of an anticipated
// future when we support multiple signer lists on one account.
@@ -97,21 +95,17 @@ doAccountInfo(Context const& context)
// This code will need to be revisited if in the future we
// support multiple SignerLists on one account.
auto const signers = context.backend->fetchLedgerObject(
signersKey.key, lgrInfo.seq, context.yield);
auto const signers = context.backend->fetchLedgerObject(signersKey.key, lgrInfo.seq, context.yield);
if (signers)
{
ripple::STLedgerEntry sleSigners{
ripple::SerialIter{signers->data(), signers->size()},
signersKey.key};
ripple::STLedgerEntry sleSigners{ripple::SerialIter{signers->data(), signers->size()}, signersKey.key};
if (!signersKey.check(sleSigners))
return Status{RippledError::rpcDB_DESERIALIZATION};
signerList.push_back(toJson(sleSigners));
}
response[JS(account_data)].as_object()[JS(signer_lists)] =
std::move(signerList);
response[JS(account_data)].as_object()[JS(signer_lists)] = std::move(signerList);
}
return response;

49
src/rpc/handlers/AccountLines.cpp
View File

@@ -64,18 +64,12 @@ addLine(
if (!viewLowest)
balance.negate();
bool lineAuth =
flags & (viewLowest ? ripple::lsfLowAuth : ripple::lsfHighAuth);
bool lineAuthPeer =
flags & (!viewLowest ? ripple::lsfLowAuth : ripple::lsfHighAuth);
bool lineNoRipple =
flags & (viewLowest ? ripple::lsfLowNoRipple : ripple::lsfHighNoRipple);
bool lineNoRipplePeer = flags &
(!viewLowest ? ripple::lsfLowNoRipple : ripple::lsfHighNoRipple);
bool lineFreeze =
flags & (viewLowest ? ripple::lsfLowFreeze : ripple::lsfHighFreeze);
bool lineFreezePeer =
flags & (!viewLowest ? ripple::lsfLowFreeze : ripple::lsfHighFreeze);
bool lineAuth = flags & (viewLowest ? ripple::lsfLowAuth : ripple::lsfHighAuth);
bool lineAuthPeer = flags & (!viewLowest ? ripple::lsfLowAuth : ripple::lsfHighAuth);
bool lineNoRipple = flags & (viewLowest ? ripple::lsfLowNoRipple : ripple::lsfHighNoRipple);
bool lineNoRipplePeer = flags & (!viewLowest ? ripple::lsfLowNoRipple : ripple::lsfHighNoRipple);
bool lineFreeze = flags & (viewLowest ? ripple::lsfLowFreeze : ripple::lsfHighFreeze);
bool lineFreezePeer = flags & (!viewLowest ? ripple::lsfLowFreeze : ripple::lsfHighFreeze);
ripple::STAmount const& saBalance(balance);
ripple::STAmount const& saLimit(lineLimit);
@@ -119,15 +113,14 @@ doAccountLines(Context const& context)
if (auto const status = getAccount(request, accountID); status)
return status;
auto rawAcct = context.backend->fetchLedgerObject(
ripple::keylet::account(accountID).key, lgrInfo.seq, context.yield);
auto rawAcct =
context.backend->fetchLedgerObject(ripple::keylet::account(accountID).key, lgrInfo.seq, context.yield);
if (!rawAcct)
return Status{RippledError::rpcACT_NOT_FOUND, "accountNotFound"};
std::optional<ripple::AccountID> peerAccount;
if (auto const status = getOptionalAccount(request, peerAccount, JS(peer));
status)
if (auto const status = getOptionalAccount(request, peerAccount, JS(peer)); status)
return status;
std::uint32_t limit;
@@ -147,8 +140,7 @@ doAccountLines(Context const& context)
if (request.contains(JS(ignore_default)))
{
if (not request.at(JS(ignore_default)).is_bool())
return Status{
RippledError::rpcINVALID_PARAMS, "ignoreDefaultNotBool"};
return Status{RippledError::rpcINVALID_PARAMS, "ignoreDefaultNotBool"};
ignoreDefault = request.at(JS(ignore_default)).as_bool();
}
@@ -166,15 +158,10 @@ doAccountLines(Context const& context)
auto ignore = false;
if (ignoreDefault)
{
if (sle.getFieldAmount(ripple::sfLowLimit).getIssuer() ==
accountID)
ignore =
!(sle.getFieldU32(ripple::sfFlags) &
ripple::lsfLowReserve);
if (sle.getFieldAmount(ripple::sfLowLimit).getIssuer() == accountID)
ignore = !(sle.getFieldU32(ripple::sfFlags) & ripple::lsfLowReserve);
else
ignore =
!(sle.getFieldU32(ripple::sfFlags) &
ripple::lsfHighReserve);
ignore = !(sle.getFieldU32(ripple::sfFlags) & ripple::lsfHighReserve);
}
if (!ignore)
@@ -182,14 +169,8 @@ doAccountLines(Context const& context)
}
};
auto next = traverseOwnedNodes(
*context.backend,
accountID,
lgrInfo.seq,
limit,
marker,
context.yield,
addToResponse);
auto next =
traverseOwnedNodes(*context.backend, accountID, lgrInfo.seq, limit, marker, context.yield, addToResponse);
if (auto status = std::get_if<RPC::Status>(&next))
return *status;

45
src/rpc/handlers/AccountObjects.cpp
View File

@@ -66,8 +66,8 @@ doAccountNFTs(Context const& context)
if (!accountID)
return Status{RippledError::rpcINVALID_PARAMS, "malformedAccount"};
auto rawAcct = context.backend->fetchLedgerObject(
ripple::keylet::account(accountID).key, lgrInfo.seq, context.yield);
auto rawAcct =
context.backend->fetchLedgerObject(ripple::keylet::account(accountID).key, lgrInfo.seq, context.yield);
if (!rawAcct)
return Status{RippledError::rpcACT_NOT_FOUND, "accountNotFound"};
@@ -90,15 +90,12 @@ doAccountNFTs(Context const& context)
// if a marker was passed, start at the page specified in marker. Else,
// start at the max page
auto const pageKey =
marker.isZero() ? ripple::keylet::nftpage_max(accountID).key : marker;
auto const pageKey = marker.isZero() ? ripple::keylet::nftpage_max(accountID).key : marker;
auto const blob =
context.backend->fetchLedgerObject(pageKey, lgrInfo.seq, context.yield);
auto const blob = context.backend->fetchLedgerObject(pageKey, lgrInfo.seq, context.yield);
if (!blob)
return response;
std::optional<ripple::SLE const> page{
ripple::SLE{ripple::SerialIter{blob->data(), blob->size()}, pageKey}};
std::optional<ripple::SLE const> page{ripple::SLE{ripple::SerialIter{blob->data(), blob->size()}, pageKey}};
// Continue iteration from the current page
while (page)
@@ -110,20 +107,16 @@ doAccountNFTs(Context const& context)
ripple::uint256 const nftokenID = o[ripple::sfNFTokenID];
{
nfts.push_back(
toBoostJson(o.getJson(ripple::JsonOptions::none)));
nfts.push_back(toBoostJson(o.getJson(ripple::JsonOptions::none)));
auto& obj = nfts.back().as_object();
// Pull out the components of the nft ID.
obj[SFS(sfFlags)] = ripple::nft::getFlags(nftokenID);
obj[SFS(sfIssuer)] =
to_string(ripple::nft::getIssuer(nftokenID));
obj[SFS(sfNFTokenTaxon)] =
ripple::nft::toUInt32(ripple::nft::getTaxon(nftokenID));
obj[SFS(sfIssuer)] = to_string(ripple::nft::getIssuer(nftokenID));
obj[SFS(sfNFTokenTaxon)] = ripple::nft::toUInt32(ripple::nft::getTaxon(nftokenID));
obj[JS(nft_serial)] = ripple::nft::getSerial(nftokenID);
if (std::uint16_t xferFee = {
ripple::nft::getTransferFee(nftokenID)})
if (std::uint16_t xferFee = {ripple::nft::getTransferFee(nftokenID)})
obj[SFS(sfTransferFee)] = xferFee;
}
}
@@ -138,12 +131,9 @@ doAccountNFTs(Context const& context)
response[JS(limit)] = numPages;
return response;
}
auto const nextBlob = context.backend->fetchLedgerObject(
nextKey.key, lgrInfo.seq, context.yield);
auto const nextBlob = context.backend->fetchLedgerObject(nextKey.key, lgrInfo.seq, context.yield);
page.emplace(ripple::SLE{
ripple::SerialIter{nextBlob->data(), nextBlob->size()},
nextKey.key});
page.emplace(ripple::SLE{ripple::SerialIter{nextBlob->data(), nextBlob->size()}, nextKey.key});
}
else
page.reset();
@@ -196,8 +186,7 @@ doAccountObjects(Context const& context)
response[JS(account)] = ripple::to_string(accountID);
response[JS(account_objects)] = boost::json::value(boost::json::array_kind);
boost::json::array& jsonObjects =
response.at(JS(account_objects)).as_array();
boost::json::array& jsonObjects = response.at(JS(account_objects)).as_array();
auto const addToResponse = [&](ripple::SLE&& sle) {
if (!objectType || objectType == sle.getType())
@@ -206,14 +195,8 @@ doAccountObjects(Context const& context)
}
};
auto next = traverseOwnedNodes(
*context.backend,
accountID,
lgrInfo.seq,
limit,
marker,
context.yield,
addToResponse);
auto next =
traverseOwnedNodes(*context.backend, accountID, lgrInfo.seq, limit, marker, context.yield, addToResponse);
response[JS(ledger_hash)] = ripple::strHex(lgrInfo.hash);
response[JS(ledger_index)] = lgrInfo.seq;

20
src/rpc/handlers/AccountOffers.cpp
View File

@@ -50,8 +50,7 @@ addOffer(boost::json::array& offersJson, ripple::SLE const& offer)
boost::json::object& takerPaysJson = obj.at(JS(taker_pays)).as_object();
takerPaysJson[JS(value)] = takerPays.getText();
takerPaysJson[JS(currency)] =
ripple::to_string(takerPays.getCurrency());
takerPaysJson[JS(currency)] = ripple::to_string(takerPays.getCurrency());
takerPaysJson[JS(issuer)] = ripple::to_string(takerPays.getIssuer());
}
else
@@ -65,8 +64,7 @@ addOffer(boost::json::array& offersJson, ripple::SLE const& offer)
boost::json::object& takerGetsJson = obj.at(JS(taker_gets)).as_object();
takerGetsJson[JS(value)] = takerGets.getText();
takerGetsJson[JS(currency)] =
ripple::to_string(takerGets.getCurrency());
takerGetsJson[JS(currency)] = ripple::to_string(takerGets.getCurrency());
takerGetsJson[JS(issuer)] = ripple::to_string(takerGets.getIssuer());
}
else
@@ -99,8 +97,8 @@ doAccountOffers(Context const& context)
if (auto const status = getAccount(request, accountID); status)
return status;
auto rawAcct = context.backend->fetchLedgerObject(
ripple::keylet::account(accountID).key, lgrInfo.seq, context.yield);
auto rawAcct =
context.backend->fetchLedgerObject(ripple::keylet::account(accountID).key, lgrInfo.seq, context.yield);
if (!rawAcct)
return Status{RippledError::rpcACT_NOT_FOUND, "accountNotFound"};
@@ -134,14 +132,8 @@ doAccountOffers(Context const& context)
return true;
};
auto next = traverseOwnedNodes(
*context.backend,
accountID,
lgrInfo.seq,
limit,
marker,
context.yield,
addToResponse);
auto next =
traverseOwnedNodes(*context.backend, accountID, lgrInfo.seq, limit, marker, context.yield, addToResponse);
if (auto status = std::get_if<RPC::Status>(&next))
return *status;

9
src/rpc/handlers/AccountTx.cpp
View File

@@ -46,13 +46,10 @@ doAccountTx(Context const& context)
bool const forward,
std::optional<Backend::TransactionsCursor> const& cursorIn,
boost::asio::yield_context& yield) {
auto [txnsAndCursor, timeDiff] = util::timed([&]() {
return backend->fetchAccountTransactions(
accountID, limit, forward, cursorIn, yield);
});
auto [txnsAndCursor, timeDiff] = util::timed(
[&]() { return backend->fetchAccountTransactions(accountID, limit, forward, cursorIn, yield); });
gLog.info() << outerFuncName << " db fetch took " << timeDiff
<< " milliseconds - num blobs = "
<< txnsAndCursor.txns.size();
<< " milliseconds - num blobs = " << txnsAndCursor.txns.size();
return txnsAndCursor;
});

44
src/rpc/handlers/BookChanges.cpp
View File

@@ -72,8 +72,7 @@ private:
public:
[[nodiscard]] std::vector<BookChange>
operator()(
std::vector<Backend::TransactionAndMetadata> const& transactions)
operator()(std::vector<Backend::TransactionAndMetadata> const& transactions)
{
for (auto const& tx : transactions)
handleBookChange(tx);
@@ -103,47 +102,36 @@ private:
// if either FF or PF are missing we can't compute
// but generally these are cancelled rather than crossed
// so skipping them is consistent
if (!node.isFieldPresent(sfFinalFields) ||
!node.isFieldPresent(sfPreviousFields))
if (!node.isFieldPresent(sfFinalFields) || !node.isFieldPresent(sfPreviousFields))
return;
auto const& finalFields =
node.peekAtField(sfFinalFields).downcast<STObject>();
auto const& previousFields =
node.peekAtField(sfPreviousFields).downcast<STObject>();
auto const& finalFields = node.peekAtField(sfFinalFields).downcast<STObject>();
auto const& previousFields = node.peekAtField(sfPreviousFields).downcast<STObject>();
// defensive case that should never be hit
if (!finalFields.isFieldPresent(sfTakerGets) ||
!finalFields.isFieldPresent(sfTakerPays) ||
!previousFields.isFieldPresent(sfTakerGets) ||
!previousFields.isFieldPresent(sfTakerPays))
if (!finalFields.isFieldPresent(sfTakerGets) || !finalFields.isFieldPresent(sfTakerPays) ||
!previousFields.isFieldPresent(sfTakerGets) || !previousFields.isFieldPresent(sfTakerPays))
return;
// filter out any offers deleted by explicit offer cancels
if (metaType == sfDeletedNode && offerCancel_ &&
finalFields.getFieldU32(sfSequence) == *offerCancel_)
if (metaType == sfDeletedNode && offerCancel_ && finalFields.getFieldU32(sfSequence) == *offerCancel_)
return;
// compute the difference in gets and pays actually
// affected onto the offer
auto const deltaGets = finalFields.getFieldAmount(sfTakerGets) -
previousFields.getFieldAmount(sfTakerGets);
auto const deltaPays = finalFields.getFieldAmount(sfTakerPays) -
previousFields.getFieldAmount(sfTakerPays);
auto const deltaGets = finalFields.getFieldAmount(sfTakerGets) - previousFields.getFieldAmount(sfTakerGets);
auto const deltaPays = finalFields.getFieldAmount(sfTakerPays) - previousFields.getFieldAmount(sfTakerPays);
transformAndStore(deltaGets, deltaPays);
}
void
transformAndStore(
ripple::STAmount const& deltaGets,
ripple::STAmount const& deltaPays)
transformAndStore(ripple::STAmount const& deltaGets, ripple::STAmount const& deltaPays)
{
auto const g = to_string(deltaGets.issue());
auto const p = to_string(deltaPays.issue());
auto const noswap =
isXRP(deltaGets) ? true : (isXRP(deltaPays) ? false : (g < p));
auto const noswap = isXRP(deltaGets) ? true : (isXRP(deltaPays) ? false : (g < p));
auto first = noswap ? deltaGets : deltaPays;
auto second = noswap ? deltaPays : deltaGets;
@@ -224,8 +212,7 @@ void
tag_invoke(json::value_from_tag, json::value& jv, BookChange const& change)
{
auto amountStr = [](STAmount const& amount) -> std::string {
return isXRP(amount) ? to_string(amount.xrp())
: to_string(amount.iou());
return isXRP(amount) ? to_string(amount.xrp()) : to_string(amount.iou());
};
auto currencyStr = [](STAmount const& amount) -> std::string {
@@ -245,9 +232,7 @@ tag_invoke(json::value_from_tag, json::value& jv, BookChange const& change)
}
json::object const
computeBookChanges(
ripple::LedgerInfo const& lgrInfo,
std::vector<Backend::TransactionAndMetadata> const& transactions)
computeBookChanges(ripple::LedgerInfo const& lgrInfo, std::vector<Backend::TransactionAndMetadata> const& transactions)
{
return {
{JS(type), "bookChanges"},
@@ -267,8 +252,7 @@ doBookChanges(Context const& context)
return *status;
auto const lgrInfo = std::get<ripple::LedgerInfo>(info);
auto const transactions = context.backend->fetchAllTransactionsInLedger(
lgrInfo.seq, context.yield);
auto const transactions = context.backend->fetchAllTransactionsInLedger(lgrInfo.seq, context.yield);
return computeBookChanges(lgrInfo, transactions);
}

15
src/rpc/handlers/BookOffers.cpp
View File

@@ -84,28 +84,21 @@ doBookOffers(Context const& context)
return status;
auto start = std::chrono::system_clock::now();
auto [offers, _] = context.backend->fetchBookOffers(
bookBase, lgrInfo.seq, limit, context.yield);
auto [offers, _] = context.backend->fetchBookOffers(bookBase, lgrInfo.seq, limit, context.yield);
auto end = std::chrono::system_clock::now();
gLog.warn() << "Time loading books: "
<< std::chrono::duration_cast<std::chrono::milliseconds>(
end - start)
.count()
gLog.warn() << "Time loading books: " << std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count()
<< " milliseconds - request = " << request;
response[JS(ledger_hash)] = ripple::strHex(lgrInfo.hash);
response[JS(ledger_index)] = lgrInfo.seq;
response[JS(offers)] = postProcessOrderBook(
offers, book, takerID, *context.backend, lgrInfo.seq, context.yield);
response[JS(offers)] = postProcessOrderBook(offers, book, takerID, *context.backend, lgrInfo.seq, context.yield);
auto end2 = std::chrono::system_clock::now();
gLog.warn() << "Time transforming to json: "
<< std::chrono::duration_cast<std::chrono::milliseconds>(
end2 - end)
.count()
<< std::chrono::duration_cast<std::chrono::milliseconds>(end2 - end).count()
<< " milliseconds - request = " << request;
return response;
}

20
src/rpc/handlers/ChannelAuthorize.cpp
View File

@@ -30,10 +30,7 @@
namespace RPC {
void
serializePayChanAuthorization(
ripple::Serializer& msg,
ripple::uint256 const& key,
ripple::XRPAmount const& amt)
serializePayChanAuthorization(ripple::Serializer& msg, ripple::uint256 const& key, ripple::XRPAmount const& amt)
{
msg.add32(ripple::HashPrefix::paymentChannelClaim);
msg.addBitString(key);
@@ -53,32 +50,27 @@ doChannelAuthorize(Context const& context)
return Status{RippledError::rpcINVALID_PARAMS, "amountNotString"};
if (!request.contains(JS(key_type)) && !request.contains(JS(secret)))
return Status{
RippledError::rpcINVALID_PARAMS, "missingKeyTypeOrSecret"};
return Status{RippledError::rpcINVALID_PARAMS, "missingKeyTypeOrSecret"};
auto v = keypairFromRequst(request);
if (auto status = std::get_if<Status>(&v))
return *status;
auto const [pk, sk] =
std::get<std::pair<ripple::PublicKey, ripple::SecretKey>>(v);
auto const [pk, sk] = std::get<std::pair<ripple::PublicKey, ripple::SecretKey>>(v);
ripple::uint256 channelId;
if (auto const status = getChannelId(request, channelId); status)
return status;
auto optDrops =
ripple::to_uint64(request.at(JS(amount)).as_string().c_str());
auto optDrops = ripple::to_uint64(request.at(JS(amount)).as_string().c_str());
if (!optDrops)
return Status{
RippledError::rpcCHANNEL_AMT_MALFORMED, "couldNotParseAmount"};
return Status{RippledError::rpcCHANNEL_AMT_MALFORMED, "couldNotParseAmount"};
std::uint64_t drops = *optDrops;
ripple::Serializer msg;
ripple::serializePayChanAuthorization(
msg, channelId, ripple::XRPAmount(drops));
ripple::serializePayChanAuthorization(msg, channelId, ripple::XRPAmount(drops));
try
{

27
src/rpc/handlers/ChannelVerify.cpp
View File

@@ -55,23 +55,18 @@ doChannelVerify(Context const& context)
std::optional<ripple::PublicKey> pk;
{
std::string const strPk =
request.at(JS(public_key)).as_string().c_str();
pk = ripple::parseBase58<ripple::PublicKey>(
ripple::TokenType::AccountPublic, strPk);
std::string const strPk = request.at(JS(public_key)).as_string().c_str();
pk = ripple::parseBase58<ripple::PublicKey>(ripple::TokenType::AccountPublic, strPk);
if (!pk)
{
auto pkHex = ripple::strUnHex(strPk);
if (!pkHex)
return Status{
RippledError::rpcPUBLIC_MALFORMED, "malformedPublicKey"};
return Status{RippledError::rpcPUBLIC_MALFORMED, "malformedPublicKey"};
auto const pkType =
ripple::publicKeyType(ripple::makeSlice(*pkHex));
auto const pkType = ripple::publicKeyType(ripple::makeSlice(*pkHex));
if (!pkType)
return Status{
RippledError::rpcPUBLIC_MALFORMED, "invalidKeyType"};
return Status{RippledError::rpcPUBLIC_MALFORMED, "invalidKeyType"};
pk.emplace(ripple::makeSlice(*pkHex));
}
@@ -81,12 +76,10 @@ doChannelVerify(Context const& context)
if (auto const status = getChannelId(request, channelId); status)
return status;
auto optDrops =
ripple::to_uint64(request.at(JS(amount)).as_string().c_str());
auto optDrops = ripple::to_uint64(request.at(JS(amount)).as_string().c_str());
if (!optDrops)
return Status{
RippledError::rpcCHANNEL_AMT_MALFORMED, "couldNotParseAmount"};
return Status{RippledError::rpcCHANNEL_AMT_MALFORMED, "couldNotParseAmount"};
std::uint64_t drops = *optDrops;
@@ -96,11 +89,9 @@ doChannelVerify(Context const& context)
return Status{RippledError::rpcINVALID_PARAMS, "invalidSignature"};
ripple::Serializer msg;
ripple::serializePayChanAuthorization(
msg, channelId, ripple::XRPAmount(drops));
ripple::serializePayChanAuthorization(msg, channelId, ripple::XRPAmount(drops));
response[JS(signature_verified)] =
ripple::verify(*pk, msg.slice(), ripple::makeSlice(*sig), true);
response[JS(signature_verified)] = ripple::verify(*pk, msg.slice(), ripple::makeSlice(*sig), true);
return response;
}

53
src/rpc/handlers/GatewayBalances.cpp
View File

@@ -46,19 +46,17 @@ doGatewayBalances(Context const& context)
if (request.contains(JS(hotwallet)))
{
auto getAccountID =
[](auto const& j) -> std::optional<ripple::AccountID> {
auto getAccountID = [](auto const& j) -> std::optional<ripple::AccountID> {
if (j.is_string())
{
auto const pk = ripple::parseBase58<ripple::PublicKey>(
ripple::TokenType::AccountPublic, j.as_string().c_str());
auto const pk =
ripple::parseBase58<ripple::PublicKey>(ripple::TokenType::AccountPublic, j.as_string().c_str());
if (pk)
{
return ripple::calcAccountID(*pk);
}
return ripple::parseBase58<ripple::AccountID>(
j.as_string().c_str());
return ripple::parseBase58<ripple::AccountID>(j.as_string().c_str());
}
return {};
};
@@ -111,8 +109,7 @@ doGatewayBalances(Context const& context)
auto lineLimit = viewLowest ? lowLimit : highLimit;
auto lineLimitPeer = !viewLowest ? lowLimit : highLimit;
auto flags = sle.getFieldU32(ripple::sfFlags);
auto freeze = flags &
(viewLowest ? ripple::lsfLowFreeze : ripple::lsfHighFreeze);
auto freeze = flags & (viewLowest ? ripple::lsfLowFreeze : ripple::lsfHighFreeze);
if (!viewLowest)
balance.negate();
@@ -188,34 +185,28 @@ doGatewayBalances(Context const& context)
response[JS(obligations)] = std::move(obj);
}
auto toJson =
[](std::map<ripple::AccountID, std::vector<ripple::STAmount>> const&
balances) {
boost::json::object obj;
if (!balances.empty())
auto toJson = [](std::map<ripple::AccountID, std::vector<ripple::STAmount>> const& balances) {
boost::json::object obj;
if (!balances.empty())
{
for (auto const& [accId, accBalances] : balances)
{
for (auto const& [accId, accBalances] : balances)
boost::json::array arr;
for (auto const& balance : accBalances)
{
boost::json::array arr;
for (auto const& balance : accBalances)
{
boost::json::object entry;
entry[JS(currency)] =
ripple::to_string(balance.issue().currency);
entry[JS(value)] = balance.getText();
arr.push_back(std::move(entry));
}
obj[ripple::to_string(accId)] = std::move(arr);
boost::json::object entry;
entry[JS(currency)] = ripple::to_string(balance.issue().currency);
entry[JS(value)] = balance.getText();
arr.push_back(std::move(entry));
}
obj[ripple::to_string(accId)] = std::move(arr);
}
return obj;
};
auto containsHotWallet = [&](auto const& hw) {
return hotBalances.contains(hw);
}
return obj;
};
if (not std::all_of(
hotWallets.begin(), hotWallets.end(), containsHotWallet))
auto containsHotWallet = [&](auto const& hw) { return hotBalances.contains(hw); };
if (not std::all_of(hotWallets.begin(), hotWallets.end(), containsHotWallet))
return Status{RippledError::rpcINVALID_PARAMS, "invalidHotWallet"};
if (auto balances = toJson(hotBalances); balances.size())

19
src/rpc/handlers/Ledger.cpp
View File

@@ -41,8 +41,7 @@ doLedger(Context const& context)
if (params.contains(JS(transactions)))
{
if (!params.at(JS(transactions)).is_bool())
return Status{
RippledError::rpcINVALID_PARAMS, "transactionsFlagNotBool"};
return Status{RippledError::rpcINVALID_PARAMS, "transactionsFlagNotBool"};
transactions = params.at(JS(transactions)).as_bool();
}
@@ -94,8 +93,7 @@ doLedger(Context const& context)
header[JS(hash)] = ripple::strHex(lgrInfo.hash);
header[JS(ledger_hash)] = ripple::strHex(lgrInfo.hash);
header[JS(ledger_index)] = std::to_string(lgrInfo.seq);
header[JS(parent_close_time)] =
lgrInfo.parentCloseTime.time_since_epoch().count();
header[JS(parent_close_time)] = lgrInfo.parentCloseTime.time_since_epoch().count();
header[JS(parent_hash)] = ripple::strHex(lgrInfo.parentHash);
header[JS(seqNum)] = std::to_string(lgrInfo.seq);
header[JS(totalCoins)] = ripple::to_string(lgrInfo.drops);
@@ -110,8 +108,7 @@ doLedger(Context const& context)
boost::json::array& jsonTxs = header.at(JS(transactions)).as_array();
if (expand)
{
auto txns = context.backend->fetchAllTransactionsInLedger(
lgrInfo.seq, context.yield);
auto txns = context.backend->fetchAllTransactionsInLedger(lgrInfo.seq, context.yield);
std::transform(
std::move_iterator(txns.begin()),
@@ -136,8 +133,7 @@ doLedger(Context const& context)
}
else
{
auto hashes = context.backend->fetchAllTransactionHashesInLedger(
lgrInfo.seq, context.yield);
auto hashes = context.backend->fetchAllTransactionHashesInLedger(lgrInfo.seq, context.yield);
std::transform(
std::move_iterator(hashes.begin()),
std::move_iterator(hashes.end()),
@@ -153,8 +149,7 @@ doLedger(Context const& context)
{
header["diff"] = boost::json::value(boost::json::array_kind);
boost::json::array& jsonDiff = header.at("diff").as_array();
auto diff =
context.backend->fetchLedgerDiff(lgrInfo.seq, context.yield);
auto diff = context.backend->fetchLedgerDiff(lgrInfo.seq, context.yield);
for (auto const& obj : diff)
{
boost::json::object entry;
@@ -163,9 +158,7 @@ doLedger(Context const& context)
entry["object"] = ripple::strHex(obj.blob);
else if (obj.blob.size())
{
ripple::STLedgerEntry sle{
ripple::SerialIter{obj.blob.data(), obj.blob.size()},
obj.key};
ripple::STLedgerEntry sle{ripple::SerialIter{obj.blob.data(), obj.blob.size()}, obj.key};
entry["object"] = toJson(sle);
}
else

54
src/rpc/handlers/LedgerData.cpp
View File

@@ -82,14 +82,11 @@ doLedgerData(Context const& context)
if (outOfOrder)
{
if (!request.at(JS(marker)).is_int64())
return Status{
RippledError::rpcINVALID_PARAMS,
"markerNotStringOrInt"};
return Status{RippledError::rpcINVALID_PARAMS, "markerNotStringOrInt"};
diffMarker = value_to<uint32_t>(request.at(JS(marker)));
}
else
return Status{
RippledError::rpcINVALID_PARAMS, "markerNotString"};
return Status{RippledError::rpcINVALID_PARAMS, "markerNotString"};
}
else
{
@@ -97,8 +94,7 @@ doLedgerData(Context const& context)
marker = ripple::uint256{};
if (!marker->parseHex(request.at(JS(marker)).as_string().c_str()))
return Status{
RippledError::rpcINVALID_PARAMS, "markerMalformed"};
return Status{RippledError::rpcINVALID_PARAMS, "markerMalformed"};
}
}
@@ -121,16 +117,13 @@ doLedgerData(Context const& context)
header[JS(accepted)] = true;
header[JS(account_hash)] = ripple::strHex(lgrInfo.accountHash);
header[JS(close_flags)] = lgrInfo.closeFlags;
header[JS(close_time)] =
lgrInfo.closeTime.time_since_epoch().count();
header[JS(close_time)] = lgrInfo.closeTime.time_since_epoch().count();
header[JS(close_time_human)] = ripple::to_string(lgrInfo.closeTime);
header[JS(close_time_resolution)] =
lgrInfo.closeTimeResolution.count();
header[JS(close_time_resolution)] = lgrInfo.closeTimeResolution.count();
header[JS(hash)] = ripple::strHex(lgrInfo.hash);
header[JS(ledger_hash)] = ripple::strHex(lgrInfo.hash);
header[JS(ledger_index)] = std::to_string(lgrInfo.seq);
header[JS(parent_close_time)] =
lgrInfo.parentCloseTime.time_since_epoch().count();
header[JS(parent_close_time)] = lgrInfo.parentCloseTime.time_since_epoch().count();
header[JS(parent_hash)] = ripple::strHex(lgrInfo.parentHash);
header[JS(seqNum)] = std::to_string(lgrInfo.seq);
header[JS(totalCoins)] = ripple::to_string(lgrInfo.drops);
@@ -143,11 +136,8 @@ doLedgerData(Context const& context)
}
else
{
if (!outOfOrder &&
!context.backend->fetchLedgerObject(
*marker, lgrInfo.seq, context.yield))
return Status{
RippledError::rpcINVALID_PARAMS, "markerDoesNotExist"};
if (!outOfOrder && !context.backend->fetchLedgerObject(*marker, lgrInfo.seq, context.yield))
return Status{RippledError::rpcINVALID_PARAMS, "markerDoesNotExist"};
}
response[JS(ledger_hash)] = ripple::strHex(lgrInfo.hash);
@@ -158,8 +148,7 @@ doLedgerData(Context const& context)
if (diffMarker)
{
assert(outOfOrder);
auto diff =
context.backend->fetchLedgerDiff(*diffMarker, context.yield);
auto diff = context.backend->fetchLedgerDiff(*diffMarker, context.yield);
std::vector<ripple::uint256> keys;
for (auto&& [key, object] : diff)
{
@@ -168,8 +157,7 @@ doLedgerData(Context const& context)
keys.push_back(std::move(key));
}
}
auto objs = context.backend->fetchLedgerObjects(
keys, lgrInfo.seq, context.yield);
auto objs = context.backend->fetchLedgerObjects(keys, lgrInfo.seq, context.yield);
for (size_t i = 0; i < objs.size(); ++i)
{
auto&& obj = objs[i];
@@ -181,29 +169,23 @@ doLedgerData(Context const& context)
}
else
{
auto page = context.backend->fetchLedgerPage(
marker, lgrInfo.seq, limit, outOfOrder, context.yield);
auto page = context.backend->fetchLedgerPage(marker, lgrInfo.seq, limit, outOfOrder, context.yield);
results = std::move(page.objects);
if (page.cursor)
response["marker"] = ripple::strHex(*(page.cursor));
else if (outOfOrder)
response["marker"] =
context.backend->fetchLedgerRange()->maxSequence;
response["marker"] = context.backend->fetchLedgerRange()->maxSequence;
}
auto end = std::chrono::system_clock::now();
auto time =
std::chrono::duration_cast<std::chrono::microseconds>(end - start)
.count();
auto time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
gLog.debug() << "Number of results = " << results.size() << " fetched in "
<< time << " microseconds";
gLog.debug() << "Number of results = " << results.size() << " fetched in " << time << " microseconds";
boost::json::array objects;
objects.reserve(results.size());
for (auto const& [key, object] : results)
{
ripple::STLedgerEntry sle{
ripple::SerialIter{object.data(), object.size()}, key};
ripple::STLedgerEntry sle{ripple::SerialIter{object.data(), object.size()}, key};
if (binary)
{
boost::json::object entry;
@@ -219,10 +201,8 @@ doLedgerData(Context const& context)
response["cache_full"] = context.backend->cache().isFull();
auto end2 = std::chrono::system_clock::now();
time = std::chrono::duration_cast<std::chrono::microseconds>(end2 - end)
.count();
gLog.debug() << "Number of results = " << results.size()
<< " serialized in " << time << " microseconds";
time = std::chrono::duration_cast<std::chrono::microseconds>(end2 - end).count();
gLog.debug() << "Number of results = " << results.size() << " serialized in " << time << " microseconds";
return response;
}

166
src/rpc/handlers/LedgerEntry.cpp
View File

@@ -64,11 +64,9 @@ doLedgerEntry(Context const& context)
else if (request.contains(JS(account_root)))
{
if (!request.at(JS(account_root)).is_string())
return Status{
RippledError::rpcINVALID_PARAMS, "account_rootNotString"};
return Status{RippledError::rpcINVALID_PARAMS, "account_rootNotString"};
auto const account = ripple::parseBase58<ripple::AccountID>(
request.at(JS(account_root)).as_string().c_str());
auto const account = ripple::parseBase58<ripple::AccountID>(request.at(JS(account_root)).as_string().c_str());
expectedType = ripple::ltACCOUNT_ROOT;
if (!account || account->isZero())
return Status{ClioError::rpcMALFORMED_ADDRESS};
@@ -92,52 +90,37 @@ doLedgerEntry(Context const& context)
if (!request.at(JS(deposit_preauth)).is_object())
{
if (!request.at(JS(deposit_preauth)).is_string() ||
!key.parseHex(
request.at(JS(deposit_preauth)).as_string().c_str()))
!key.parseHex(request.at(JS(deposit_preauth)).as_string().c_str()))
{
return Status{
RippledError::rpcINVALID_PARAMS,
"deposit_preauthMalformed"};
return Status{RippledError::rpcINVALID_PARAMS, "deposit_preauthMalformed"};
}
}
else if (
!request.at(JS(deposit_preauth)).as_object().contains(JS(owner)) ||
!request.at(JS(deposit_preauth))
.as_object()
.at(JS(owner))
.is_string())
!request.at(JS(deposit_preauth)).as_object().at(JS(owner)).is_string())
{
return Status{RippledError::rpcINVALID_PARAMS, "malformedOwner"};
}
else if (
!request.at(JS(deposit_preauth))
.as_object()
.contains(JS(authorized)) ||
!request.at(JS(deposit_preauth))
.as_object()
.at(JS(authorized))
.is_string())
!request.at(JS(deposit_preauth)).as_object().contains(JS(authorized)) ||
!request.at(JS(deposit_preauth)).as_object().at(JS(authorized)).is_string())
{
return Status{
RippledError::rpcINVALID_PARAMS, "authorizedNotString"};
return Status{RippledError::rpcINVALID_PARAMS, "authorizedNotString"};
}
else
{
boost::json::object const& deposit_preauth =
request.at(JS(deposit_preauth)).as_object();
boost::json::object const& deposit_preauth = request.at(JS(deposit_preauth)).as_object();
auto const owner = ripple::parseBase58<ripple::AccountID>(
deposit_preauth.at(JS(owner)).as_string().c_str());
auto const owner =
ripple::parseBase58<ripple::AccountID>(deposit_preauth.at(JS(owner)).as_string().c_str());
auto const authorized = ripple::parseBase58<ripple::AccountID>(
deposit_preauth.at(JS(authorized)).as_string().c_str());
auto const authorized =
ripple::parseBase58<ripple::AccountID>(deposit_preauth.at(JS(authorized)).as_string().c_str());
if (!owner)
return Status{
RippledError::rpcINVALID_PARAMS, "malformedOwner"};
return Status{RippledError::rpcINVALID_PARAMS, "malformedOwner"};
else if (!authorized)
return Status{
RippledError::rpcINVALID_PARAMS, "malformedAuthorized"};
return Status{RippledError::rpcINVALID_PARAMS, "malformedAuthorized"};
else
key = ripple::keylet::depositPreauth(*owner, *authorized).key;
}
@@ -148,13 +131,11 @@ doLedgerEntry(Context const& context)
if (!request.at(JS(directory)).is_object())
{
if (!request.at(JS(directory)).is_string())
return Status{
RippledError::rpcINVALID_PARAMS, "directoryNotString"};
return Status{RippledError::rpcINVALID_PARAMS, "directoryNotString"};
if (!key.parseHex(request.at(JS(directory)).as_string().c_str()))
{
return Status{
RippledError::rpcINVALID_PARAMS, "malformedDirectory"};
return Status{RippledError::rpcINVALID_PARAMS, "malformedDirectory"};
}
}
else if (
@@ -167,8 +148,7 @@ doLedgerEntry(Context const& context)
{
auto directory = request.at(JS(directory)).as_object();
std::uint64_t subIndex = directory.contains(JS(sub_index))
? boost::json::value_to<std::uint64_t>(
directory.at(JS(sub_index)))
? boost::json::value_to<std::uint64_t>(directory.at(JS(sub_index)))
: 0;
if (directory.contains(JS(dir_root)))
@@ -178,15 +158,11 @@ doLedgerEntry(Context const& context)
if (directory.contains(JS(owner)))
{
// May not specify both dir_root and owner.
return Status{
RippledError::rpcINVALID_PARAMS,
"mayNotSpecifyBothDirRootAndOwner"};
return Status{RippledError::rpcINVALID_PARAMS, "mayNotSpecifyBothDirRootAndOwner"};
}
else if (!uDirRoot.parseHex(
directory.at(JS(dir_root)).as_string().c_str()))
else if (!uDirRoot.parseHex(directory.at(JS(dir_root)).as_string().c_str()))
{
return Status{
RippledError::rpcINVALID_PARAMS, "malformedDirRoot"};
return Status{RippledError::rpcINVALID_PARAMS, "malformedDirRoot"};
}
else
{
@@ -195,8 +171,8 @@ doLedgerEntry(Context const& context)
}
else if (directory.contains(JS(owner)))
{
auto const ownerID = ripple::parseBase58<ripple::AccountID>(
directory.at(JS(owner)).as_string().c_str());
auto const ownerID =
ripple::parseBase58<ripple::AccountID>(directory.at(JS(owner)).as_string().c_str());
if (!ownerID)
{
@@ -204,15 +180,12 @@ doLedgerEntry(Context const& context)
}
else
{
key = ripple::keylet::page(
ripple::keylet::ownerDir(*ownerID), subIndex)
.key;
key = ripple::keylet::page(ripple::keylet::ownerDir(*ownerID), subIndex).key;
}
}
else
{
return Status{
RippledError::rpcINVALID_PARAMS, "missingOwnerOrDirRoot"};
return Status{RippledError::rpcINVALID_PARAMS, "missingOwnerOrDirRoot"};
}
}
}
@@ -222,8 +195,7 @@ doLedgerEntry(Context const& context)
if (!request.at(JS(escrow)).is_object())
{
if (!key.parseHex(request.at(JS(escrow)).as_string().c_str()))
return Status{
RippledError::rpcINVALID_PARAMS, "malformedEscrow"};
return Status{RippledError::rpcINVALID_PARAMS, "malformedEscrow"};
}
else if (
!request.at(JS(escrow)).as_object().contains(JS(owner)) ||
@@ -239,19 +211,14 @@ doLedgerEntry(Context const& context)
}
else
{
auto const id =
ripple::parseBase58<ripple::AccountID>(request.at(JS(escrow))
.as_object()
.at(JS(owner))
.as_string()
.c_str());
auto const id = ripple::parseBase58<ripple::AccountID>(
request.at(JS(escrow)).as_object().at(JS(owner)).as_string().c_str());
if (!id)
return Status{ClioError::rpcMALFORMED_ADDRESS};
else
{
std::uint32_t seq =
request.at(JS(escrow)).as_object().at(JS(seq)).as_int64();
std::uint32_t seq = request.at(JS(escrow)).as_object().at(JS(seq)).as_int64();
key = ripple::keylet::escrow(*id, seq).key;
}
}
@@ -262,8 +229,7 @@ doLedgerEntry(Context const& context)
if (!request.at(JS(offer)).is_object())
{
if (!key.parseHex(request.at(JS(offer)).as_string().c_str()))
return Status{
RippledError::rpcINVALID_PARAMS, "malformedOffer"};
return Status{RippledError::rpcINVALID_PARAMS, "malformedOffer"};
}
else if (
!request.at(JS(offer)).as_object().contains(JS(account)) ||
@@ -280,15 +246,13 @@ doLedgerEntry(Context const& context)
else
{
auto offer = request.at(JS(offer)).as_object();
auto const id = ripple::parseBase58<ripple::AccountID>(
offer.at(JS(account)).as_string().c_str());
auto const id = ripple::parseBase58<ripple::AccountID>(offer.at(JS(account)).as_string().c_str());
if (!id)
return Status{ClioError::rpcMALFORMED_ADDRESS};
else
{
std::uint32_t seq =
boost::json::value_to<std::uint32_t>(offer.at(JS(seq)));
std::uint32_t seq = boost::json::value_to<std::uint32_t>(offer.at(JS(seq)));
key = ripple::keylet::offer(*id, seq).key;
}
}
@@ -297,34 +261,27 @@ doLedgerEntry(Context const& context)
{
expectedType = ripple::ltPAYCHAN;
if (!request.at(JS(payment_channel)).is_string())
return Status{
RippledError::rpcINVALID_PARAMS, "paymentChannelNotString"};
return Status{RippledError::rpcINVALID_PARAMS, "paymentChannelNotString"};
if (!key.parseHex(request.at(JS(payment_channel)).as_string().c_str()))
return Status{
RippledError::rpcINVALID_PARAMS, "malformedPaymentChannel"};
return Status{RippledError::rpcINVALID_PARAMS, "malformedPaymentChannel"};
}
else if (request.contains(JS(ripple_state)))
{
if (!request.at(JS(ripple_state)).is_object())
return Status{
RippledError::rpcINVALID_PARAMS, "rippleStateNotObject"};
return Status{RippledError::rpcINVALID_PARAMS, "rippleStateNotObject"};
expectedType = ripple::ltRIPPLE_STATE;
ripple::Currency currency;
boost::json::object const& state =
request.at(JS(ripple_state)).as_object();
boost::json::object const& state = request.at(JS(ripple_state)).as_object();
if (!state.contains(JS(currency)) ||
!state.at(JS(currency)).is_string())
if (!state.contains(JS(currency)) || !state.at(JS(currency)).is_string())
{
return Status{RippledError::rpcINVALID_PARAMS, "currencyNotString"};
}
if (!state.contains(JS(accounts)) ||
!state.at(JS(accounts)).is_array() ||
2 != state.at(JS(accounts)).as_array().size() ||
!state.at(JS(accounts)).as_array().at(0).is_string() ||
if (!state.contains(JS(accounts)) || !state.at(JS(accounts)).is_array() ||
2 != state.at(JS(accounts)).as_array().size() || !state.at(JS(accounts)).as_array().at(0).is_string() ||
!state.at(JS(accounts)).as_array().at(1).is_string() ||
(state.at(JS(accounts)).as_array().at(0).as_string() ==
state.at(JS(accounts)).as_array().at(1).as_string()))
@@ -332,19 +289,16 @@ doLedgerEntry(Context const& context)
return Status{RippledError::rpcINVALID_PARAMS, "malformedAccounts"};
}
auto const id1 = ripple::parseBase58<ripple::AccountID>(
state.at(JS(accounts)).as_array().at(0).as_string().c_str());
auto const id2 = ripple::parseBase58<ripple::AccountID>(
state.at(JS(accounts)).as_array().at(1).as_string().c_str());
auto const id1 =
ripple::parseBase58<ripple::AccountID>(state.at(JS(accounts)).as_array().at(0).as_string().c_str());
auto const id2 =
ripple::parseBase58<ripple::AccountID>(state.at(JS(accounts)).as_array().at(1).as_string().c_str());
if (!id1 || !id2)
return Status{
ClioError::rpcMALFORMED_ADDRESS, "malformedAddresses"};
return Status{ClioError::rpcMALFORMED_ADDRESS, "malformedAddresses"};
else if (!ripple::to_currency(
currency, state.at(JS(currency)).as_string().c_str()))
return Status{
ClioError::rpcMALFORMED_CURRENCY, "malformedCurrency"};
else if (!ripple::to_currency(currency, state.at(JS(currency)).as_string().c_str()))
return Status{ClioError::rpcMALFORMED_CURRENCY, "malformedCurrency"};
key = ripple::keylet::line(*id1, *id2, currency).key;
}
@@ -355,12 +309,10 @@ doLedgerEntry(Context const& context)
if (!request.at(JS(ticket)).is_object())
{
if (!request.at(JS(ticket)).is_string())
return Status{
ClioError::rpcMALFORMED_REQUEST, "ticketNotString"};
return Status{ClioError::rpcMALFORMED_REQUEST, "ticketNotString"};
if (!key.parseHex(request.at(JS(ticket)).as_string().c_str()))
return Status{
ClioError::rpcMALFORMED_REQUEST, "malformedTicket"};
return Status{ClioError::rpcMALFORMED_REQUEST, "malformedTicket"};
}
else if (
!request.at(JS(ticket)).as_object().contains(JS(account)) ||
@@ -372,26 +324,18 @@ doLedgerEntry(Context const& context)
!request.at(JS(ticket)).as_object().contains(JS(ticket_seq)) ||
!request.at(JS(ticket)).as_object().at(JS(ticket_seq)).is_int64())
{
return Status{
ClioError::rpcMALFORMED_REQUEST, "malformedTicketSeq"};
return Status{ClioError::rpcMALFORMED_REQUEST, "malformedTicketSeq"};
}
else
{
auto const id =
ripple::parseBase58<ripple::AccountID>(request.at(JS(ticket))
.as_object()
.at(JS(account))
.as_string()
.c_str());
auto const id = ripple::parseBase58<ripple::AccountID>(
request.at(JS(ticket)).as_object().at(JS(account)).as_string().c_str());
if (!id)
return Status{ClioError::rpcMALFORMED_OWNER};
else
{
std::uint32_t seq = request.at(JS(ticket))
.as_object()
.at(JS(ticket_seq))
.as_int64();
std::uint32_t seq = request.at(JS(ticket)).as_object().at(JS(ticket_seq)).as_int64();
key = ripple::getTicketIndex(*id, seq);
}
@@ -402,15 +346,13 @@ doLedgerEntry(Context const& context)
return Status{RippledError::rpcINVALID_PARAMS, "unknownOption"};
}
auto dbResponse =
context.backend->fetchLedgerObject(key, lgrInfo.seq, context.yield);
auto dbResponse = context.backend->fetchLedgerObject(key, lgrInfo.seq, context.yield);
if (!dbResponse or dbResponse->size() == 0)
return Status{"entryNotFound"};
// check expected type matches actual type
ripple::STLedgerEntry sle{
ripple::SerialIter{dbResponse->data(), dbResponse->size()}, key};
ripple::STLedgerEntry sle{ripple::SerialIter{dbResponse->data(), dbResponse->size()}, key};
if (expectedType != ripple::ltANY && sle.getType() != expectedType)
return Status{"unexpectedLedgerType"};

14
src/rpc/handlers/NFTHistory.cpp
View File

@@ -46,16 +46,12 @@ doNFTHistory(Context const& context)
std::uint32_t const limit,
bool const forward,
std::optional<Backend::TransactionsCursor> const& cursorIn,
boost::asio::yield_context& yield)
-> Backend::TransactionsAndCursor {
auto const [txnsAndCursor, timeDiff] =
util::timed([&, &tokenID = tokenID]() {
return backend->fetchNFTTransactions(
tokenID, limit, forward, cursorIn, yield);
});
boost::asio::yield_context& yield) -> Backend::TransactionsAndCursor {
auto const [txnsAndCursor, timeDiff] = util::timed([&, &tokenID = tokenID]() {
return backend->fetchNFTTransactions(tokenID, limit, forward, cursorIn, yield);
});
gLog.info() << outerFuncName << " db fetch took " << timeDiff
<< " milliseconds - num blobs = "
<< txnsAndCursor.txns.size();
<< " milliseconds - num blobs = " << txnsAndCursor.txns.size();
return txnsAndCursor;
});

9
src/rpc/handlers/NFTInfo.cpp
View File

@@ -42,8 +42,7 @@ doNFTInfo(Context const& context)
return *status;
auto const lgrInfo = std::get<ripple::LedgerInfo>(maybeLedgerInfo);
auto const dbResponse =
context.backend->fetchNFT(tokenID, lgrInfo.seq, context.yield);
auto const dbResponse = context.backend->fetchNFT(tokenID, lgrInfo.seq, context.yield);
if (!dbResponse)
return Status{RippledError::rpcOBJECT_NOT_FOUND, "NFT not found"};
@@ -55,10 +54,8 @@ doNFTInfo(Context const& context)
response[JS(flags)] = ripple::nft::getFlags(dbResponse->tokenID);
response["transfer_fee"] = ripple::nft::getTransferFee(dbResponse->tokenID);
response[JS(issuer)] =
ripple::toBase58(ripple::nft::getIssuer(dbResponse->tokenID));
response["nft_taxon"] =
ripple::nft::toUInt32(ripple::nft::getTaxon(dbResponse->tokenID));
response[JS(issuer)] = ripple::toBase58(ripple::nft::getIssuer(dbResponse->tokenID));
response["nft_taxon"] = ripple::nft::toUInt32(ripple::nft::getTaxon(dbResponse->tokenID));
response[JS(nft_serial)] = ripple::nft::getSerial(dbResponse->tokenID);
return response;

36
src/rpc/handlers/NFTOffers.cpp
View File

@@ -36,8 +36,7 @@ namespace ripple {
inline void
tag_invoke(json::value_from_tag, json::value& jv, SLE const& offer)
{
auto amount = ::RPC::toBoostJson(
offer.getFieldAmount(sfAmount).getJson(JsonOptions::none));
auto amount = ::RPC::toBoostJson(offer.getFieldAmount(sfAmount).getJson(JsonOptions::none));
json::object obj = {
{JS(nft_offer_index), to_string(offer.key())},
@@ -47,8 +46,7 @@ tag_invoke(json::value_from_tag, json::value& jv, SLE const& offer)
};
if (offer.isFieldPresent(sfDestination))
obj.insert_or_assign(
JS(destination), toBase58(offer.getAccountID(sfDestination)));
obj.insert_or_assign(JS(destination), toBase58(offer.getAccountID(sfDestination)));
if (offer.isFieldPresent(sfExpiration))
obj.insert_or_assign(JS(expiration), offer.getFieldU32(sfExpiration));
@@ -61,10 +59,7 @@ tag_invoke(json::value_from_tag, json::value& jv, SLE const& offer)
namespace RPC {
Result
enumerateNFTOffers(
Context const& context,
ripple::uint256 const& tokenid,
ripple::Keylet const& directory)
enumerateNFTOffers(Context const& context, ripple::uint256 const& tokenid, ripple::Keylet const& directory)
{
auto const& request = context.params;
@@ -75,8 +70,7 @@ enumerateNFTOffers(
auto lgrInfo = std::get<ripple::LedgerInfo>(v);
// TODO: just check for existence without pulling
if (!context.backend->fetchLedgerObject(
directory.key, lgrInfo.seq, context.yield))
if (!context.backend->fetchLedgerObject(directory.key, lgrInfo.seq, context.yield))
return Status{RippledError::rpcOBJECT_NOT_FOUND, "notFound"};
std::uint32_t limit;
@@ -104,12 +98,9 @@ enumerateNFTOffers(
if (!cursor.parseHex(marker.as_string().c_str()))
return Status{RippledError::rpcINVALID_PARAMS, "malformedCursor"};
auto const sle =
read(ripple::keylet::nftoffer(cursor), lgrInfo, context);
auto const sle = read(ripple::keylet::nftoffer(cursor), lgrInfo, context);
if (!sle ||
sle->getFieldU16(ripple::sfLedgerEntryType) !=
ripple::ltNFTOKEN_OFFER ||
if (!sle || sle->getFieldU16(ripple::sfLedgerEntryType) != ripple::ltNFTOKEN_OFFER ||
tokenid != sle->getFieldH256(ripple::sfNFTokenID))
return Status{RippledError::rpcINVALID_PARAMS};
@@ -152,14 +143,10 @@ enumerateNFTOffers(
offers.pop_back();
}
std::transform(
std::cbegin(offers),
std::cend(offers),
std::back_inserter(jsonOffers),
[](auto const& offer) {
// uses tag_invoke at the top of this file
return json::value_from(offer);
});
std::transform(std::cbegin(offers), std::cend(offers), std::back_inserter(jsonOffers), [](auto const& offer) {
// uses tag_invoke at the top of this file
return json::value_from(offer);
});
response.insert_or_assign(JS(offers), std::move(jsonOffers));
return response;
@@ -179,8 +166,7 @@ doNFTOffers(Context const& context, bool sells)
return ripple::keylet::nft_buys(std::get<ripple::uint256>(v));
};
return enumerateNFTOffers(
context, std::get<ripple::uint256>(v), getKeylet());
return enumerateNFTOffers(context, std::get<ripple::uint256>(v), getKeylet());
}
Result

58
src/rpc/handlers/NoRippleCheck.cpp
View File

@@ -23,10 +23,7 @@
namespace RPC {
boost::json::object
getBaseTx(
ripple::AccountID const& accountID,
std::uint32_t accountSeq,
ripple::Fees const& fees)
getBaseTx(ripple::AccountID const& accountID, std::uint32_t accountSeq, ripple::Fees const& fees)
{
boost::json::object tx;
tx[JS(Sequence)] = accountSeq;
@@ -50,8 +47,7 @@ doNoRippleCheck(Context const& context)
if (role == "gateway")
roleGateway = true;
else if (role != "user")
return Status{
RippledError::rpcINVALID_PARAMS, "role field is invalid"};
return Status{RippledError::rpcINVALID_PARAMS, "role field is invalid"};
}
std::uint32_t limit = 300;
@@ -65,15 +61,13 @@ doNoRippleCheck(Context const& context)
return *status;
auto lgrInfo = std::get<ripple::LedgerInfo>(v);
std::optional<ripple::Fees> fees = includeTxs
? context.backend->fetchFees(lgrInfo.seq, context.yield)
: std::nullopt;
std::optional<ripple::Fees> fees =
includeTxs ? context.backend->fetchFees(lgrInfo.seq, context.yield) : std::nullopt;
boost::json::array transactions;
auto keylet = ripple::keylet::account(accountID);
auto accountObj = context.backend->fetchLedgerObject(
keylet.key, lgrInfo.seq, context.yield);
auto accountObj = context.backend->fetchLedgerObject(keylet.key, lgrInfo.seq, context.yield);
if (!accountObj)
throw AccountNotFoundError(ripple::toBase58(accountID));
@@ -83,8 +77,7 @@ doNoRippleCheck(Context const& context)
std::uint32_t accountSeq = sle.getFieldU32(ripple::sfSequence);
boost::json::array problems;
bool bDefaultRipple =
sle.getFieldU32(ripple::sfFlags) & ripple::lsfDefaultRipple;
bool bDefaultRipple = sle.getFieldU32(ripple::sfFlags) & ripple::lsfDefaultRipple;
if (bDefaultRipple & !roleGateway)
{
problems.push_back(
@@ -95,8 +88,7 @@ doNoRippleCheck(Context const& context)
}
else if (roleGateway & !bDefaultRipple)
{
problems.push_back(
"You should immediately set your default ripple flag");
problems.push_back("You should immediately set your default ripple flag");
if (includeTxs)
{
auto tx = getBaseTx(accountID, accountSeq++, *fees);
@@ -113,21 +105,14 @@ doNoRippleCheck(Context const& context)
std::numeric_limits<std::uint32_t>::max(),
{},
context.yield,
[roleGateway,
includeTxs,
&fees,
&transactions,
&accountSeq,
&limit,
&accountID,
&problems](ripple::SLE&& ownedItem) {
[roleGateway, includeTxs, &fees, &transactions, &accountSeq, &limit, &accountID, &problems](
ripple::SLE&& ownedItem) {
if (ownedItem.getType() == ripple::ltRIPPLE_STATE)
{
bool const bLow = accountID ==
ownedItem.getFieldAmount(ripple::sfLowLimit).getIssuer();
bool const bLow = accountID == ownedItem.getFieldAmount(ripple::sfLowLimit).getIssuer();
bool const bNoRipple = ownedItem.getFieldU32(ripple::sfFlags) &
(bLow ? ripple::lsfLowNoRipple : ripple::lsfHighNoRipple);
bool const bNoRipple =
ownedItem.getFieldU32(ripple::sfFlags) & (bLow ? ripple::lsfLowNoRipple : ripple::lsfHighNoRipple);
std::string problem;
bool needFix = false;
@@ -146,27 +131,22 @@ doNoRippleCheck(Context const& context)
if (needFix)
{
ripple::AccountID peer =
ownedItem
.getFieldAmount(
bLow ? ripple::sfHighLimit : ripple::sfLowLimit)
.getIssuer();
ripple::STAmount peerLimit = ownedItem.getFieldAmount(
bLow ? ripple::sfHighLimit : ripple::sfLowLimit);
ownedItem.getFieldAmount(bLow ? ripple::sfHighLimit : ripple::sfLowLimit).getIssuer();
ripple::STAmount peerLimit =
ownedItem.getFieldAmount(bLow ? ripple::sfHighLimit : ripple::sfLowLimit);
problem += to_string(peerLimit.getCurrency());
problem += " line to ";
problem += to_string(peerLimit.getIssuer());
problems.emplace_back(problem);
if (includeTxs)
{
ripple::STAmount limitAmount(ownedItem.getFieldAmount(
bLow ? ripple::sfLowLimit : ripple::sfHighLimit));
ripple::STAmount limitAmount(
ownedItem.getFieldAmount(bLow ? ripple::sfLowLimit : ripple::sfHighLimit));
limitAmount.setIssuer(peer);
auto tx = getBaseTx(accountID, accountSeq++, *fees);
tx[JS(TransactionType)] = JS(TrustSet);
tx[JS(LimitAmount)] = RPC::toBoostJson(
limitAmount.getJson(ripple::JsonOptions::none));
tx[JS(Flags)] = bNoRipple ? ripple::tfClearNoRipple
: ripple::tfSetNoRipple;
tx[JS(LimitAmount)] = RPC::toBoostJson(limitAmount.getJson(ripple::JsonOptions::none));
tx[JS(Flags)] = bNoRipple ? ripple::tfClearNoRipple : ripple::tfSetNoRipple;
transactions.push_back(tx);
}

29
src/rpc/handlers/ServerInfo.cpp
View File

@@ -33,23 +33,18 @@ doServerInfo(Context const& context)
auto range = context.backend->fetchLedgerRange();
if (!range)
{
return Status{
RippledError::rpcNOT_READY,
"emptyDatabase",
"The server has no data in the database"};
return Status{RippledError::rpcNOT_READY, "emptyDatabase", "The server has no data in the database"};
}
auto lgrInfo = context.backend->fetchLedgerBySequence(
range->maxSequence, context.yield);
auto lgrInfo = context.backend->fetchLedgerBySequence(range->maxSequence, context.yield);
auto fees = context.backend->fetchFees(lgrInfo->seq, context.yield);
if (!lgrInfo || !fees)
return Status{RippledError::rpcINTERNAL};
auto age = std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count() -
auto age =
std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count() -
lgrInfo->closeTime.time_since_epoch().count() - 946684800;
if (age < 0)
@@ -58,20 +53,18 @@ doServerInfo(Context const& context)
response[JS(info)] = boost::json::object{};
boost::json::object& info = response[JS(info)].as_object();
info[JS(complete_ledgers)] = std::to_string(range->minSequence) + "-" +
std::to_string(range->maxSequence);
info[JS(complete_ledgers)] = std::to_string(range->minSequence) + "-" + std::to_string(range->maxSequence);
bool admin = context.clientIp == "127.0.0.1";
if (admin)
{
info[JS(counters)] = context.counters.report();
info[JS(counters)].as_object()["subscriptions"] =
context.subscriptions->report();
info[JS(counters)].as_object()["subscriptions"] = context.subscriptions->report();
}
auto serverInfoRippled = context.balancer->forwardToRippled(
{{"command", "server_info"}}, context.clientIp, context.yield);
auto serverInfoRippled =
context.balancer->forwardToRippled({{"command", "server_info"}}, context.clientIp, context.yield);
info[JS(load_factor)] = 1;
info["clio_version"] = Build::getClioVersionString();
@@ -105,11 +98,9 @@ doServerInfo(Context const& context)
cache["size"] = context.backend->cache().size();
cache["is_full"] = context.backend->cache().isFull();
cache["latest_ledger_seq"] =
context.backend->cache().latestLedgerSequence();
cache["latest_ledger_seq"] = context.backend->cache().latestLedgerSequence();
cache["object_hit_rate"] = context.backend->cache().getObjectHitRate();
cache["successor_hit_rate"] =
context.backend->cache().getSuccessorHitRate();
cache["successor_hit_rate"] = context.backend->cache().getSuccessorHitRate();
if (admin)
{

85
src/rpc/handlers/Subscribe.cpp
View File

@@ -26,19 +26,13 @@
namespace RPC {
// these are the streams that take no arguments
static std::unordered_set<std::string> validCommonStreams{
"ledger",
"transactions",
"transactions_proposed",
"validations",
"manifests",
"book_changes"};
static std::unordered_set<std::string>
validCommonStreams{"ledger", "transactions", "transactions_proposed", "validations", "manifests", "book_changes"};
Status
validateStreams(boost::json::object const& request)
{
for (auto const& streams = request.at(JS(streams)).as_array();
auto const& stream : streams)
for (auto const& streams = request.at(JS(streams)).as_array(); auto const& stream : streams)
{
if (!stream.is_string())
return Status{RippledError::rpcINVALID_PARAMS, "streamNotString"};
@@ -83,10 +77,7 @@ subscribeToStreams(
}
void
unsubscribeToStreams(
boost::json::object const& request,
std::shared_ptr<WsBase> session,
SubscriptionManager& manager)
unsubscribeToStreams(boost::json::object const& request, std::shared_ptr<WsBase> session, SubscriptionManager& manager)
{
boost::json::array const& streams = request.at(JS(streams)).as_array();
@@ -127,10 +118,7 @@ validateAccounts(boost::json::array const& accounts)
}
void
subscribeToAccounts(
boost::json::object const& request,
std::shared_ptr<WsBase> session,
SubscriptionManager& manager)
subscribeToAccounts(boost::json::object const& request, std::shared_ptr<WsBase> session, SubscriptionManager& manager)
{
boost::json::array const& accounts = request.at(JS(accounts)).as_array();
@@ -151,10 +139,7 @@ subscribeToAccounts(
}
void
unsubscribeToAccounts(
boost::json::object const& request,
std::shared_ptr<WsBase> session,
SubscriptionManager& manager)
unsubscribeToAccounts(boost::json::object const& request, std::shared_ptr<WsBase> session, SubscriptionManager& manager)
{
boost::json::array const& accounts = request.at(JS(accounts)).as_array();
@@ -180,8 +165,7 @@ subscribeToAccountsProposed(
std::shared_ptr<WsBase> session,
SubscriptionManager& manager)
{
boost::json::array const& accounts =
request.at(JS(accounts_proposed)).as_array();
boost::json::array const& accounts = request.at(JS(accounts_proposed)).as_array();
for (auto const& account : accounts)
{
@@ -205,8 +189,7 @@ unsubscribeToAccountsProposed(
std::shared_ptr<WsBase> session,
SubscriptionManager& manager)
{
boost::json::array const& accounts =
request.at(JS(accounts_proposed)).as_array();
boost::json::array const& accounts = request.at(JS(accounts_proposed)).as_array();
for (auto const& account : accounts)
{
@@ -255,23 +238,15 @@ validateAndGetBooks(
rng = backend->fetchLedgerRange();
ripple::AccountID takerID = beast::zero;
if (book.as_object().contains(JS(taker)))
if (auto const status = getTaker(book.as_object(), takerID);
status)
if (auto const status = getTaker(book.as_object(), takerID); status)
return status;
auto getOrderBook = [&snapshot, &backend, &rng, &takerID](
auto book,
boost::asio::yield_context& yield) {
auto getOrderBook = [&snapshot, &backend, &rng, &takerID](auto book, boost::asio::yield_context& yield) {
auto bookBase = getBookBase(book);
auto [offers, _] = backend->fetchBookOffers(
bookBase, rng->maxSequence, 200, yield);
auto [offers, _] = backend->fetchBookOffers(bookBase, rng->maxSequence, 200, yield);
auto orderBook = postProcessOrderBook(
offers, book, takerID, *backend, rng->maxSequence, yield);
std::copy(
orderBook.begin(),
orderBook.end(),
std::back_inserter(snapshot));
auto orderBook = postProcessOrderBook(offers, book, takerID, *backend, rng->maxSequence, yield);
std::copy(orderBook.begin(), orderBook.end(), std::back_inserter(snapshot));
};
getOrderBook(b, yield);
if (both)
@@ -282,10 +257,7 @@ validateAndGetBooks(
}
void
subscribeToBooks(
std::vector<ripple::Book> const& books,
std::shared_ptr<WsBase> session,
SubscriptionManager& manager)
subscribeToBooks(std::vector<ripple::Book> const& books, std::shared_ptr<WsBase> session, SubscriptionManager& manager)
{
for (auto const& book : books)
{
@@ -340,8 +312,7 @@ doSubscribe(Context const& context)
{
auto const& jsonAccounts = request.at(JS(accounts_proposed));
if (!jsonAccounts.is_array())
return Status{
RippledError::rpcINVALID_PARAMS, "accountsProposedNotArray"};
return Status{RippledError::rpcINVALID_PARAMS, "accountsProposedNotArray"};
auto const& accounts = jsonAccounts.as_array();
if (accounts.empty())
@@ -357,27 +328,22 @@ doSubscribe(Context const& context)
if (request.contains(JS(books)))
{
auto parsed =
validateAndGetBooks(context.yield, request, context.backend);
auto parsed = validateAndGetBooks(context.yield, request, context.backend);
if (auto status = std::get_if<Status>(&parsed))
return *status;
auto [bks, snap] =
std::get<std::pair<std::vector<ripple::Book>, boost::json::array>>(
parsed);
auto [bks, snap] = std::get<std::pair<std::vector<ripple::Book>, boost::json::array>>(parsed);
books = std::move(bks);
response[JS(offers)] = std::move(snap);
}
if (request.contains(JS(streams)))
response = subscribeToStreams(
context.yield, request, context.session, *context.subscriptions);
response = subscribeToStreams(context.yield, request, context.session, *context.subscriptions);
if (request.contains(JS(accounts)))
subscribeToAccounts(request, context.session, *context.subscriptions);
if (request.contains(JS(accounts_proposed)))
subscribeToAccountsProposed(
request, context.session, *context.subscriptions);
subscribeToAccountsProposed(request, context.session, *context.subscriptions);
if (request.contains(JS(books)))
subscribeToBooks(books, context.session, *context.subscriptions);
@@ -420,8 +386,7 @@ doUnsubscribe(Context const& context)
{
auto const& jsonAccounts = request.at(JS(accounts_proposed));
if (!jsonAccounts.is_array())
return Status{
RippledError::rpcINVALID_PARAMS, "accountsProposedNotArray"};
return Status{RippledError::rpcINVALID_PARAMS, "accountsProposedNotArray"};
auto const& accounts = jsonAccounts.as_array();
if (accounts.empty())
@@ -435,15 +400,12 @@ doUnsubscribe(Context const& context)
std::vector<ripple::Book> books;
if (request.contains(JS(books)))
{
auto parsed =
validateAndGetBooks(context.yield, request, context.backend);
auto parsed = validateAndGetBooks(context.yield, request, context.backend);
if (auto status = std::get_if<Status>(&parsed))
return *status;
auto [bks, snap] =
std::get<std::pair<std::vector<ripple::Book>, boost::json::array>>(
parsed);
auto [bks, snap] = std::get<std::pair<std::vector<ripple::Book>, boost::json::array>>(parsed);
books = std::move(bks);
}
@@ -455,8 +417,7 @@ doUnsubscribe(Context const& context)
unsubscribeToAccounts(request, context.session, *context.subscriptions);
if (request.contains(JS(accounts_proposed)))
unsubscribeToAccountsProposed(
request, context.session, *context.subscriptions);
unsubscribeToAccountsProposed(request, context.session, *context.subscriptions);
if (request.contains("books"))
unsubscribeToBooks(books, context.session, *context.subscriptions);

5
src/rpc/handlers/TransactionEntry.cpp
View File

@@ -46,10 +46,7 @@ doTransactionEntry(Context const& context)
// ledger; we simulate that here by returning not found if the transaction
// is in a different ledger than the one specified.
if (!dbResponse || dbResponse->ledgerSequence != lgrInfo.seq)
return Status{
RippledError::rpcTXN_NOT_FOUND,
"transactionNotFound",
"Transaction not found."};
return Status{RippledError::rpcTXN_NOT_FOUND, "transactionNotFound", "Transaction not found."};
auto [txn, meta] = toExpandedJson(*dbResponse);
response[JS(tx_json)] = std::move(txn);

3
src/rpc/handlers/Tx.cpp
View File

@@ -71,8 +71,7 @@ doTx(Context const& context)
{
if (rangeSupplied)
{
bool searchedAll = range->maxSequence >= *maxLedger &&
range->minSequence <= *minLedger;
bool searchedAll = range->maxSequence >= *maxLedger && range->minSequence <= *minLedger;
boost::json::object extra;
extra["searched_all"] = searchedAll;
return Status{RippledError::rpcTXN_NOT_FOUND, std::move(extra)};

66
src/rpc/ngHandlers/AccountChannels.cpp
View File

@@ -23,16 +23,12 @@
namespace RPCng {
void
AccountChannelsHandler::addChannel(
std::vector<ChannelResponse>& jsonChannels,
ripple::SLE const& channelSle) const
AccountChannelsHandler::addChannel(std::vector<ChannelResponse>& jsonChannels, ripple::SLE const& channelSle) const
{
ChannelResponse channel;
channel.channelID = ripple::to_string(channelSle.key());
channel.account =
ripple::to_string(channelSle.getAccountID(ripple::sfAccount));
channel.accountDestination =
ripple::to_string(channelSle.getAccountID(ripple::sfDestination));
channel.account = ripple::to_string(channelSle.getAccountID(ripple::sfAccount));
channel.accountDestination = ripple::to_string(channelSle.getAccountID(ripple::sfDestination));
channel.amount = channelSle[ripple::sfAmount].getText();
channel.balance = channelSle[ripple::sfBalance].getText();
if (publicKeyType(channelSle[ripple::sfPublicKey]))
@@ -55,17 +51,11 @@ AccountChannelsHandler::addChannel(
}
AccountChannelsHandler::Result
AccountChannelsHandler::process(
AccountChannelsHandler::Input input,
Context const& ctx) const
AccountChannelsHandler::process(AccountChannelsHandler::Input input, Context const& ctx) const
{
auto const range = sharedPtrBackend_->fetchLedgerRange();
auto const lgrInfoOrStatus = RPC::getLedgerInfoFromHashOrSeq(
*sharedPtrBackend_,
ctx.yield,
input.ledgerHash,
input.ledgerIndex,
range->maxSequence);
*sharedPtrBackend_, ctx.yield, input.ledgerHash, input.ledgerIndex, range->maxSequence);
if (auto status = std::get_if<RPC::Status>(&lgrInfoOrStatus))
return Error{*status};
@@ -75,22 +65,18 @@ AccountChannelsHandler::process(
// no need to check the return value, validator check for us
auto const accountID = RPC::accountFromStringStrict(input.account);
auto const accountLedgerObject = sharedPtrBackend_->fetchLedgerObject(
ripple::keylet::account(*accountID).key, lgrInfo.seq, ctx.yield);
auto const accountLedgerObject =
sharedPtrBackend_->fetchLedgerObject(ripple::keylet::account(*accountID).key, lgrInfo.seq, ctx.yield);
if (!accountLedgerObject)
return Error{RPC::Status{
RPC::RippledError::rpcACT_NOT_FOUND, "accountNotFound"}};
return Error{RPC::Status{RPC::RippledError::rpcACT_NOT_FOUND, "accountNotFound"}};
auto const destAccountID = input.destinationAccount
? RPC::accountFromStringStrict(input.destinationAccount.value())
: std::optional<ripple::AccountID>{};
auto const destAccountID = input.destinationAccount ? RPC::accountFromStringStrict(input.destinationAccount.value())
: std::optional<ripple::AccountID>{};
Output response;
auto const addToResponse = [&](ripple::SLE&& sle) {
if (sle.getType() == ripple::ltPAYCHAN &&
sle.getAccountID(ripple::sfAccount) == accountID &&
(!destAccountID ||
*destAccountID == sle.getAccountID(ripple::sfDestination)))
if (sle.getType() == ripple::ltPAYCHAN && sle.getAccountID(ripple::sfAccount) == accountID &&
(!destAccountID || *destAccountID == sle.getAccountID(ripple::sfDestination)))
{
addChannel(response.channels, sle);
}
@@ -98,13 +84,7 @@ AccountChannelsHandler::process(
};
auto const next = RPC::ngTraverseOwnedNodes(
*sharedPtrBackend_,
*accountID,
lgrInfo.seq,
input.limit,
input.marker,
ctx.yield,
addToResponse);
*sharedPtrBackend_, *accountID, lgrInfo.seq, input.limit, input.marker, ctx.yield, addToResponse);
response.account = input.account;
response.limit = input.limit;
@@ -119,9 +99,7 @@ AccountChannelsHandler::process(
}
AccountChannelsHandler::Input
tag_invoke(
boost::json::value_to_tag<AccountChannelsHandler::Input>,
boost::json::value const& jv)
tag_invoke(boost::json::value_to_tag<AccountChannelsHandler::Input>, boost::json::value const& jv)
{
auto const& jsonObject = jv.as_object();
AccountChannelsHandler::Input input;
@@ -140,8 +118,7 @@ tag_invoke(
}
if (jsonObject.contains(JS(destination_account)))
{
input.destinationAccount =
jv.at(JS(destination_account)).as_string().c_str();
input.destinationAccount = jv.at(JS(destination_account)).as_string().c_str();
}
if (jsonObject.contains(JS(ledger_index)))
{
@@ -151,8 +128,7 @@ tag_invoke(
}
else if (jsonObject.at(JS(ledger_index)).as_string() != "validated")
{
input.ledgerIndex =
std::stoi(jv.at(JS(ledger_index)).as_string().c_str());
input.ledgerIndex = std::stoi(jv.at(JS(ledger_index)).as_string().c_str());
}
}
@@ -160,10 +136,7 @@ tag_invoke(
}
void
tag_invoke(
boost::json::value_from_tag,
boost::json::value& jv,
AccountChannelsHandler::Output const& output)
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, AccountChannelsHandler::Output const& output)
{
boost::json::object obj;
obj = {
@@ -179,10 +152,7 @@ tag_invoke(
}
void
tag_invoke(
boost::json::value_from_tag,
boost::json::value& jv,
AccountChannelsHandler::ChannelResponse const& channel)
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, AccountChannelsHandler::ChannelResponse const& channel)
{
boost::json::object obj;
obj = {

26
src/rpc/ngHandlers/AccountChannels.h
View File

@@ -74,8 +74,7 @@ public:
using Result = RPCng::HandlerReturnType<Output>;
AccountChannelsHandler(
std::shared_ptr<BackendInterface> const& sharedPtrBackend)
AccountChannelsHandler(std::shared_ptr<BackendInterface> const& sharedPtrBackend)
: sharedPtrBackend_(sharedPtrBackend)
{
}
@@ -83,16 +82,14 @@ public:
RpcSpecConstRef
spec() const
{
// clang-format off
static auto const rpcSpec = RpcSpec{
static auto const rpcSpec = RpcSpec{
{JS(account), validation::Required{}, validation::AccountValidator},
{JS(destination_account), validation::Type<std::string>{},validation::AccountValidator},
{JS(destination_account), validation::Type<std::string>{}, validation::AccountValidator},
{JS(ledger_hash), validation::Uint256HexStringValidator},
{JS(limit), validation::Type<uint32_t>{},validation::Between{10,400}},
{JS(limit), validation::Type<uint32_t>{}, validation::Between{10, 400}},
{JS(ledger_index), validation::LedgerIndexValidator},
{JS(marker), validation::AccountMarkerValidator}
{JS(marker), validation::AccountMarkerValidator},
};
// clang-format on
return rpcSpec;
}
@@ -102,22 +99,15 @@ public:
private:
void
addChannel(std::vector<ChannelResponse>& jsonLines, ripple::SLE const& line)
const;
addChannel(std::vector<ChannelResponse>& jsonLines, ripple::SLE const& line) const;
friend void
tag_invoke(
boost::json::value_from_tag,
boost::json::value& jv,
Output const& output);
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, Output const& output);
friend Input
tag_invoke(boost::json::value_to_tag<Input>, boost::json::value const& jv);
friend void
tag_invoke(
boost::json::value_from_tag,
boost::json::value& jv,
ChannelResponse const& channel);
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, ChannelResponse const& channel);
};
} // namespace RPCng

35
src/rpc/ngHandlers/AccountCurrencies.cpp
View File

@@ -21,17 +21,11 @@
namespace RPCng {
AccountCurrenciesHandler::Result
AccountCurrenciesHandler::process(
AccountCurrenciesHandler::Input input,
Context const& ctx) const
AccountCurrenciesHandler::process(AccountCurrenciesHandler::Input input, Context const& ctx) const
{
auto const range = sharedPtrBackend_->fetchLedgerRange();
auto const lgrInfoOrStatus = RPC::getLedgerInfoFromHashOrSeq(
*sharedPtrBackend_,
ctx.yield,
input.ledgerHash,
input.ledgerIndex,
range->maxSequence);
*sharedPtrBackend_, ctx.yield, input.ledgerHash, input.ledgerIndex, range->maxSequence);
if (auto const status = std::get_if<RPC::Status>(&lgrInfoOrStatus))
return Error{*status};
@@ -40,11 +34,10 @@ AccountCurrenciesHandler::process(
auto const accountID = RPC::accountFromStringStrict(input.account);
auto const accountLedgerObject = sharedPtrBackend_->fetchLedgerObject(
ripple::keylet::account(*accountID).key, lgrInfo.seq, ctx.yield);
auto const accountLedgerObject =
sharedPtrBackend_->fetchLedgerObject(ripple::keylet::account(*accountID).key, lgrInfo.seq, ctx.yield);
if (!accountLedgerObject)
return Error{RPC::Status{
RPC::RippledError::rpcACT_NOT_FOUND, "accountNotFound"}};
return Error{RPC::Status{RPC::RippledError::rpcACT_NOT_FOUND, "accountNotFound"}};
Output response;
auto const addToResponse = [&](ripple::SLE&& sle) {
@@ -59,11 +52,9 @@ AccountCurrenciesHandler::process(
if (!viewLowest)
balance.negate();
if (balance < lineLimit)
response.receiveCurrencies.insert(
ripple::to_string(balance.getCurrency()));
response.receiveCurrencies.insert(ripple::to_string(balance.getCurrency()));
if ((-balance) < lineLimitPeer)
response.sendCurrencies.insert(
ripple::to_string(balance.getCurrency()));
response.sendCurrencies.insert(ripple::to_string(balance.getCurrency()));
}
return true;
};
@@ -84,10 +75,7 @@ AccountCurrenciesHandler::process(
}
void
tag_invoke(
boost::json::value_from_tag,
boost::json::value& jv,
AccountCurrenciesHandler::Output const& output)
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, AccountCurrenciesHandler::Output const& output)
{
jv = {
{JS(ledger_hash), output.ledgerHash},
@@ -98,9 +86,7 @@ tag_invoke(
}
AccountCurrenciesHandler::Input
tag_invoke(
boost::json::value_to_tag<AccountCurrenciesHandler::Input>,
boost::json::value const& jv)
tag_invoke(boost::json::value_to_tag<AccountCurrenciesHandler::Input>, boost::json::value const& jv)
{
auto const& jsonObject = jv.as_object();
AccountCurrenciesHandler::Input input;
@@ -117,8 +103,7 @@ tag_invoke(
}
else if (jsonObject.at(JS(ledger_index)).as_string() != "validated")
{
input.ledgerIndex =
std::stoi(jv.at(JS(ledger_index)).as_string().c_str());
input.ledgerIndex = std::stoi(jv.at(JS(ledger_index)).as_string().c_str());
}
}
return input;

12
src/rpc/ngHandlers/AccountCurrencies.h
View File

@@ -53,8 +53,7 @@ public:
using Result = RPCng::HandlerReturnType<Output>;
AccountCurrenciesHandler(
std::shared_ptr<BackendInterface> const& sharedPtrBackend)
AccountCurrenciesHandler(std::shared_ptr<BackendInterface> const& sharedPtrBackend)
: sharedPtrBackend_(sharedPtrBackend)
{
}
@@ -65,7 +64,9 @@ public:
static auto const rpcSpec = RpcSpec{
{JS(account), validation::Required{}, validation::AccountValidator},
{JS(ledger_hash), validation::Uint256HexStringValidator},
{JS(ledger_index), validation::LedgerIndexValidator}};
{JS(ledger_index), validation::LedgerIndexValidator},
};
return rpcSpec;
}
@@ -74,10 +75,7 @@ public:
private:
friend void
tag_invoke(
boost::json::value_from_tag,
boost::json::value& jv,
Output const& output);
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, Output const& output);
friend Input
tag_invoke(boost::json::value_to_tag<Input>, boost::json::value const& jv);

46
src/rpc/ngHandlers/AccountInfo.cpp
View File

@@ -21,19 +21,14 @@
namespace RPCng {
AccountInfoHandler::Result
AccountInfoHandler::process(AccountInfoHandler::Input input, Context const& ctx)
const
AccountInfoHandler::process(AccountInfoHandler::Input input, Context const& ctx) const
{
if (!input.account && !input.ident)
return Error{RPC::Status{RPC::RippledError::rpcACT_MALFORMED}};
auto const range = sharedPtrBackend_->fetchLedgerRange();
auto const lgrInfoOrStatus = RPC::getLedgerInfoFromHashOrSeq(
*sharedPtrBackend_,
ctx.yield,
input.ledgerHash,
input.ledgerIndex,
range->maxSequence);
*sharedPtrBackend_, ctx.yield, input.ledgerHash, input.ledgerIndex, range->maxSequence);
if (auto const status = std::get_if<RPC::Status>(&lgrInfoOrStatus))
return Error{*status};
@@ -43,16 +38,12 @@ AccountInfoHandler::process(AccountInfoHandler::Input input, Context const& ctx)
auto const accountStr = input.account.value_or(input.ident.value_or(""));
auto const accountID = RPC::accountFromStringStrict(accountStr);
auto const accountKeylet = ripple::keylet::account(*accountID);
auto const accountLedgerObject = sharedPtrBackend_->fetchLedgerObject(
accountKeylet.key, lgrInfo.seq, ctx.yield);
auto const accountLedgerObject = sharedPtrBackend_->fetchLedgerObject(accountKeylet.key, lgrInfo.seq, ctx.yield);
if (!accountLedgerObject)
return Error{RPC::Status{
RPC::RippledError::rpcACT_NOT_FOUND, "accountNotFound"}};
return Error{RPC::Status{RPC::RippledError::rpcACT_NOT_FOUND, "accountNotFound"}};
ripple::STLedgerEntry const sle{
ripple::SerialIter{
accountLedgerObject->data(), accountLedgerObject->size()},
accountKeylet.key};
ripple::SerialIter{accountLedgerObject->data(), accountLedgerObject->size()}, accountKeylet.key};
if (!accountKeylet.check(sle))
return Error{RPC::Status{RPC::RippledError::rpcDB_DESERIALIZATION}};
// Return SignerList(s) if that is requested.
@@ -63,31 +54,24 @@ AccountInfoHandler::process(AccountInfoHandler::Input input, Context const& ctx)
auto const signersKey = ripple::keylet::signers(*accountID);
// This code will need to be revisited if in the future we
// support multiple SignerLists on one account.
auto const signers = sharedPtrBackend_->fetchLedgerObject(
signersKey.key, lgrInfo.seq, ctx.yield);
auto const signers = sharedPtrBackend_->fetchLedgerObject(signersKey.key, lgrInfo.seq, ctx.yield);
std::vector<ripple::STLedgerEntry> signerList;
if (signers)
{
ripple::STLedgerEntry const sleSigners{
ripple::SerialIter{signers->data(), signers->size()},
signersKey.key};
ripple::SerialIter{signers->data(), signers->size()}, signersKey.key};
if (!signersKey.check(sleSigners))
return Error{
RPC::Status{RPC::RippledError::rpcDB_DESERIALIZATION}};
return Error{RPC::Status{RPC::RippledError::rpcDB_DESERIALIZATION}};
signerList.push_back(sleSigners);
}
return Output(
lgrInfo.seq, ripple::strHex(lgrInfo.hash), sle, signerList);
return Output(lgrInfo.seq, ripple::strHex(lgrInfo.hash), sle, signerList);
}
return Output(lgrInfo.seq, ripple::strHex(lgrInfo.hash), sle);
}
void
tag_invoke(
boost::json::value_from_tag,
boost::json::value& jv,
AccountInfoHandler::Output const& output)
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, AccountInfoHandler::Output const& output)
{
jv = boost::json::object{
{JS(account_data), RPC::toJson(output.accountData)},
@@ -97,15 +81,12 @@ tag_invoke(
{
jv.as_object()[JS(signer_lists)] = boost::json::array();
for (auto const& signerList : output.signerLists.value())
jv.as_object()[JS(signer_lists)].as_array().push_back(
RPC::toJson(signerList));
jv.as_object()[JS(signer_lists)].as_array().push_back(RPC::toJson(signerList));
}
}
AccountInfoHandler::Input
tag_invoke(
boost::json::value_to_tag<AccountInfoHandler::Input>,
boost::json::value const& jv)
tag_invoke(boost::json::value_to_tag<AccountInfoHandler::Input>, boost::json::value const& jv)
{
auto const& jsonObject = jv.as_object();
AccountInfoHandler::Input input;
@@ -129,8 +110,7 @@ tag_invoke(
}
else if (jsonObject.at(JS(ledger_index)).as_string() != "validated")
{
input.ledgerIndex =
std::stoi(jsonObject.at(JS(ledger_index)).as_string().c_str());
input.ledgerIndex = std::stoi(jsonObject.at(JS(ledger_index)).as_string().c_str());
}
}
if (jsonObject.contains(JS(signer_lists)))

22
src/rpc/ngHandlers/AccountInfo.h
View File

@@ -51,13 +51,8 @@ public:
{
}
Output(
uint32_t ledgerId,
std::string ledgerHash,
ripple::STLedgerEntry sle)
: ledgerIndex(ledgerId)
, ledgerHash(std::move(ledgerHash))
, accountData(std::move(sle))
Output(uint32_t ledgerId, std::string ledgerHash, ripple::STLedgerEntry sle)
: ledgerIndex(ledgerId), ledgerHash(std::move(ledgerHash)), accountData(std::move(sle))
{
}
};
@@ -75,9 +70,7 @@ public:
using Result = RPCng::HandlerReturnType<Output>;
AccountInfoHandler(
std::shared_ptr<BackendInterface> const& sharedPtrBackend)
: sharedPtrBackend_(sharedPtrBackend)
AccountInfoHandler(std::shared_ptr<BackendInterface> const& sharedPtrBackend) : sharedPtrBackend_(sharedPtrBackend)
{
}
@@ -89,7 +82,9 @@ public:
{JS(ident), validation::AccountValidator},
{JS(ledger_hash), validation::Uint256HexStringValidator},
{JS(ledger_index), validation::LedgerIndexValidator},
{JS(signer_lists), validation::Type<bool>{}}};
{JS(signer_lists), validation::Type<bool>{}},
};
return rpcSpec;
}
@@ -98,10 +93,7 @@ public:
private:
friend void
tag_invoke(
boost::json::value_from_tag,
boost::json::value& jv,
Output const& output);
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, Output const& output);
friend Input
tag_invoke(boost::json::value_to_tag<Input>, boost::json::value const& jv);

76
src/rpc/ngHandlers/AccountLines.cpp
View File

@@ -53,18 +53,12 @@ AccountLinesHandler::addLine(
if (not viewLowest)
balance.negate();
bool const lineAuth =
flags & (viewLowest ? ripple::lsfLowAuth : ripple::lsfHighAuth);
bool const lineAuthPeer =
flags & (not viewLowest ? ripple::lsfLowAuth : ripple::lsfHighAuth);
bool const lineNoRipple =
flags & (viewLowest ? ripple::lsfLowNoRipple : ripple::lsfHighNoRipple);
bool const lineNoRipplePeer = flags &
(not viewLowest ? ripple::lsfLowNoRipple : ripple::lsfHighNoRipple);
bool const lineFreeze =
flags & (viewLowest ? ripple::lsfLowFreeze : ripple::lsfHighFreeze);
bool const lineFreezePeer =
flags & (not viewLowest ? ripple::lsfLowFreeze : ripple::lsfHighFreeze);
bool const lineAuth = flags & (viewLowest ? ripple::lsfLowAuth : ripple::lsfHighAuth);
bool const lineAuthPeer = flags & (not viewLowest ? ripple::lsfLowAuth : ripple::lsfHighAuth);
bool const lineNoRipple = flags & (viewLowest ? ripple::lsfLowNoRipple : ripple::lsfHighNoRipple);
bool const lineNoRipplePeer = flags & (not viewLowest ? ripple::lsfLowNoRipple : ripple::lsfHighNoRipple);
bool const lineFreeze = flags & (viewLowest ? ripple::lsfLowFreeze : ripple::lsfHighFreeze);
bool const lineFreezePeer = flags & (not viewLowest ? ripple::lsfLowFreeze : ripple::lsfHighFreeze);
ripple::STAmount const& saBalance = balance;
ripple::STAmount const& saLimit = lineLimit;
@@ -93,33 +87,25 @@ AccountLinesHandler::addLine(
}
AccountLinesHandler::Result
AccountLinesHandler::process(
AccountLinesHandler::Input input,
Context const& ctx) const
AccountLinesHandler::process(AccountLinesHandler::Input input, Context const& ctx) const
{
auto const range = sharedPtrBackend_->fetchLedgerRange();
auto const lgrInfoOrStatus = RPC::getLedgerInfoFromHashOrSeq(
*sharedPtrBackend_,
ctx.yield,
input.ledgerHash,
input.ledgerIndex,
range->maxSequence);
*sharedPtrBackend_, ctx.yield, input.ledgerHash, input.ledgerIndex, range->maxSequence);
if (auto status = std::get_if<RPC::Status>(&lgrInfoOrStatus))
return Error{*status};
auto const lgrInfo = std::get<ripple::LedgerInfo>(lgrInfoOrStatus);
auto const accountID = RPC::accountFromStringStrict(input.account);
auto const accountLedgerObject = sharedPtrBackend_->fetchLedgerObject(
ripple::keylet::account(*accountID).key, lgrInfo.seq, ctx.yield);
auto const accountLedgerObject =
sharedPtrBackend_->fetchLedgerObject(ripple::keylet::account(*accountID).key, lgrInfo.seq, ctx.yield);
if (not accountLedgerObject)
return Error{RPC::Status{
RPC::RippledError::rpcACT_NOT_FOUND, "accountNotFound"}};
return Error{RPC::Status{RPC::RippledError::rpcACT_NOT_FOUND, "accountNotFound"}};
auto const peerAccountID = input.peer
? RPC::accountFromStringStrict(*(input.peer))
: std::optional<ripple::AccountID>{};
auto const peerAccountID =
input.peer ? RPC::accountFromStringStrict(*(input.peer)) : std::optional<ripple::AccountID>{};
Output response;
response.lines.reserve(input.limit);
@@ -130,18 +116,13 @@ AccountLinesHandler::process(
auto ignore = false;
if (input.ignoreDefault)
{
if (sle.getFieldAmount(ripple::sfLowLimit).getIssuer() ==
accountID)
if (sle.getFieldAmount(ripple::sfLowLimit).getIssuer() == accountID)
{
ignore =
!(sle.getFieldU32(ripple::sfFlags) &
ripple::lsfLowReserve);
ignore = !(sle.getFieldU32(ripple::sfFlags) & ripple::lsfLowReserve);
}
else
{
ignore =
!(sle.getFieldU32(ripple::sfFlags) &
ripple::lsfHighReserve);
ignore = !(sle.getFieldU32(ripple::sfFlags) & ripple::lsfHighReserve);
}
}
@@ -151,18 +132,11 @@ AccountLinesHandler::process(
};
auto const next = RPC::ngTraverseOwnedNodes(
*sharedPtrBackend_,
*accountID,
lgrInfo.seq,
input.limit,
input.marker,
ctx.yield,
addToResponse);
*sharedPtrBackend_, *accountID, lgrInfo.seq, input.limit, input.marker, ctx.yield, addToResponse);
response.account = input.account;
response.limit =
input.limit; // not documented,
// https://github.com/XRPLF/xrpl-dev-portal/issues/1838
response.limit = input.limit; // not documented,
// https://github.com/XRPLF/xrpl-dev-portal/issues/1838
response.ledgerHash = ripple::strHex(lgrInfo.hash);
response.ledgerIndex = lgrInfo.seq;
@@ -174,9 +148,7 @@ AccountLinesHandler::process(
}
AccountLinesHandler::Input
tag_invoke(
boost::json::value_to_tag<AccountLinesHandler::Input>,
boost::json::value const& jv)
tag_invoke(boost::json::value_to_tag<AccountLinesHandler::Input>, boost::json::value const& jv)
{
auto const& jsonObject = jv.as_object();
AccountLinesHandler::Input input;
@@ -210,8 +182,7 @@ tag_invoke(
}
else if (jsonObject.at(JS(ledger_index)).as_string() != "validated")
{
input.ledgerIndex =
std::stoi(jv.at(JS(ledger_index)).as_string().c_str());
input.ledgerIndex = std::stoi(jv.at(JS(ledger_index)).as_string().c_str());
}
}
@@ -219,10 +190,7 @@ tag_invoke(
}
void
tag_invoke(
boost::json::value_from_tag,
boost::json::value& jv,
AccountLinesHandler::Output const& output)
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, AccountLinesHandler::Output const& output)
{
auto obj = boost::json::object{
{JS(ledger_hash), output.ledgerHash},

21
src/rpc/ngHandlers/AccountLines.h
View File

@@ -68,25 +68,21 @@ public:
std::optional<std::string> ledgerHash;
std::optional<uint32_t> ledgerIndex;
std::optional<std::string> peer;
bool ignoreDefault =
false; // TODO: document
// https://github.com/XRPLF/xrpl-dev-portal/issues/1839
bool ignoreDefault = false; // TODO: document
// https://github.com/XRPLF/xrpl-dev-portal/issues/1839
uint32_t limit = 50;
std::optional<std::string> marker;
};
using Result = RPCng::HandlerReturnType<Output>;
AccountLinesHandler(
std::shared_ptr<BackendInterface> const& sharedPtrBackend)
: sharedPtrBackend_(sharedPtrBackend)
AccountLinesHandler(std::shared_ptr<BackendInterface> const& sharedPtrBackend) : sharedPtrBackend_(sharedPtrBackend)
{
}
RpcSpecConstRef
spec() const
{
// clang-format off
static auto const rpcSpec = RpcSpec{
{JS(account), validation::Required{}, validation::AccountValidator},
{JS(peer), validation::Type<std::string>{}, validation::AccountValidator},
@@ -96,7 +92,6 @@ public:
{JS(ledger_index), validation::LedgerIndexValidator},
{JS(marker), validation::AccountMarkerValidator},
};
// clang-format on
return rpcSpec;
}
@@ -114,18 +109,12 @@ private:
private:
friend void
tag_invoke(
boost::json::value_from_tag,
boost::json::value& jv,
Output const& output);
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, Output const& output);
friend Input
tag_invoke(boost::json::value_to_tag<Input>, boost::json::value const& jv);
friend void
tag_invoke(
boost::json::value_from_tag,
boost::json::value& jv,
LineResponse const& line);
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, LineResponse const& line);
};
} // namespace RPCng

57
src/rpc/ngHandlers/AccountOffers.cpp
View File

@@ -22,9 +22,7 @@
namespace RPCng {
void
AccountOffersHandler::addOffer(
std::vector<Offer>& offers,
ripple::SLE const& offerSle) const
AccountOffersHandler::addOffer(std::vector<Offer>& offers, ripple::SLE const& offerSle) const
{
AccountOffersHandler::Offer offer;
offer.takerPays = offerSle.getFieldAmount(ripple::sfTakerPays);
@@ -32,8 +30,7 @@ AccountOffersHandler::addOffer(
offer.seq = offerSle.getFieldU32(ripple::sfSequence);
offer.flags = offerSle.getFieldU32(ripple::sfFlags);
auto const quality =
getQuality(offerSle.getFieldH256(ripple::sfBookDirectory));
auto const quality = getQuality(offerSle.getFieldH256(ripple::sfBookDirectory));
ripple::STAmount const rate = ripple::amountFromQuality(quality);
offer.quality = rate.getText();
if (offerSle.isFieldPresent(ripple::sfExpiration))
@@ -42,17 +39,11 @@ AccountOffersHandler::addOffer(
};
AccountOffersHandler::Result
AccountOffersHandler::process(
AccountOffersHandler::Input input,
Context const& ctx) const
AccountOffersHandler::process(AccountOffersHandler::Input input, Context const& ctx) const
{
auto const range = sharedPtrBackend_->fetchLedgerRange();
auto const lgrInfoOrStatus = RPC::getLedgerInfoFromHashOrSeq(
*sharedPtrBackend_,
ctx.yield,
input.ledgerHash,
input.ledgerIndex,
range->maxSequence);
*sharedPtrBackend_, ctx.yield, input.ledgerHash, input.ledgerIndex, range->maxSequence);
if (auto const status = std::get_if<RPC::Status>(&lgrInfoOrStatus))
return Error{*status};
@@ -61,11 +52,10 @@ AccountOffersHandler::process(
auto const accountID = RPC::accountFromStringStrict(input.account);
auto const accountLedgerObject = sharedPtrBackend_->fetchLedgerObject(
ripple::keylet::account(*accountID).key, lgrInfo.seq, ctx.yield);
auto const accountLedgerObject =
sharedPtrBackend_->fetchLedgerObject(ripple::keylet::account(*accountID).key, lgrInfo.seq, ctx.yield);
if (!accountLedgerObject)
return Error{RPC::Status{
RPC::RippledError::rpcACT_NOT_FOUND, "accountNotFound"}};
return Error{RPC::Status{RPC::RippledError::rpcACT_NOT_FOUND, "accountNotFound"}};
Output response;
response.account = ripple::to_string(*accountID);
@@ -82,13 +72,7 @@ AccountOffersHandler::process(
};
auto const next = RPC::ngTraverseOwnedNodes(
*sharedPtrBackend_,
*accountID,
lgrInfo.seq,
input.limit,
input.marker,
ctx.yield,
addToResponse);
*sharedPtrBackend_, *accountID, lgrInfo.seq, input.limit, input.marker, ctx.yield, addToResponse);
if (auto const status = std::get_if<RPC::Status>(&next))
return Error{*status};
@@ -102,10 +86,7 @@ AccountOffersHandler::process(
}
void
tag_invoke(
boost::json::value_from_tag,
boost::json::value& jv,
AccountOffersHandler::Output const& output)
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, AccountOffersHandler::Output const& output)
{
jv = {
{JS(ledger_hash), output.ledgerHash},
@@ -118,21 +99,14 @@ tag_invoke(
}
void
tag_invoke(
boost::json::value_from_tag,
boost::json::value& jv,
AccountOffersHandler::Offer const& offer)
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, AccountOffersHandler::Offer const& offer)
{
jv = {
{JS(seq), offer.seq},
{JS(flags), offer.flags},
{JS(quality), offer.quality}};
jv = {{JS(seq), offer.seq}, {JS(flags), offer.flags}, {JS(quality), offer.quality}};
auto& jsonObject = jv.as_object();
if (offer.expiration)
jsonObject[JS(expiration)] = *offer.expiration;
auto const convertAmount = [&](const char* field,
ripple::STAmount const& amount) {
auto const convertAmount = [&](const char* field, ripple::STAmount const& amount) {
if (amount.native())
{
jsonObject[field] = amount.getText();
@@ -150,9 +124,7 @@ tag_invoke(
}
AccountOffersHandler::Input
tag_invoke(
boost::json::value_to_tag<AccountOffersHandler::Input>,
boost::json::value const& jv)
tag_invoke(boost::json::value_to_tag<AccountOffersHandler::Input>, boost::json::value const& jv)
{
auto const& jsonObject = jv.as_object();
AccountOffersHandler::Input input;
@@ -169,8 +141,7 @@ tag_invoke(
}
else if (jsonObject.at(JS(ledger_index)).as_string() != "validated")
{
input.ledgerIndex =
std::stoi(jsonObject.at(JS(ledger_index)).as_string().c_str());
input.ledgerIndex = std::stoi(jsonObject.at(JS(ledger_index)).as_string().c_str());
}
}
if (jsonObject.contains(JS(limit)))

19
src/rpc/ngHandlers/AccountOffers.h
View File

@@ -63,8 +63,7 @@ public:
using Result = RPCng::HandlerReturnType<Output>;
AccountOffersHandler(
std::shared_ptr<BackendInterface> const& sharedPtrBackend)
AccountOffersHandler(std::shared_ptr<BackendInterface> const& sharedPtrBackend)
: sharedPtrBackend_(sharedPtrBackend)
{
}
@@ -77,9 +76,9 @@ public:
{JS(ledger_hash), validation::Uint256HexStringValidator},
{JS(ledger_index), validation::LedgerIndexValidator},
{JS(marker), validation::AccountMarkerValidator},
{JS(limit),
validation::Type<uint32_t>{},
validation::Between{10, 400}}};
{JS(limit), validation::Type<uint32_t>{}, validation::Between{10, 400}},
};
return rpcSpec;
}
@@ -91,18 +90,12 @@ private:
addOffer(std::vector<Offer>& offers, ripple::SLE const& offerSle) const;
friend void
tag_invoke(
boost::json::value_from_tag,
boost::json::value& jv,
Output const& output);
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, Output const& output);
friend Input
tag_invoke(boost::json::value_to_tag<Input>, boost::json::value const& jv);
friend void
tag_invoke(
boost::json::value_from_tag,
boost::json::value& jv,
Offer const& offer);
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, Offer const& offer);
};
} // namespace RPCng

65
src/rpc/ngHandlers/AccountTx.cpp
View File

@@ -25,51 +25,37 @@ namespace RPCng {
// TODO: this is currently very similar to nft_history but its own copy for time
// being. we should aim to reuse common logic in some way in the future.
AccountTxHandler::Result
AccountTxHandler::process(AccountTxHandler::Input input, Context const& ctx)
const
AccountTxHandler::process(AccountTxHandler::Input input, Context const& ctx) const
{
auto const range = sharedPtrBackend_->fetchLedgerRange();
auto [minIndex, maxIndex] = *range;
if (input.ledgerIndexMin)
{
if (range->maxSequence < input.ledgerIndexMin ||
range->minSequence > input.ledgerIndexMin)
if (range->maxSequence < input.ledgerIndexMin || range->minSequence > input.ledgerIndexMin)
{
return Error{RPC::Status{
RPC::RippledError::rpcLGR_IDX_MALFORMED,
"ledgerSeqMinOutOfRange"}};
return Error{RPC::Status{RPC::RippledError::rpcLGR_IDX_MALFORMED, "ledgerSeqMinOutOfRange"}};
}
minIndex = *input.ledgerIndexMin;
}
if (input.ledgerIndexMax)
{
if (range->maxSequence < input.ledgerIndexMax ||
range->minSequence > input.ledgerIndexMax)
return Error{RPC::Status{
RPC::RippledError::rpcLGR_IDX_MALFORMED,
"ledgerSeqMaxOutOfRange"}};
if (range->maxSequence < input.ledgerIndexMax || range->minSequence > input.ledgerIndexMax)
return Error{RPC::Status{RPC::RippledError::rpcLGR_IDX_MALFORMED, "ledgerSeqMaxOutOfRange"}};
maxIndex = *input.ledgerIndexMax;
}
if (minIndex > maxIndex)
return Error{
RPC::Status{RPC::RippledError::rpcINVALID_PARAMS, "invalidIndex"}};
return Error{RPC::Status{RPC::RippledError::rpcINVALID_PARAMS, "invalidIndex"}};
if (input.ledgerHash || input.ledgerIndex)
{
// rippled does not have this check
if (input.ledgerIndexMax || input.ledgerIndexMin)
return Error{RPC::Status{
RPC::RippledError::rpcINVALID_PARAMS,
"containsLedgerSpecifierAndRange"}};
return Error{RPC::Status{RPC::RippledError::rpcINVALID_PARAMS, "containsLedgerSpecifierAndRange"}};
auto const lgrInfoOrStatus = RPC::getLedgerInfoFromHashOrSeq(
*sharedPtrBackend_,
ctx.yield,
input.ledgerHash,
input.ledgerIndex,
range->maxSequence);
*sharedPtrBackend_, ctx.yield, input.ledgerHash, input.ledgerIndex, range->maxSequence);
if (auto status = std::get_if<RPC::Status>(&lgrInfoOrStatus))
return Error{*status};
@@ -96,17 +82,14 @@ AccountTxHandler::process(AccountTxHandler::Input input, Context const& ctx)
auto const limit = input.limit.value_or(limitDefault);
auto const accountID = RPC::accountFromStringStrict(input.account);
auto const [txnsAndCursor, timeDiff] = util::timed([&]() {
return sharedPtrBackend_->fetchAccountTransactions(
*accountID, limit, input.forward, cursor, ctx.yield);
return sharedPtrBackend_->fetchAccountTransactions(*accountID, limit, input.forward, cursor, ctx.yield);
});
log_.info() << "db fetch took " << timeDiff
<< " milliseconds - num blobs = " << txnsAndCursor.txns.size();
log_.info() << "db fetch took " << timeDiff << " milliseconds - num blobs = " << txnsAndCursor.txns.size();
auto const [blobs, retCursor] = txnsAndCursor;
Output response;
if (retCursor)
response.marker = {
retCursor->ledgerSequence, retCursor->transactionIndex};
response.marker = {retCursor->ledgerSequence, retCursor->transactionIndex};
for (auto const& txnPlusMeta : blobs)
{
@@ -129,8 +112,7 @@ AccountTxHandler::process(AccountTxHandler::Input input, Context const& ctx)
auto [txn, meta] = RPC::toExpandedJson(txnPlusMeta);
obj[JS(meta)] = std::move(meta);
obj[JS(tx)] = std::move(txn);
obj[JS(tx)].as_object()[JS(ledger_index)] =
txnPlusMeta.ledgerSequence;
obj[JS(tx)].as_object()[JS(ledger_index)] = txnPlusMeta.ledgerSequence;
obj[JS(tx)].as_object()[JS(date)] = txnPlusMeta.date;
}
else
@@ -155,10 +137,7 @@ AccountTxHandler::process(AccountTxHandler::Input input, Context const& ctx)
}
void
tag_invoke(
boost::json::value_from_tag,
boost::json::value& jv,
AccountTxHandler::Output const& output)
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, AccountTxHandler::Output const& output)
{
jv = {
{JS(account), output.account},
@@ -173,29 +152,22 @@ tag_invoke(
}
void
tag_invoke(
boost::json::value_from_tag,
boost::json::value& jv,
AccountTxHandler::Marker const& marker)
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, AccountTxHandler::Marker const& marker)
{
jv = {{JS(ledger), marker.ledger}, {JS(seq), marker.seq}};
}
AccountTxHandler::Input
tag_invoke(
boost::json::value_to_tag<AccountTxHandler::Input>,
boost::json::value const& jv)
tag_invoke(boost::json::value_to_tag<AccountTxHandler::Input>, boost::json::value const& jv)
{
auto const& jsonObject = jv.as_object();
AccountTxHandler::Input input;
input.account = jsonObject.at(JS(account)).as_string().c_str();
if (jsonObject.contains(JS(ledger_index_min)) &&
jsonObject.at(JS(ledger_index_min)).as_int64() != -1)
if (jsonObject.contains(JS(ledger_index_min)) && jsonObject.at(JS(ledger_index_min)).as_int64() != -1)
{
input.ledgerIndexMin = jsonObject.at(JS(ledger_index_min)).as_int64();
}
if (jsonObject.contains(JS(ledger_index_max)) &&
jsonObject.at(JS(ledger_index_max)).as_int64() != -1)
if (jsonObject.contains(JS(ledger_index_max)) && jsonObject.at(JS(ledger_index_max)).as_int64() != -1)
{
input.ledgerIndexMax = jsonObject.at(JS(ledger_index_max)).as_int64();
}
@@ -211,8 +183,7 @@ tag_invoke(
}
else if (jsonObject.at(JS(ledger_index)).as_string() != "validated")
{
input.ledgerIndex =
std::stoi(jsonObject.at(JS(ledger_index)).as_string().c_str());
input.ledgerIndex = std::stoi(jsonObject.at(JS(ledger_index)).as_string().c_str());
}
}
if (jsonObject.contains(JS(binary)))

32
src/rpc/ngHandlers/AccountTx.h
View File

@@ -69,8 +69,7 @@ public:
using Result = RPCng::HandlerReturnType<Output>;
AccountTxHandler(std::shared_ptr<BackendInterface> const& sharedPtrBackend)
: sharedPtrBackend_(sharedPtrBackend)
AccountTxHandler(std::shared_ptr<BackendInterface> const& sharedPtrBackend) : sharedPtrBackend_(sharedPtrBackend)
{
}
@@ -85,22 +84,17 @@ public:
{JS(ledger_index_max), validation::Type<int32_t>{}},
{JS(binary), validation::Type<bool>{}},
{JS(forward), validation::Type<bool>{}},
{JS(limit),
validation::Type<uint32_t>{},
validation::Between{1, 100}},
{JS(limit), validation::Type<uint32_t>{}, validation::Between{1, 100}},
{JS(marker),
validation::WithCustomError{
validation::Type<boost::json::object>{},
RPC::Status{
RPC::RippledError::rpcINVALID_PARAMS, "invalidMarker"}},
RPC::Status{RPC::RippledError::rpcINVALID_PARAMS, "invalidMarker"}},
validation::Section{
{JS(ledger),
validation::Required{},
validation::Type<uint32_t>{}},
{JS(seq),
validation::Required{},
validation::Type<uint32_t>{}},
}}};
{JS(ledger), validation::Required{}, validation::Type<uint32_t>{}},
{JS(seq), validation::Required{}, validation::Type<uint32_t>{}},
}},
};
return rpcSpec;
}
@@ -109,18 +103,12 @@ public:
private:
friend void
tag_invoke(
boost::json::value_from_tag,
boost::json::value& jv,
Output const& output);
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, Output const& output);
friend Input
tag_invoke(boost::json::value_to_tag<Input>, boost::json::value const& jv);
friend void
tag_invoke(
boost::json::value_from_tag,
boost::json::value& jv,
Marker const& marker);
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, Marker const& marker);
};
} // namespace RPCng

58
src/rpc/ngHandlers/BookOffers.cpp
View File

@@ -25,19 +25,14 @@ namespace RPCng {
BookOffersHandler::Result
BookOffersHandler::process(Input input, Context const& ctx) const
{
auto bookMaybe = RPC::parseBook(
input.paysCurrency, input.paysID, input.getsCurrency, input.getsID);
auto bookMaybe = RPC::parseBook(input.paysCurrency, input.paysID, input.getsCurrency, input.getsID);
if (auto const status = std::get_if<RPC::Status>(&bookMaybe))
return Error{*status};
// check ledger
auto const range = sharedPtrBackend_->fetchLedgerRange();
auto const lgrInfoOrStatus = RPC::getLedgerInfoFromHashOrSeq(
*sharedPtrBackend_,
ctx.yield,
input.ledgerHash,
input.ledgerIndex,
range->maxSequence);
*sharedPtrBackend_, ctx.yield, input.ledgerHash, input.ledgerIndex, range->maxSequence);
if (auto const status = std::get_if<RPC::Status>(&lgrInfoOrStatus))
return Error{*status};
@@ -47,28 +42,19 @@ BookOffersHandler::process(Input input, Context const& ctx) const
auto const bookKey = getBookBase(book);
// TODO: Add perfomance metrics if needed in future
auto [offers, _] = sharedPtrBackend_->fetchBookOffers(
bookKey, lgrInfo.seq, input.limit, ctx.yield);
auto [offers, _] = sharedPtrBackend_->fetchBookOffers(bookKey, lgrInfo.seq, input.limit, ctx.yield);
BookOffersHandler::Output output;
output.ledgerHash = ripple::strHex(lgrInfo.hash);
output.ledgerIndex = lgrInfo.seq;
output.offers = RPC::postProcessOrderBook(
offers,
book,
input.taker ? *(input.taker) : beast::zero,
*sharedPtrBackend_,
lgrInfo.seq,
ctx.yield);
offers, book, input.taker ? *(input.taker) : beast::zero, *sharedPtrBackend_, lgrInfo.seq, ctx.yield);
return output;
}
void
tag_invoke(
boost::json::value_from_tag,
boost::json::value& jv,
BookOffersHandler::Output const& output)
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, BookOffersHandler::Output const& output)
{
jv = boost::json::object{
{JS(ledger_hash), output.ledgerHash},
@@ -78,37 +64,19 @@ tag_invoke(
}
BookOffersHandler::Input
tag_invoke(
boost::json::value_to_tag<BookOffersHandler::Input>,
boost::json::value const& jv)
tag_invoke(boost::json::value_to_tag<BookOffersHandler::Input>, boost::json::value const& jv)
{
BookOffersHandler::Input input;
auto const& jsonObject = jv.as_object();
ripple::to_currency(
input.getsCurrency,
jv.at(JS(taker_gets)).as_object().at(JS(currency)).as_string().c_str());
ripple::to_currency(
input.paysCurrency,
jv.at(JS(taker_pays)).as_object().at(JS(currency)).as_string().c_str());
ripple::to_currency(input.getsCurrency, jv.at(JS(taker_gets)).as_object().at(JS(currency)).as_string().c_str());
ripple::to_currency(input.paysCurrency, jv.at(JS(taker_pays)).as_object().at(JS(currency)).as_string().c_str());
if (jv.at(JS(taker_gets)).as_object().contains(JS(issuer)))
{
ripple::to_issuer(
input.getsID,
jv.at(JS(taker_gets))
.as_object()
.at(JS(issuer))
.as_string()
.c_str());
ripple::to_issuer(input.getsID, jv.at(JS(taker_gets)).as_object().at(JS(issuer)).as_string().c_str());
}
if (jv.at(JS(taker_pays)).as_object().contains(JS(issuer)))
{
ripple::to_issuer(
input.paysID,
jv.at(JS(taker_pays))
.as_object()
.at(JS(issuer))
.as_string()
.c_str());
ripple::to_issuer(input.paysID, jv.at(JS(taker_pays)).as_object().at(JS(issuer)).as_string().c_str());
}
if (jsonObject.contains(JS(ledger_hash)))
{
@@ -122,14 +90,12 @@ tag_invoke(
}
else if (jsonObject.at(JS(ledger_index)).as_string() != "validated")
{
input.ledgerIndex =
std::stoi(jv.at(JS(ledger_index)).as_string().c_str());
input.ledgerIndex = std::stoi(jv.at(JS(ledger_index)).as_string().c_str());
}
}
if (jsonObject.contains(JS(taker)))
{
input.taker =
RPC::accountFromStringStrict(jv.at(JS(taker)).as_string().c_str());
input.taker = RPC::accountFromStringStrict(jv.at(JS(taker)).as_string().c_str());
}
if (jsonObject.contains(JS(limit)))
{

28
src/rpc/ngHandlers/BookOffers.h
View File

@@ -55,8 +55,7 @@ public:
using Result = RPCng::HandlerReturnType<Output>;
BookOffersHandler(std::shared_ptr<BackendInterface> const& sharedPtrBackend)
: sharedPtrBackend_(sharedPtrBackend)
BookOffersHandler(std::shared_ptr<BackendInterface> const& sharedPtrBackend) : sharedPtrBackend_(sharedPtrBackend)
{
}
@@ -71,12 +70,10 @@ public:
{JS(currency),
validation::Required{},
validation::WithCustomError{
validation::CurrencyValidator,
RPC::Status(RPC::RippledError::rpcDST_AMT_MALFORMED)}},
validation::CurrencyValidator, RPC::Status(RPC::RippledError::rpcDST_AMT_MALFORMED)}},
{JS(issuer),
validation::WithCustomError{
validation::IssuerValidator,
RPC::Status(RPC::RippledError::rpcDST_ISR_MALFORMED)}}}},
validation::IssuerValidator, RPC::Status(RPC::RippledError::rpcDST_ISR_MALFORMED)}}}},
{JS(taker_pays),
validation::Required{},
validation::Type<boost::json::object>{},
@@ -84,18 +81,16 @@ public:
{JS(currency),
validation::Required{},
validation::WithCustomError{
validation::CurrencyValidator,
RPC::Status(RPC::RippledError::rpcSRC_CUR_MALFORMED)}},
validation::CurrencyValidator, RPC::Status(RPC::RippledError::rpcSRC_CUR_MALFORMED)}},
{JS(issuer),
validation::WithCustomError{
validation::IssuerValidator,
RPC::Status(RPC::RippledError::rpcSRC_ISR_MALFORMED)}}}},
validation::IssuerValidator, RPC::Status(RPC::RippledError::rpcSRC_ISR_MALFORMED)}}}},
{JS(taker), validation::AccountValidator},
{JS(limit),
validation::Type<uint32_t>{},
validation::Between{1, 100}},
{JS(limit), validation::Type<uint32_t>{}, validation::Between{1, 100}},
{JS(ledger_hash), validation::Uint256HexStringValidator},
{JS(ledger_index), validation::LedgerIndexValidator}};
{JS(ledger_index), validation::LedgerIndexValidator},
};
return rpcSpec;
}
@@ -104,10 +99,7 @@ public:
private:
friend void
tag_invoke(
boost::json::value_from_tag,
boost::json::value& jv,
Output const& output);
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, Output const& output);
friend Input
tag_invoke(boost::json::value_to_tag<Input>, boost::json::value const& jv);

83
src/rpc/ngHandlers/GatewayBalances.cpp
View File

@@ -22,29 +22,22 @@
namespace RPCng {
GatewayBalancesHandler::Result
GatewayBalancesHandler::process(
GatewayBalancesHandler::Input input,
Context const& ctx) const
GatewayBalancesHandler::process(GatewayBalancesHandler::Input input, Context const& ctx) const
{
// check ledger
auto const range = sharedPtrBackend_->fetchLedgerRange();
auto const lgrInfoOrStatus = RPC::getLedgerInfoFromHashOrSeq(
*sharedPtrBackend_,
ctx.yield,
input.ledgerHash,
input.ledgerIndex,
range->maxSequence);
*sharedPtrBackend_, ctx.yield, input.ledgerHash, input.ledgerIndex, range->maxSequence);
if (auto const status = std::get_if<RPC::Status>(&lgrInfoOrStatus))
return Error{*status};
// check account
auto const lgrInfo = std::get<ripple::LedgerInfo>(lgrInfoOrStatus);
auto const accountID = RPC::accountFromStringStrict(input.account);
auto const accountLedgerObject = sharedPtrBackend_->fetchLedgerObject(
ripple::keylet::account(*accountID).key, lgrInfo.seq, ctx.yield);
auto const accountLedgerObject =
sharedPtrBackend_->fetchLedgerObject(ripple::keylet::account(*accountID).key, lgrInfo.seq, ctx.yield);
if (!accountLedgerObject)
return Error{RPC::Status{
RPC::RippledError::rpcACT_NOT_FOUND, "accountNotFound"}};
return Error{RPC::Status{RPC::RippledError::rpcACT_NOT_FOUND, "accountNotFound"}};
GatewayBalancesHandler::Output output;
@@ -58,8 +51,7 @@ GatewayBalancesHandler::process(
auto const highID = highLimit.getIssuer();
auto const viewLowest = (lowLimit.getIssuer() == accountID);
auto const flags = sle.getFieldU32(ripple::sfFlags);
auto const freeze = flags &
(viewLowest ? ripple::lsfLowFreeze : ripple::lsfHighFreeze);
auto const freeze = flags & (viewLowest ? ripple::lsfLowFreeze : ripple::lsfHighFreeze);
if (!viewLowest)
balance.negate();
@@ -126,12 +118,10 @@ GatewayBalancesHandler::process(
if (auto status = std::get_if<RPC::Status>(&ret))
return Error{*status};
if (not std::all_of(
input.hotWallets.begin(),
input.hotWallets.end(),
[&](auto const& hw) { return output.hotBalances.contains(hw); }))
return Error{RPC::Status{
RPC::RippledError::rpcINVALID_PARAMS, "invalidHotWallet"}};
if (not std::all_of(input.hotWallets.begin(), input.hotWallets.end(), [&](auto const& hw) {
return output.hotBalances.contains(hw);
}))
return Error{RPC::Status{RPC::RippledError::rpcINVALID_PARAMS, "invalidHotWallet"}};
output.accountID = input.account;
output.ledgerHash = ripple::strHex(lgrInfo.hash);
@@ -140,10 +130,7 @@ GatewayBalancesHandler::process(
}
void
tag_invoke(
boost::json::value_from_tag,
boost::json::value& jv,
GatewayBalancesHandler::Output const& output)
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, GatewayBalancesHandler::Output const& output)
{
boost::json::object obj;
if (!output.sums.empty())
@@ -156,28 +143,25 @@ tag_invoke(
obj[JS(obligations)] = std::move(obligations);
}
auto const toJson =
[](std::map<ripple::AccountID, std::vector<ripple::STAmount>> const&
balances) {
boost::json::object balancesObj;
if (!balances.empty())
auto const toJson = [](std::map<ripple::AccountID, std::vector<ripple::STAmount>> const& balances) {
boost::json::object balancesObj;
if (!balances.empty())
{
for (auto const& [accId, accBalances] : balances)
{
for (auto const& [accId, accBalances] : balances)
boost::json::array arr;
for (auto const& balance : accBalances)
{
boost::json::array arr;
for (auto const& balance : accBalances)
{
boost::json::object entry;
entry[JS(currency)] =
ripple::to_string(balance.issue().currency);
entry[JS(value)] = balance.getText();
arr.push_back(std::move(entry));
}
balancesObj[ripple::to_string(accId)] = std::move(arr);
boost::json::object entry;
entry[JS(currency)] = ripple::to_string(balance.issue().currency);
entry[JS(value)] = balance.getText();
arr.push_back(std::move(entry));
}
balancesObj[ripple::to_string(accId)] = std::move(arr);
}
return balancesObj;
};
}
return balancesObj;
};
if (auto balances = toJson(output.hotBalances); balances.size())
obj[JS(balances)] = balances;
@@ -197,9 +181,7 @@ tag_invoke(
}
GatewayBalancesHandler::Input
tag_invoke(
boost::json::value_to_tag<GatewayBalancesHandler::Input>,
boost::json::value const& jv)
tag_invoke(boost::json::value_to_tag<GatewayBalancesHandler::Input>, boost::json::value const& jv)
{
auto const& jsonObject = jv.as_object();
GatewayBalancesHandler::Input input;
@@ -216,16 +198,14 @@ tag_invoke(
}
else if (jsonObject.at(JS(ledger_index)).as_string() != "validated")
{
input.ledgerIndex =
std::stoi(jv.at(JS(ledger_index)).as_string().c_str());
input.ledgerIndex = std::stoi(jv.at(JS(ledger_index)).as_string().c_str());
}
}
if (jsonObject.contains(JS(hotwallet)))
{
if (jsonObject.at(JS(hotwallet)).is_string())
{
input.hotWallets.insert(*RPC::accountFromStringStrict(
jv.at(JS(hotwallet)).as_string().c_str()));
input.hotWallets.insert(*RPC::accountFromStringStrict(jv.at(JS(hotwallet)).as_string().c_str()));
}
else
{
@@ -234,10 +214,7 @@ tag_invoke(
hotWallets.begin(),
hotWallets.end(),
std::inserter(input.hotWallets, input.hotWallets.begin()),
[](auto const& hotWallet) {
return *RPC::accountFromStringStrict(
hotWallet.as_string().c_str());
});
[](auto const& hotWallet) { return *RPC::accountFromStringStrict(hotWallet.as_string().c_str()); });
}
}
return input;

Some files were not shown because too many files have changed in this diff Show More