From 053ffd7b18e43da90b63e0afd9be505bb0a4992d Mon Sep 17 00:00:00 2001 From: Richard Holland Date: Sat, 1 Feb 2025 11:56:46 +1100 Subject: [PATCH] add strict filtering to account_tx api --- src/ripple/app/rdb/RelationalDatabase.h | 3 + src/ripple/app/rdb/backend/RWDBDatabase.h | 102 +++++++++++++++--- .../app/rdb/backend/detail/impl/Node.cpp | 24 +++-- src/ripple/basics/strHex.h | 11 ++ src/ripple/rpc/handlers/AccountTx.cpp | 6 +- 5 files changed, 124 insertions(+), 22 deletions(-) diff --git a/src/ripple/app/rdb/RelationalDatabase.h b/src/ripple/app/rdb/RelationalDatabase.h index a269bf256..11c592485 100644 --- a/src/ripple/app/rdb/RelationalDatabase.h +++ b/src/ripple/app/rdb/RelationalDatabase.h @@ -69,6 +69,7 @@ public: std::uint32_t offset; std::uint32_t limit; bool bUnlimited; + bool strict; }; struct AccountTxPageOptions @@ -79,6 +80,7 @@ public: std::optional marker; std::uint32_t limit; bool bAdmin; + bool strict; }; using AccountTx = @@ -101,6 +103,7 @@ public: bool forward = false; uint32_t limit = 0; std::optional marker; + bool strict; }; struct AccountTxResult diff --git a/src/ripple/app/rdb/backend/RWDBDatabase.h b/src/ripple/app/rdb/backend/RWDBDatabase.h index 9c6d70e7e..8653e6cf0 100644 --- a/src/ripple/app/rdb/backend/RWDBDatabase.h +++ b/src/ripple/app/rdb/backend/RWDBDatabase.h @@ -43,6 +43,34 @@ private: std::map transactionMap_; std::map accountTxMap_; + // helper function to scan for an account ID inside the tx and meta blobs + // used for strict filtering of account_tx + bool + isAccountInvolvedInTx(AccountID const& account, AccountTx const& accountTx) + { + auto const& txn = accountTx.first; + auto const& meta = accountTx.second; + + // Search metadata + Blob const metaBlob = meta->getAsObject().getSerializer().peekData(); + if (metaBlob.size() >= account.size() && + std::search( + metaBlob.begin(), + metaBlob.end(), + account.data(), + account.data() + account.size()) != metaBlob.end()) + return true; + + // Search transaction blob + Blob const txnBlob = txn->getSTransaction()->getSerializer().peekData(); + return txnBlob.size() >= account.size() && + std::search( + txnBlob.begin(), + txnBlob.end(), + account.data(), + account.data() + account.size()) != txnBlob.end(); + } + public: RWDBDatabase(Application& app, Config const& config, JobQueue& jobQueue) : app_(app), useTxTables_(config.useTxTables()) @@ -193,7 +221,17 @@ public: std::size_t count = 0; for (const auto& [_, accountData] : accountTxMap_) { - count += accountData.transactions.size(); + for (const auto& tx : accountData.transactions) + { + // RH NOTE: options isn't provided to this function + // but this function is probably only used internally + // so make it reflect the true number (unfiltered) + + // if (options.strict && + // !isAccountInvolvedInTx(options.account, tx)) + // continue; + count++; + } } return count; } @@ -607,12 +645,17 @@ public: { for (const auto& [txSeq, txIndex] : txIt->second) { + AccountTx const accountTx = accountData.transactions[txIndex]; + if (options.strict && + !isAccountInvolvedInTx(options.account, accountTx)) + continue; + if (skipped < options.offset) { ++skipped; continue; } - AccountTx const accountTx = accountData.transactions[txIndex]; + std::uint32_t const inLedger = rangeCheckedCast( accountTx.second->getLgrSeq()); accountTx.first->setStatus(COMMITTED); @@ -652,13 +695,18 @@ public: innerRIt != rIt->second.rend(); ++innerRIt) { + AccountTx const accountTx = + accountData.transactions[innerRIt->second]; + + if (options.strict && + !isAccountInvolvedInTx(options.account, accountTx)) + continue; + if (skipped < options.offset) { ++skipped; continue; } - AccountTx const accountTx = - accountData.transactions[innerRIt->second]; std::uint32_t const inLedger = rangeCheckedCast( accountTx.second->getLgrSeq()); accountTx.first->setLedger(inLedger); @@ -694,12 +742,19 @@ public: { for (const auto& [txSeq, txIndex] : txIt->second) { + AccountTx const accountTx = accountData.transactions[txIndex]; + + if (options.strict && + !isAccountInvolvedInTx(options.account, accountTx)) + continue; + + const auto& [txn, txMeta] = accountTx; + if (skipped < options.offset) { ++skipped; continue; } - const auto& [txn, txMeta] = accountData.transactions[txIndex]; result.emplace_back( txn->getSTransaction()->getSerializer().peekData(), txMeta->getAsObject().getSerializer().peekData(), @@ -738,13 +793,20 @@ public: innerRIt != rIt->second.rend(); ++innerRIt) { + AccountTx const accountTx = + accountData.transactions[innerRIt->second]; + + if (options.strict && + !isAccountInvolvedInTx(options.account, accountTx)) + continue; + + const auto& [txn, txMeta] = accountTx; + if (skipped < options.offset) { ++skipped; continue; } - const auto& [txn, txMeta] = - accountData.transactions[innerRIt->second]; result.emplace_back( txn->getSTransaction()->getSerializer().peekData(), txMeta->getAsObject().getSerializer().peekData(), @@ -838,18 +900,23 @@ public: return {newmarker, total}; } - Blob rawTxn = accountData.transactions[index] - .first->getSTransaction() + AccountTx const& accountTx = + accountData.transactions[index]; + + Blob rawTxn = accountTx.first->getSTransaction() ->getSerializer() .peekData(); - Blob rawMeta = accountData.transactions[index] - .second->getAsObject() + Blob rawMeta = accountTx.second->getAsObject() .getSerializer() .peekData(); if (rawMeta.size() == 0) onUnsavedLedger(ledgerSeq); + if (options.strict && + !isAccountInvolvedInTx(options.account, accountTx)) + continue; + onTransaction( rangeCheckedCast(ledgerSeq), "COMMITTED", @@ -893,18 +960,23 @@ public: return {newmarker, total}; } - Blob rawTxn = accountData.transactions[index] - .first->getSTransaction() + AccountTx const& accountTx = + accountData.transactions[index]; + + Blob rawTxn = accountTx.first->getSTransaction() ->getSerializer() .peekData(); - Blob rawMeta = accountData.transactions[index] - .second->getAsObject() + Blob rawMeta = accountTx.second->getAsObject() .getSerializer() .peekData(); if (rawMeta.size() == 0) onUnsavedLedger(ledgerSeq); + if (options.strict && + !isAccountInvolvedInTx(options.account, accountTx)) + continue; + onTransaction( rangeCheckedCast(ledgerSeq), "COMMITTED", diff --git a/src/ripple/app/rdb/backend/detail/impl/Node.cpp b/src/ripple/app/rdb/backend/detail/impl/Node.cpp index c80038ef7..63e80f498 100644 --- a/src/ripple/app/rdb/backend/detail/impl/Node.cpp +++ b/src/ripple/app/rdb/backend/detail/impl/Node.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -758,14 +759,24 @@ transactionsSQL( options.minLedger); } + // Convert account ID to hex string for binary search + std::string accountHex = + strHex(options.account.data(), options.account.size()); + std::string sql; + std::string filterClause = options.strict ? "AND (hex(TxnMeta) LIKE '%" + + accountHex + "%' OR hex(RawTxn) LIKE '%" + accountHex + "%')" + : ""; + if (count) sql = boost::str( boost::format("SELECT %s FROM AccountTransactions " - "WHERE Account = '%s' %s %s LIMIT %u, %u;") % - selection % toBase58(options.account) % maxClause % minClause % - beast::lexicalCastThrow(options.offset) % + "INNER JOIN Transactions ON Transactions.TransID = " + "AccountTransactions.TransID " + "WHERE Account = '%s' %s %s %s LIMIT %u, %u;") % + selection % toBase58(options.account) % filterClause % maxClause % + minClause % beast::lexicalCastThrow(options.offset) % beast::lexicalCastThrow(numberOfResults)); else sql = boost::str( @@ -773,15 +784,16 @@ transactionsSQL( "SELECT %s FROM " "AccountTransactions INNER JOIN Transactions " "ON Transactions.TransID = AccountTransactions.TransID " - "WHERE Account = '%s' %s %s " + "WHERE Account = '%s' %s %s %s " "ORDER BY AccountTransactions.LedgerSeq %s, " "AccountTransactions.TxnSeq %s, AccountTransactions.TransID %s " "LIMIT %u, %u;") % - selection % toBase58(options.account) % maxClause % minClause % + selection % toBase58(options.account) % filterClause % maxClause % + minClause % (descending ? "DESC" : "ASC") % (descending ? "DESC" : "ASC") % (descending ? "DESC" : "ASC") % - (descending ? "DESC" : "ASC") % beast::lexicalCastThrow(options.offset) % beast::lexicalCastThrow(numberOfResults)); + JLOG(j.trace()) << "txSQL query: " << sql; return sql; } diff --git a/src/ripple/basics/strHex.h b/src/ripple/basics/strHex.h index 257fb540b..b55ee9e87 100644 --- a/src/ripple/basics/strHex.h +++ b/src/ripple/basics/strHex.h @@ -40,6 +40,17 @@ strHex(FwdIt begin, FwdIt end) return result; } +template +std::string +strHex(FwdIt begin, std::size_t length) +{ + std::string result; + result.reserve(2 * length); + boost::algorithm::hex( + begin, std::next(begin, length), std::back_inserter(result)); + return result; +} + template ().begin())> std::string strHex(T const& from) diff --git a/src/ripple/rpc/handlers/AccountTx.cpp b/src/ripple/rpc/handlers/AccountTx.cpp index f65657d92..52389f4e5 100644 --- a/src/ripple/rpc/handlers/AccountTx.cpp +++ b/src/ripple/rpc/handlers/AccountTx.cpp @@ -223,7 +223,8 @@ doAccountTxHelp(RPC::Context& context, AccountTxArgs const& args) result.ledgerRange.max, result.marker, args.limit, - isUnlimited(context.role)}; + isUnlimited(context.role), + args.strict}; auto const db = dynamic_cast(&context.app.getRelationalDatabase()); @@ -369,6 +370,9 @@ doAccountTxJson(RPC::JsonContext& context) args.forward = params.isMember(jss::forward) && params[jss::forward].asBool(); + args.strict = + params.isMember(jss::strict) ? params[jss::strict].asBool() : true; + if (!params.isMember(jss::account)) return rpcError(rpcINVALID_PARAMS);