From 3c4c9c87c5b12960a2b3f3ea9a9faa06cfbe0045 Mon Sep 17 00:00:00 2001 From: Niq Dudfield Date: Tue, 26 Aug 2025 11:00:58 +0700 Subject: [PATCH] Fix rwdb memory leak with online_delete and remove flatmap (#570) Co-authored-by: Denis Angell --- Builds/CMake/RippledCore.cmake | 6 +- Builds/levelization/results/ordering.txt | 4 + cfg/rippled-example.cfg | 31 +- src/ripple/app/rdb/backend/FlatmapDatabase.h | 851 ------------------ src/ripple/app/rdb/backend/RWDBDatabase.h | 197 ++-- .../app/rdb/impl/RelationalDatabase.cpp | 10 - src/ripple/core/Config.h | 4 +- src/ripple/core/impl/Config.cpp | 25 +- .../nodestore/backend/FlatmapFactory.cpp | 235 ----- src/test/app/SHAMapStore_test.cpp | 4 + src/test/core/Config_test.cpp | 92 ++ src/test/rdb/RelationalDatabase_test.cpp | 756 ++++++++++++++++ 12 files changed, 1018 insertions(+), 1197 deletions(-) delete mode 100644 src/ripple/app/rdb/backend/FlatmapDatabase.h delete mode 100644 src/ripple/nodestore/backend/FlatmapFactory.cpp create mode 100644 src/test/rdb/RelationalDatabase_test.cpp diff --git a/Builds/CMake/RippledCore.cmake b/Builds/CMake/RippledCore.cmake index 6fc33cdee..6b876997b 100644 --- a/Builds/CMake/RippledCore.cmake +++ b/Builds/CMake/RippledCore.cmake @@ -548,7 +548,6 @@ target_sources (rippled PRIVATE src/ripple/nodestore/backend/CassandraFactory.cpp src/ripple/nodestore/backend/RWDBFactory.cpp src/ripple/nodestore/backend/MemoryFactory.cpp - src/ripple/nodestore/backend/FlatmapFactory.cpp src/ripple/nodestore/backend/NuDBFactory.cpp src/ripple/nodestore/backend/NullFactory.cpp src/ripple/nodestore/backend/RocksDBFactory.cpp @@ -995,6 +994,11 @@ if (tests) subdir: resource #]===============================] src/test/resource/Logic_test.cpp + #[===============================[ + test sources: + subdir: rdb + #]===============================] + src/test/rdb/RelationalDatabase_test.cpp #[===============================[ test sources: subdir: rpc diff --git a/Builds/levelization/results/ordering.txt b/Builds/levelization/results/ordering.txt index 12df1a86e..e409855a7 100644 --- a/Builds/levelization/results/ordering.txt +++ b/Builds/levelization/results/ordering.txt @@ -186,6 +186,10 @@ test.protocol > ripple.crypto test.protocol > ripple.json test.protocol > ripple.protocol test.protocol > test.toplevel +test.rdb > ripple.app +test.rdb > ripple.core +test.rdb > test.jtx +test.rdb > test.toplevel test.resource > ripple.basics test.resource > ripple.beast test.resource > ripple.resource diff --git a/cfg/rippled-example.cfg b/cfg/rippled-example.cfg index 41fe12158..80ec0a980 100644 --- a/cfg/rippled-example.cfg +++ b/cfg/rippled-example.cfg @@ -1063,14 +1063,16 @@ # RWDB is recommended for Validator and Peer nodes that are not required to # store history. # -# RWDB maintains its high speed regardless of the amount of history -# stored. Online delete should NOT be used instead RWDB will use the -# ledger_history config value to determine how many ledgers to keep in memory. -# -# Required keys for NuDB, RWDB and RocksDB: +# Required keys for NuDB and RocksDB: # # path Location to store the database # +# Required keys for RWDB: +# +# online_delete Required. RWDB stores data in memory and will +# grow unbounded without online_delete. See the +# online_delete section below. +# # Required keys for Cassandra: # # contact_points IP of a node in the Cassandra cluster @@ -1110,7 +1112,17 @@ # if sufficient IOPS capacity is available. # Default 0. # -# Optional keys for NuDB or RocksDB: +# online_delete for RWDB, NuDB and RocksDB: +# +# online_delete Minimum value of 256. Enable automatic purging +# of older ledger information. Maintain at least this +# number of ledger records online. Must be greater +# than or equal to ledger_history. +# +# REQUIRED for RWDB to prevent out-of-memory errors. +# Optional for NuDB and RocksDB. +# +# Optional keys for NuDB and RocksDB: # # earliest_seq The default is 32570 to match the XRP ledger # network's earliest allowed sequence. Alternate @@ -1120,12 +1132,7 @@ # it must be defined with the same value in both # sections. # -# online_delete Minimum value of 256. Enable automatic purging -# of older ledger information. Maintain at least this -# number of ledger records online. Must be greater -# than or equal to ledger_history. If using RWDB -# this value is ignored. -# + # These keys modify the behavior of online_delete, and thus are only # relevant if online_delete is defined and non-zero: # diff --git a/src/ripple/app/rdb/backend/FlatmapDatabase.h b/src/ripple/app/rdb/backend/FlatmapDatabase.h deleted file mode 100644 index 00927b1d4..000000000 --- a/src/ripple/app/rdb/backend/FlatmapDatabase.h +++ /dev/null @@ -1,851 +0,0 @@ -#ifndef RIPPLE_APP_RDB_BACKEND_FLATMAPDATABASE_H_INCLUDED -#define RIPPLE_APP_RDB_BACKEND_FLATMAPDATABASE_H_INCLUDED - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace ripple { - -struct base_uint_hasher -{ - using result_type = std::size_t; - - result_type - operator()(base_uint<256> const& value) const - { - return hardened_hash<>{}(value); - } - - result_type - operator()(AccountID const& value) const - { - return hardened_hash<>{}(value); - } -}; - -class FlatmapDatabase : public SQLiteDatabase -{ -private: - struct LedgerData - { - LedgerInfo info; - boost::unordered:: - concurrent_flat_map - transactions; - }; - - struct AccountTxData - { - boost::unordered:: - concurrent_flat_map, AccountTx> - transactions; - }; - - Application& app_; - - boost::unordered::concurrent_flat_map ledgers_; - boost::unordered:: - concurrent_flat_map - ledgerHashToSeq_; - boost::unordered::concurrent_flat_map - transactionMap_; - boost::unordered:: - concurrent_flat_map - accountTxMap_; - -public: - FlatmapDatabase(Application& app, Config const& config, JobQueue& jobQueue) - : app_(app) - { - } - - std::optional - getMinLedgerSeq() override - { - std::optional minSeq; - ledgers_.visit_all([&minSeq](auto const& pair) { - if (!minSeq || pair.first < *minSeq) - { - minSeq = pair.first; - } - }); - return minSeq; - } - - std::optional - getTransactionsMinLedgerSeq() override - { - std::optional minSeq; - transactionMap_.visit_all([&minSeq](auto const& pair) { - LedgerIndex seq = pair.second.second->getLgrSeq(); - if (!minSeq || seq < *minSeq) - { - minSeq = seq; - } - }); - return minSeq; - } - - std::optional - getAccountTransactionsMinLedgerSeq() override - { - std::optional minSeq; - accountTxMap_.visit_all([&minSeq](auto const& pair) { - pair.second.transactions.visit_all([&minSeq](auto const& tx) { - if (!minSeq || tx.first.first < *minSeq) - { - minSeq = tx.first.first; - } - }); - }); - return minSeq; - } - - std::optional - getMaxLedgerSeq() override - { - std::optional maxSeq; - ledgers_.visit_all([&maxSeq](auto const& pair) { - if (!maxSeq || pair.first > *maxSeq) - { - maxSeq = pair.first; - } - }); - return maxSeq; - } - void - deleteTransactionByLedgerSeq(LedgerIndex ledgerSeq) override - { - ledgers_.visit(ledgerSeq, [this](auto& item) { - item.second.transactions.visit_all([this](auto const& txPair) { - transactionMap_.erase(txPair.first); - }); - item.second.transactions.clear(); - }); - - accountTxMap_.visit_all([ledgerSeq](auto& item) { - item.second.transactions.erase_if([ledgerSeq](auto const& tx) { - return tx.first.first == ledgerSeq; - }); - }); - } - - void - deleteBeforeLedgerSeq(LedgerIndex ledgerSeq) override - { - ledgers_.erase_if([this, ledgerSeq](auto const& item) { - if (item.first < ledgerSeq) - { - item.second.transactions.visit_all([this](auto const& txPair) { - transactionMap_.erase(txPair.first); - }); - ledgerHashToSeq_.erase(item.second.info.hash); - return true; - } - return false; - }); - - accountTxMap_.visit_all([ledgerSeq](auto& item) { - item.second.transactions.erase_if([ledgerSeq](auto const& tx) { - return tx.first.first < ledgerSeq; - }); - }); - } - - void - deleteTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) override - { - ledgers_.visit_all([this, ledgerSeq](auto& item) { - if (item.first < ledgerSeq) - { - item.second.transactions.visit_all([this](auto const& txPair) { - transactionMap_.erase(txPair.first); - }); - item.second.transactions.clear(); - } - }); - - accountTxMap_.visit_all([ledgerSeq](auto& item) { - item.second.transactions.erase_if([ledgerSeq](auto const& tx) { - return tx.first.first < ledgerSeq; - }); - }); - } - - void - deleteAccountTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) override - { - accountTxMap_.visit_all([ledgerSeq](auto& item) { - item.second.transactions.erase_if([ledgerSeq](auto const& tx) { - return tx.first.first < ledgerSeq; - }); - }); - } - std::size_t - getTransactionCount() override - { - return transactionMap_.size(); - } - - std::size_t - getAccountTransactionCount() override - { - std::size_t count = 0; - accountTxMap_.visit_all([&count](auto const& item) { - count += item.second.transactions.size(); - }); - return count; - } - - CountMinMax - getLedgerCountMinMax() override - { - CountMinMax result{0, 0, 0}; - ledgers_.visit_all([&result](auto const& item) { - result.numberOfRows++; - if (result.minLedgerSequence == 0 || - item.first < result.minLedgerSequence) - { - result.minLedgerSequence = item.first; - } - if (item.first > result.maxLedgerSequence) - { - result.maxLedgerSequence = item.first; - } - }); - return result; - } - - bool - saveValidatedLedger( - std::shared_ptr const& ledger, - bool current) override - { - try - { - LedgerData ledgerData; - ledgerData.info = ledger->info(); - - auto aLedger = std::make_shared(ledger, app_); - for (auto const& acceptedLedgerTx : *aLedger) - { - auto const& txn = acceptedLedgerTx->getTxn(); - auto const& meta = acceptedLedgerTx->getMeta(); - auto const& id = txn->getTransactionID(); - - std::string reason; - auto accTx = std::make_pair( - std::make_shared(txn, reason, app_), - std::make_shared(meta)); - - ledgerData.transactions.emplace(id, accTx); - transactionMap_.emplace(id, accTx); - - for (auto const& account : meta.getAffectedAccounts()) - { - accountTxMap_.visit(account, [&](auto& data) { - data.second.transactions.emplace( - std::make_pair( - ledger->info().seq, - acceptedLedgerTx->getTxnSeq()), - accTx); - }); - } - } - - ledgers_.emplace(ledger->info().seq, std::move(ledgerData)); - ledgerHashToSeq_.emplace(ledger->info().hash, ledger->info().seq); - - if (current) - { - auto const cutoffSeq = - ledger->info().seq > app_.config().LEDGER_HISTORY - ? ledger->info().seq - app_.config().LEDGER_HISTORY - : 0; - - if (cutoffSeq > 0) - { - const std::size_t BATCH_SIZE = 128; - std::size_t deleted = 0; - - ledgers_.erase_if([&](auto const& item) { - if (deleted >= BATCH_SIZE) - return false; - - if (item.first < cutoffSeq) - { - item.second.transactions.visit_all( - [this](auto const& txPair) { - transactionMap_.erase(txPair.first); - }); - ledgerHashToSeq_.erase(item.second.info.hash); - deleted++; - return true; - } - return false; - }); - - if (deleted > 0) - { - accountTxMap_.visit_all([cutoffSeq](auto& item) { - item.second.transactions.erase_if( - [cutoffSeq](auto const& tx) { - return tx.first.first < cutoffSeq; - }); - }); - } - - app_.getLedgerMaster().clearPriorLedgers(cutoffSeq); - } - } - - return true; - } - catch (std::exception const&) - { - deleteTransactionByLedgerSeq(ledger->info().seq); - return false; - } - } - - std::optional - getLedgerInfoByIndex(LedgerIndex ledgerSeq) override - { - std::optional result; - ledgers_.visit(ledgerSeq, [&result](auto const& item) { - result = item.second.info; - }); - return result; - } - - std::optional - getNewestLedgerInfo() override - { - std::optional result; - ledgers_.visit_all([&result](auto const& item) { - if (!result || item.second.info.seq > result->seq) - { - result = item.second.info; - } - }); - return result; - } - - std::optional - getLimitedOldestLedgerInfo(LedgerIndex ledgerFirstIndex) override - { - std::optional result; - ledgers_.visit_all([&](auto const& item) { - if (item.first >= ledgerFirstIndex && - (!result || item.first < result->seq)) - { - result = item.second.info; - } - }); - return result; - } - - std::optional - getLimitedNewestLedgerInfo(LedgerIndex ledgerFirstIndex) override - { - std::optional result; - ledgers_.visit_all([&](auto const& item) { - if (item.first >= ledgerFirstIndex && - (!result || item.first > result->seq)) - { - result = item.second.info; - } - }); - return result; - } - - std::optional - getLedgerInfoByHash(uint256 const& ledgerHash) override - { - std::optional result; - ledgerHashToSeq_.visit(ledgerHash, [this, &result](auto const& item) { - ledgers_.visit(item.second, [&result](auto const& item) { - result = item.second.info; - }); - }); - return result; - } - uint256 - getHashByIndex(LedgerIndex ledgerIndex) override - { - uint256 result; - ledgers_.visit(ledgerIndex, [&result](auto const& item) { - result = item.second.info.hash; - }); - return result; - } - - std::optional - getHashesByIndex(LedgerIndex ledgerIndex) override - { - std::optional result; - ledgers_.visit(ledgerIndex, [&result](auto const& item) { - result = LedgerHashPair{ - item.second.info.hash, item.second.info.parentHash}; - }); - return result; - } - - std::map - getHashesByIndex(LedgerIndex minSeq, LedgerIndex maxSeq) override - { - std::map result; - ledgers_.visit_all([&](auto const& item) { - if (item.first >= minSeq && item.first <= maxSeq) - { - result[item.first] = LedgerHashPair{ - item.second.info.hash, item.second.info.parentHash}; - } - }); - return result; - } - - std::variant - getTransaction( - uint256 const& id, - std::optional> const& range, - error_code_i& ec) override - { - std::variant result = TxSearched::unknown; - transactionMap_.visit(id, [&](auto const& item) { - auto const& tx = item.second; - if (!range || - (range->lower() <= tx.second->getLgrSeq() && - tx.second->getLgrSeq() <= range->upper())) - { - result = tx; - } - else - { - result = TxSearched::all; - } - }); - return result; - } - - bool - ledgerDbHasSpace(Config const& config) override - { - return true; // In-memory database always has space - } - - bool - transactionDbHasSpace(Config const& config) override - { - return true; // In-memory database always has space - } - - std::uint32_t - getKBUsedAll() override - { - std::uint32_t size = sizeof(*this); - size += ledgers_.size() * (sizeof(LedgerIndex) + sizeof(LedgerData)); - size += - ledgerHashToSeq_.size() * (sizeof(uint256) + sizeof(LedgerIndex)); - size += transactionMap_.size() * (sizeof(uint256) + sizeof(AccountTx)); - accountTxMap_.visit_all([&size](auto const& item) { - size += sizeof(AccountID) + sizeof(AccountTxData); - size += item.second.transactions.size() * sizeof(AccountTx); - }); - return size / 1024; // Convert to KB - } - - std::uint32_t - getKBUsedLedger() override - { - std::uint32_t size = - ledgers_.size() * (sizeof(LedgerIndex) + sizeof(LedgerData)); - size += - ledgerHashToSeq_.size() * (sizeof(uint256) + sizeof(LedgerIndex)); - return size / 1024; - } - - std::uint32_t - getKBUsedTransaction() override - { - std::uint32_t size = - transactionMap_.size() * (sizeof(uint256) + sizeof(AccountTx)); - accountTxMap_.visit_all([&size](auto const& item) { - size += sizeof(AccountID) + sizeof(AccountTxData); - size += item.second.transactions.size() * sizeof(AccountTx); - }); - return size / 1024; - } - - void - closeLedgerDB() override - { - // No-op for in-memory database - } - - void - closeTransactionDB() override - { - // No-op for in-memory database - } - - ~FlatmapDatabase() - { - // Concurrent maps need visit_all - accountTxMap_.visit_all( - [](auto& pair) { pair.second.transactions.clear(); }); - accountTxMap_.clear(); - - transactionMap_.clear(); - - ledgers_.visit_all( - [](auto& pair) { pair.second.transactions.clear(); }); - ledgers_.clear(); - - ledgerHashToSeq_.clear(); - } - - std::vector> - getTxHistory(LedgerIndex startIndex) override - { - std::vector> result; - transactionMap_.visit_all([&](auto const& item) { - if (item.second.second->getLgrSeq() >= startIndex) - { - result.push_back(item.second.first); - } - }); - std::sort( - result.begin(), result.end(), [](auto const& a, auto const& b) { - return a->getLedger() > b->getLedger(); - }); - if (result.size() > 20) - { - result.resize(20); - } - return result; - } - // Helper function to handle limits - template - void - applyLimit(Container& container, std::size_t limit, bool bUnlimited) - { - if (!bUnlimited && limit > 0 && container.size() > limit) - { - container.resize(limit); - } - } - - AccountTxs - getOldestAccountTxs(AccountTxOptions const& options) override - { - AccountTxs result; - accountTxMap_.visit(options.account, [&](auto const& item) { - item.second.transactions.visit_all([&](auto const& tx) { - if (tx.first.first >= options.minLedger && - tx.first.first <= options.maxLedger) - { - result.push_back(tx.second); - } - }); - }); - std::sort( - result.begin(), result.end(), [](auto const& a, auto const& b) { - return a.second->getLgrSeq() < b.second->getLgrSeq(); - }); - applyLimit(result, options.limit, options.bUnlimited); - return result; - } - - AccountTxs - getNewestAccountTxs(AccountTxOptions const& options) override - { - AccountTxs result; - accountTxMap_.visit(options.account, [&](auto const& item) { - item.second.transactions.visit_all([&](auto const& tx) { - if (tx.first.first >= options.minLedger && - tx.first.first <= options.maxLedger) - { - result.push_back(tx.second); - } - }); - }); - std::sort( - result.begin(), result.end(), [](auto const& a, auto const& b) { - return a.second->getLgrSeq() > b.second->getLgrSeq(); - }); - applyLimit(result, options.limit, options.bUnlimited); - return result; - } - - MetaTxsList - getOldestAccountTxsB(AccountTxOptions const& options) override - { - MetaTxsList result; - accountTxMap_.visit(options.account, [&](auto const& item) { - item.second.transactions.visit_all([&](auto const& tx) { - if (tx.first.first >= options.minLedger && - tx.first.first <= options.maxLedger) - { - result.emplace_back( - tx.second.first->getSTransaction() - ->getSerializer() - .peekData(), - tx.second.second->getAsObject() - .getSerializer() - .peekData(), - tx.first.first); - } - }); - }); - std::sort( - result.begin(), result.end(), [](auto const& a, auto const& b) { - return std::get<2>(a) < std::get<2>(b); - }); - applyLimit(result, options.limit, options.bUnlimited); - return result; - } - - MetaTxsList - getNewestAccountTxsB(AccountTxOptions const& options) override - { - MetaTxsList result; - accountTxMap_.visit(options.account, [&](auto const& item) { - item.second.transactions.visit_all([&](auto const& tx) { - if (tx.first.first >= options.minLedger && - tx.first.first <= options.maxLedger) - { - result.emplace_back( - tx.second.first->getSTransaction() - ->getSerializer() - .peekData(), - tx.second.second->getAsObject() - .getSerializer() - .peekData(), - tx.first.first); - } - }); - }); - std::sort( - result.begin(), result.end(), [](auto const& a, auto const& b) { - return std::get<2>(a) > std::get<2>(b); - }); - applyLimit(result, options.limit, options.bUnlimited); - return result; - } - std::pair> - oldestAccountTxPage(AccountTxPageOptions const& options) override - { - AccountTxs result; - std::optional marker; - - accountTxMap_.visit(options.account, [&](auto const& item) { - std::vector, AccountTx>> - txs; - item.second.transactions.visit_all([&](auto const& tx) { - if (tx.first.first >= options.minLedger && - tx.first.first <= options.maxLedger) - { - txs.emplace_back(tx); - } - }); - - std::sort(txs.begin(), txs.end(), [](auto const& a, auto const& b) { - return a.first < b.first; - }); - - auto it = txs.begin(); - if (options.marker) - { - it = std::find_if(txs.begin(), txs.end(), [&](auto const& tx) { - return tx.first.first == options.marker->ledgerSeq && - tx.first.second == options.marker->txnSeq; - }); - if (it != txs.end()) - ++it; - } - - for (; it != txs.end() && - (options.limit == 0 || result.size() < options.limit); - ++it) - { - result.push_back(it->second); - } - - if (it != txs.end()) - { - marker = AccountTxMarker{it->first.first, it->first.second}; - } - }); - - return {result, marker}; - } - - std::pair> - newestAccountTxPage(AccountTxPageOptions const& options) override - { - AccountTxs result; - std::optional marker; - - accountTxMap_.visit(options.account, [&](auto const& item) { - std::vector, AccountTx>> - txs; - item.second.transactions.visit_all([&](auto const& tx) { - if (tx.first.first >= options.minLedger && - tx.first.first <= options.maxLedger) - { - txs.emplace_back(tx); - } - }); - - std::sort(txs.begin(), txs.end(), [](auto const& a, auto const& b) { - return a.first > b.first; - }); - - auto it = txs.begin(); - if (options.marker) - { - it = std::find_if(txs.begin(), txs.end(), [&](auto const& tx) { - return tx.first.first == options.marker->ledgerSeq && - tx.first.second == options.marker->txnSeq; - }); - if (it != txs.end()) - ++it; - } - - for (; it != txs.end() && - (options.limit == 0 || result.size() < options.limit); - ++it) - { - result.push_back(it->second); - } - - if (it != txs.end()) - { - marker = AccountTxMarker{it->first.first, it->first.second}; - } - }); - - return {result, marker}; - } - - std::pair> - oldestAccountTxPageB(AccountTxPageOptions const& options) override - { - MetaTxsList result; - std::optional marker; - - accountTxMap_.visit(options.account, [&](auto const& item) { - std::vector> txs; - item.second.transactions.visit_all([&](auto const& tx) { - if (tx.first.first >= options.minLedger && - tx.first.first <= options.maxLedger) - { - txs.emplace_back( - tx.first.first, tx.first.second, tx.second); - } - }); - - std::sort(txs.begin(), txs.end()); - - auto it = txs.begin(); - if (options.marker) - { - it = std::find_if(txs.begin(), txs.end(), [&](auto const& tx) { - return std::get<0>(tx) == options.marker->ledgerSeq && - std::get<1>(tx) == options.marker->txnSeq; - }); - if (it != txs.end()) - ++it; - } - - for (; it != txs.end() && - (options.limit == 0 || result.size() < options.limit); - ++it) - { - const auto& [_, __, tx] = *it; - result.emplace_back( - tx.first->getSTransaction()->getSerializer().peekData(), - tx.second->getAsObject().getSerializer().peekData(), - std::get<0>(*it)); - } - - if (it != txs.end()) - { - marker = AccountTxMarker{std::get<0>(*it), std::get<1>(*it)}; - } - }); - - return {result, marker}; - } - - std::pair> - newestAccountTxPageB(AccountTxPageOptions const& options) override - { - MetaTxsList result; - std::optional marker; - - accountTxMap_.visit(options.account, [&](auto const& item) { - std::vector> txs; - item.second.transactions.visit_all([&](auto const& tx) { - if (tx.first.first >= options.minLedger && - tx.first.first <= options.maxLedger) - { - txs.emplace_back( - tx.first.first, tx.first.second, tx.second); - } - }); - - std::sort(txs.begin(), txs.end(), std::greater<>()); - - auto it = txs.begin(); - if (options.marker) - { - it = std::find_if(txs.begin(), txs.end(), [&](auto const& tx) { - return std::get<0>(tx) == options.marker->ledgerSeq && - std::get<1>(tx) == options.marker->txnSeq; - }); - if (it != txs.end()) - ++it; - } - - for (; it != txs.end() && - (options.limit == 0 || result.size() < options.limit); - ++it) - { - const auto& [_, __, tx] = *it; - result.emplace_back( - tx.first->getSTransaction()->getSerializer().peekData(), - tx.second->getAsObject().getSerializer().peekData(), - std::get<0>(*it)); - } - - if (it != txs.end()) - { - marker = AccountTxMarker{std::get<0>(*it), std::get<1>(*it)}; - } - }); - - return {result, marker}; - } -}; - -// Factory function -std::unique_ptr -getFlatmapDatabase(Application& app, Config const& config, JobQueue& jobQueue) -{ - return std::make_unique(app, config, jobQueue); -} - -} // namespace ripple -#endif // RIPPLE_APP_RDB_BACKEND_FLATMAPDATABASE_H_INCLUDED diff --git a/src/ripple/app/rdb/backend/RWDBDatabase.h b/src/ripple/app/rdb/backend/RWDBDatabase.h index 3981691bc..f91e50cda 100644 --- a/src/ripple/app/rdb/backend/RWDBDatabase.h +++ b/src/ripple/app/rdb/backend/RWDBDatabase.h @@ -28,9 +28,8 @@ private: struct AccountTxData { - AccountTxs transactions; - std::map> - ledgerTxMap; // ledgerSeq -> txSeq -> index in transactions + std::map> + ledgerTxMap; // ledgerSeq -> vector of transactions }; Application& app_; @@ -65,9 +64,12 @@ public: return {}; std::shared_lock lock(mutex_); - if (transactionMap_.empty()) - return std::nullopt; - return transactionMap_.begin()->second.second->getLgrSeq(); + for (const auto& [ledgerSeq, ledgerData] : ledgers_) + { + if (!ledgerData.transactions.empty()) + return ledgerSeq; + } + return std::nullopt; } std::optional @@ -163,14 +165,6 @@ public: { txIt = accountData.ledgerTxMap.erase(txIt); } - accountData.transactions.erase( - std::remove_if( - accountData.transactions.begin(), - accountData.transactions.end(), - [ledgerSeq](const AccountTx& tx) { - return tx.second->getLgrSeq() < ledgerSeq; - }), - accountData.transactions.end()); } } std::size_t @@ -193,7 +187,10 @@ public: std::size_t count = 0; for (const auto& [_, accountData] : accountTxMap_) { - count += accountData.transactions.size(); + for (const auto& [_, txVector] : accountData.ledgerTxMap) + { + count += txVector.size(); + } } return count; } @@ -293,10 +290,7 @@ public: accountTxMap_[account] = AccountTxData(); auto& accountData = accountTxMap_[account]; - accountData.transactions.push_back(accTx); - accountData - .ledgerTxMap[seq][acceptedLedgerTx->getTxnSeq()] = - accountData.transactions.size() - 1; + accountData.ledgerTxMap[seq].push_back(accTx); } app_.getMasterTransaction().inLedger( @@ -451,59 +445,108 @@ public: return true; // In-memory database always has space } + // Red-black tree node overhead per map entry + static constexpr size_t MAP_NODE_OVERHEAD = 40; + +private: + std::uint64_t + getBytesUsedLedger_unlocked() const + { + std::uint64_t size = 0; + + // Count structural overhead of ledger storage including map node + // overhead Note: sizeof(LedgerData) includes the map container for + // transactions, but not the actual transaction data + size += ledgers_.size() * + (sizeof(LedgerIndex) + sizeof(LedgerData) + MAP_NODE_OVERHEAD); + + // Add the transaction map nodes inside each ledger (ledger's view of + // its transactions) + for (const auto& [_, ledgerData] : ledgers_) + { + size += ledgerData.transactions.size() * + (sizeof(uint256) + sizeof(AccountTx) + MAP_NODE_OVERHEAD); + } + + // Count the ledger hash to sequence lookup map + size += ledgerHashToSeq_.size() * + (sizeof(uint256) + sizeof(LedgerIndex) + MAP_NODE_OVERHEAD); + + return size; + } + + std::uint64_t + getBytesUsedTransaction_unlocked() const + { + if (!useTxTables_) + return 0; + + std::uint64_t size = 0; + + // Count structural overhead of transaction map + // sizeof(AccountTx) is just the size of two shared_ptrs (~32 bytes) + size += transactionMap_.size() * + (sizeof(uint256) + sizeof(AccountTx) + MAP_NODE_OVERHEAD); + + // Add actual transaction and metadata data sizes + for (const auto& [_, accountTx] : transactionMap_) + { + if (accountTx.first) + size += accountTx.first->getSTransaction() + ->getSerializer() + .peekData() + .size(); + if (accountTx.second) + size += accountTx.second->getAsObject() + .getSerializer() + .peekData() + .size(); + } + + // Count structural overhead of account transaction index + // The actual transaction data is already counted above from + // transactionMap_ + for (const auto& [accountId, accountData] : accountTxMap_) + { + size += + sizeof(accountId) + sizeof(AccountTxData) + MAP_NODE_OVERHEAD; + for (const auto& [ledgerSeq, txVector] : accountData.ledgerTxMap) + { + // Use capacity() to account for actual allocated memory + size += sizeof(ledgerSeq) + MAP_NODE_OVERHEAD; + size += txVector.capacity() * sizeof(AccountTx); + } + } + + return size; + } + +public: std::uint32_t getKBUsedAll() override { std::shared_lock lock(mutex_); - std::uint32_t size = sizeof(*this); - size += ledgers_.size() * (sizeof(LedgerIndex) + sizeof(LedgerData)); - size += - ledgerHashToSeq_.size() * (sizeof(uint256) + sizeof(LedgerIndex)); - size += transactionMap_.size() * (sizeof(uint256) + sizeof(AccountTx)); - for (const auto& [_, accountData] : accountTxMap_) - { - size += sizeof(AccountID) + sizeof(AccountTxData); - size += accountData.transactions.size() * sizeof(AccountTx); - for (const auto& [_, innerMap] : accountData.ledgerTxMap) - { - size += sizeof(uint32_t) + - innerMap.size() * (sizeof(uint32_t) + sizeof(size_t)); - } - } - return size / 1024; + + // Total = base object + ledger infrastructure + transaction data + std::uint64_t size = sizeof(*this) + getBytesUsedLedger_unlocked() + + getBytesUsedTransaction_unlocked(); + + return static_cast(size / 1024); } std::uint32_t getKBUsedLedger() override { std::shared_lock lock(mutex_); - std::uint32_t size = 0; - size += ledgers_.size() * (sizeof(LedgerIndex) + sizeof(LedgerData)); - size += - ledgerHashToSeq_.size() * (sizeof(uint256) + sizeof(LedgerIndex)); - return size / 1024; + return static_cast(getBytesUsedLedger_unlocked() / 1024); } std::uint32_t getKBUsedTransaction() override { - if (!useTxTables_) - return 0; - std::shared_lock lock(mutex_); - std::uint32_t size = 0; - size += transactionMap_.size() * (sizeof(uint256) + sizeof(AccountTx)); - for (const auto& [_, accountData] : accountTxMap_) - { - size += sizeof(AccountID) + sizeof(AccountTxData); - size += accountData.transactions.size() * sizeof(AccountTx); - for (const auto& [_, innerMap] : accountData.ledgerTxMap) - { - size += sizeof(uint32_t) + - innerMap.size() * (sizeof(uint32_t) + sizeof(size_t)); - } - } - return size / 1024; + return static_cast( + getBytesUsedTransaction_unlocked() / 1024); } void @@ -605,14 +648,13 @@ public: (options.bUnlimited || result.size() < options.limit); ++txIt) { - for (const auto& [txSeq, txIndex] : txIt->second) + for (const auto& accountTx : txIt->second) { if (skipped < options.offset) { ++skipped; continue; } - AccountTx const accountTx = accountData.transactions[txIndex]; std::uint32_t const inLedger = rangeCheckedCast( accountTx.second->getLgrSeq()); accountTx.first->setStatus(COMMITTED); @@ -657,8 +699,7 @@ public: ++skipped; continue; } - AccountTx const accountTx = - accountData.transactions[innerRIt->second]; + AccountTx const accountTx = *innerRIt; std::uint32_t const inLedger = rangeCheckedCast( accountTx.second->getLgrSeq()); accountTx.first->setLedger(inLedger); @@ -692,14 +733,14 @@ public: (options.bUnlimited || result.size() < options.limit); ++txIt) { - for (const auto& [txSeq, txIndex] : txIt->second) + for (const auto& accountTx : txIt->second) { if (skipped < options.offset) { ++skipped; continue; } - const auto& [txn, txMeta] = accountData.transactions[txIndex]; + const auto& [txn, txMeta] = accountTx; result.emplace_back( txn->getSTransaction()->getSerializer().peekData(), txMeta->getAsObject().getSerializer().peekData(), @@ -743,8 +784,7 @@ public: ++skipped; continue; } - const auto& [txn, txMeta] = - accountData.transactions[innerRIt->second]; + const auto& [txn, txMeta] = *innerRIt; result.emplace_back( txn->getSTransaction()->getSerializer().peekData(), txMeta->getAsObject().getSerializer().peekData(), @@ -816,11 +856,9 @@ public: for (; txIt != txEnd; ++txIt) { std::uint32_t const ledgerSeq = txIt->first; - for (auto seqIt = txIt->second.begin(); - seqIt != txIt->second.end(); - ++seqIt) + std::uint32_t txnSeq = 0; + for (const auto& accountTx : txIt->second) { - const auto& [txnSeq, index] = *seqIt; if (lookingForMarker) { if (findLedger == ledgerSeq && findSeq == txnSeq) @@ -828,7 +866,10 @@ public: lookingForMarker = false; } else + { + ++txnSeq; continue; + } } else if (numberOfResults == 0) { @@ -837,12 +878,10 @@ public: return {newmarker, total}; } - Blob rawTxn = accountData.transactions[index] - .first->getSTransaction() + Blob rawTxn = accountTx.first->getSTransaction() ->getSerializer() .peekData(); - Blob rawMeta = accountData.transactions[index] - .second->getAsObject() + Blob rawMeta = accountTx.second->getAsObject() .getSerializer() .peekData(); @@ -856,6 +895,7 @@ public: std::move(rawMeta)); --numberOfResults; ++total; + ++txnSeq; } } } @@ -871,11 +911,11 @@ public: for (; rtxIt != rtxEnd; ++rtxIt) { std::uint32_t const ledgerSeq = rtxIt->first; + std::uint32_t txnSeq = rtxIt->second.size() - 1; for (auto innerRIt = rtxIt->second.rbegin(); innerRIt != rtxIt->second.rend(); ++innerRIt) { - const auto& [txnSeq, index] = *innerRIt; if (lookingForMarker) { if (findLedger == ledgerSeq && findSeq == txnSeq) @@ -883,7 +923,10 @@ public: lookingForMarker = false; } else + { + --txnSeq; continue; + } } else if (numberOfResults == 0) { @@ -892,12 +935,11 @@ public: return {newmarker, total}; } - Blob rawTxn = accountData.transactions[index] - .first->getSTransaction() + const auto& accountTx = *innerRIt; + Blob rawTxn = accountTx.first->getSTransaction() ->getSerializer() .peekData(); - Blob rawMeta = accountData.transactions[index] - .second->getAsObject() + Blob rawMeta = accountTx.second->getAsObject() .getSerializer() .peekData(); @@ -911,6 +953,7 @@ public: std::move(rawMeta)); --numberOfResults; ++total; + --txnSeq; } } } diff --git a/src/ripple/app/rdb/impl/RelationalDatabase.cpp b/src/ripple/app/rdb/impl/RelationalDatabase.cpp index 64161bd53..bf24d7dc7 100644 --- a/src/ripple/app/rdb/impl/RelationalDatabase.cpp +++ b/src/ripple/app/rdb/impl/RelationalDatabase.cpp @@ -19,7 +19,6 @@ #include #include -#include #include #include #include @@ -41,7 +40,6 @@ RelationalDatabase::init( bool use_sqlite = false; bool use_postgres = false; bool use_rwdb = false; - bool use_flatmap = false; if (config.reporting()) { @@ -60,10 +58,6 @@ RelationalDatabase::init( { use_rwdb = true; } - else if (boost::iequals(get(rdb_section, "backend"), "flatmap")) - { - use_flatmap = true; - } else { Throw( @@ -89,10 +83,6 @@ RelationalDatabase::init( { return getRWDBDatabase(app, config, jobQueue); } - else if (use_flatmap) - { - return getFlatmapDatabase(app, config, jobQueue); - } return std::unique_ptr(); } diff --git a/src/ripple/core/Config.h b/src/ripple/core/Config.h index 3e2c3c81a..0909f88ac 100644 --- a/src/ripple/core/Config.h +++ b/src/ripple/core/Config.h @@ -361,9 +361,7 @@ public: boost::beast::iequals( get(section(SECTION_RELATIONAL_DB), "backend"), "rwdb")) || (!section("node_db").empty() && - (boost::beast::iequals(get(section("node_db"), "type"), "rwdb") || - boost::beast::iequals( - get(section("node_db"), "type"), "flatmap"))); + boost::beast::iequals(get(section("node_db"), "type"), "rwdb")); // RHNOTE: memory type is not selected for here because it breaks // tests return isMem; diff --git a/src/ripple/core/impl/Config.cpp b/src/ripple/core/impl/Config.cpp index 7673d16ec..9fd23f33e 100644 --- a/src/ripple/core/impl/Config.cpp +++ b/src/ripple/core/impl/Config.cpp @@ -45,7 +45,6 @@ namespace ripple { namespace detail { - [[nodiscard]] std::uint64_t getMemorySize() { @@ -54,7 +53,6 @@ getMemorySize() return 0; } - } // namespace detail } // namespace ripple #endif @@ -64,7 +62,6 @@ getMemorySize() namespace ripple { namespace detail { - [[nodiscard]] std::uint64_t getMemorySize() { @@ -73,7 +70,6 @@ getMemorySize() return 0; } - } // namespace detail } // namespace ripple @@ -85,7 +81,6 @@ getMemorySize() namespace ripple { namespace detail { - [[nodiscard]] std::uint64_t getMemorySize() { @@ -98,13 +93,11 @@ getMemorySize() return 0; } - } // namespace detail } // namespace ripple #endif namespace ripple { - // clang-format off // The configurable node sizes are "tiny", "small", "medium", "large", "huge" inline constexpr std::array>, 13> @@ -1007,6 +1000,23 @@ Config::loadFromString(std::string const& fileContents) "the maximum number of allowed peers (peers_max)"); } } + + if (!RUN_STANDALONE) + { + auto db_section = section(ConfigSection::nodeDatabase()); + if (auto type = get(db_section, "type", ""); type == "rwdb") + { + if (auto delete_interval = get(db_section, "online_delete", 0); + delete_interval == 0) + { + Throw( + "RWDB (in-memory backend) requires online_delete to " + "prevent OOM " + "Exception: standalone mode (used by tests) doesn't need " + "online_delete"); + } + } + } } boost::filesystem::path @@ -1071,5 +1081,4 @@ setup_FeeVote(Section const& section) } return setup; } - } // namespace ripple diff --git a/src/ripple/nodestore/backend/FlatmapFactory.cpp b/src/ripple/nodestore/backend/FlatmapFactory.cpp deleted file mode 100644 index 4cec115ef..000000000 --- a/src/ripple/nodestore/backend/FlatmapFactory.cpp +++ /dev/null @@ -1,235 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ripple { -namespace NodeStore { - -class FlatmapBackend : public Backend -{ -private: - std::string name_; - beast::Journal journal_; - bool isOpen_{false}; - - struct base_uint_hasher - { - using result_type = std::size_t; - - result_type - operator()(base_uint<256> const& value) const - { - return hardened_hash<>{}(value); - } - }; - - using DataStore = boost::unordered::concurrent_flat_map< - uint256, - std::vector, // Store compressed blob data - base_uint_hasher>; - - DataStore table_; - -public: - FlatmapBackend( - size_t keyBytes, - Section const& keyValues, - beast::Journal journal) - : name_(get(keyValues, "path")), journal_(journal) - { - boost::ignore_unused(journal_); - if (name_.empty()) - name_ = "node_db"; - } - - ~FlatmapBackend() override - { - close(); - } - - std::string - getName() override - { - return name_; - } - - void - open(bool createIfMissing) override - { - if (isOpen_) - Throw("already open"); - isOpen_ = true; - } - - bool - isOpen() override - { - return isOpen_; - } - - void - close() override - { - table_.clear(); - isOpen_ = false; - } - - Status - fetch(void const* key, std::shared_ptr* pObject) override - { - if (!isOpen_) - return notFound; - - uint256 const hash(uint256::fromVoid(key)); - - bool found = table_.visit(hash, [&](const auto& key_value_pair) { - nudb::detail::buffer bf; - auto const result = nodeobject_decompress( - key_value_pair.second.data(), key_value_pair.second.size(), bf); - DecodedBlob decoded(hash.data(), result.first, result.second); - if (!decoded.wasOk()) - { - *pObject = nullptr; - return; - } - *pObject = decoded.createObject(); - }); - return found ? (*pObject ? ok : dataCorrupt) : notFound; - } - - std::pair>, Status> - fetchBatch(std::vector const& hashes) override - { - std::vector> results; - results.reserve(hashes.size()); - for (auto const& h : hashes) - { - std::shared_ptr nObj; - Status status = fetch(h->begin(), &nObj); - if (status != ok) - results.push_back({}); - else - results.push_back(nObj); - } - return {results, ok}; - } - - void - store(std::shared_ptr const& object) override - { - if (!isOpen_) - return; - - if (!object) - return; - - EncodedBlob encoded(object); - nudb::detail::buffer bf; - auto const result = - nodeobject_compress(encoded.getData(), encoded.getSize(), bf); - - std::vector compressed( - static_cast(result.first), - static_cast(result.first) + result.second); - - table_.insert_or_assign(object->getHash(), std::move(compressed)); - } - - void - storeBatch(Batch const& batch) override - { - for (auto const& e : batch) - store(e); - } - - void - sync() override - { - } - - void - for_each(std::function)> f) override - { - if (!isOpen_) - return; - - table_.visit_all([&f](const auto& entry) { - nudb::detail::buffer bf; - auto const result = nodeobject_decompress( - entry.second.data(), entry.second.size(), bf); - DecodedBlob decoded( - entry.first.data(), result.first, result.second); - if (decoded.wasOk()) - f(decoded.createObject()); - }); - } - - int - getWriteLoad() override - { - return 0; - } - - void - setDeletePath() override - { - close(); - } - - int - fdRequired() const override - { - return 0; - } - -private: - size_t - size() const - { - return table_.size(); - } -}; - -class FlatmapFactory : public Factory -{ -public: - FlatmapFactory() - { - Manager::instance().insert(*this); - } - - ~FlatmapFactory() override - { - Manager::instance().erase(*this); - } - - std::string - getName() const override - { - return "Flatmap"; - } - - std::unique_ptr - createInstance( - size_t keyBytes, - Section const& keyValues, - std::size_t burstSize, - Scheduler& scheduler, - beast::Journal journal) override - { - return std::make_unique(keyBytes, keyValues, journal); - } -}; - -static FlatmapFactory flatmapFactory; - -} // namespace NodeStore -} // namespace ripple diff --git a/src/test/app/SHAMapStore_test.cpp b/src/test/app/SHAMapStore_test.cpp index 010c83a42..8a3ca0f89 100644 --- a/src/test/app/SHAMapStore_test.cpp +++ b/src/test/app/SHAMapStore_test.cpp @@ -216,6 +216,10 @@ public: } BEAST_EXPECT(store.getLastRotated() == lastRotated); + SQLiteDatabase* const db = + dynamic_cast(&env.app().getRelationalDatabase()); + BEAST_EXPECT(*db->getTransactionsMinLedgerSeq() == 3); + for (auto i = 3; i < deleteInterval + lastRotated; ++i) { ledgers.emplace( diff --git a/src/test/core/Config_test.cpp b/src/test/core/Config_test.cpp index 3d7991d74..dbf187fa6 100644 --- a/src/test/core/Config_test.cpp +++ b/src/test/core/Config_test.cpp @@ -1206,6 +1206,97 @@ r.ripple.com:51235 } } + void + testRWDBOnlineDelete() + { + testcase("RWDB online_delete validation"); + + // Test 1: RWDB without online_delete in standalone mode (should + // succeed) + { + Config c; + std::string toLoad = + "[node_db]\n" + "type=rwdb\n" + "path=main\n"; + c.setupControl(true, true, true); // standalone = true + try + { + c.loadFromString(toLoad); + pass(); // Should succeed + } + catch (std::runtime_error const& e) + { + fail("Should not throw in standalone mode"); + } + } + + // Test 2: RWDB without online_delete NOT in standalone mode (should + // throw) + { + Config c; + std::string toLoad = + "[node_db]\n" + "type=rwdb\n" + "path=main\n"; + c.setupControl(true, true, false); // standalone = false + try + { + c.loadFromString(toLoad); + fail("Expected exception for RWDB without online_delete"); + } + catch (std::runtime_error const& e) + { + BEAST_EXPECT( + std::string(e.what()).find( + "RWDB (in-memory backend) requires online_delete") != + std::string::npos); + pass(); + } + } + + // Test 3: RWDB with online_delete NOT in standalone mode (should + // succeed) + { + Config c; + std::string toLoad = + "[node_db]\n" + "type=rwdb\n" + "path=main\n" + "online_delete=256\n"; + c.setupControl(true, true, false); // standalone = false + try + { + c.loadFromString(toLoad); + pass(); // Should succeed + } + catch (std::runtime_error const& e) + { + fail("Should not throw when online_delete is configured"); + } + } + + // Test 4: Non-RWDB without online_delete NOT in standalone mode (should + // succeed) + { + Config c; + std::string toLoad = + "[node_db]\n" + "type=NuDB\n" + "path=main\n"; + c.setupControl(true, true, false); // standalone = false + try + { + c.loadFromString(toLoad); + pass(); // Should succeed + } + catch (std::runtime_error const& e) + { + fail("Should not throw for non-RWDB backends"); + } + } + } + void testOverlay() { @@ -1295,6 +1386,7 @@ r.ripple.com:51235 testComments(); testGetters(); testAmendment(); + testRWDBOnlineDelete(); testOverlay(); testNetworkID(); } diff --git a/src/test/rdb/RelationalDatabase_test.cpp b/src/test/rdb/RelationalDatabase_test.cpp new file mode 100644 index 000000000..8f4ea3907 --- /dev/null +++ b/src/test/rdb/RelationalDatabase_test.cpp @@ -0,0 +1,756 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2025 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include +#include +#include +#include +#include + +namespace ripple { +namespace test { + +class RelationalDatabase_test : public beast::unit_test::suite +{ +private: + // Helper to get SQLiteDatabase* (works for both SQLite and RWDB since RWDB + // inherits from SQLiteDatabase) + static SQLiteDatabase* + getInterface(Application& app) + { + return dynamic_cast(&app.getRelationalDatabase()); + } + + static SQLiteDatabase* + getInterface(RelationalDatabase& db) + { + return dynamic_cast(&db); + } + + static std::unique_ptr + makeConfig(std::string const& backend) + { + auto config = test::jtx::envconfig(); + // Sqlite backend doesn't need a database_path as it will just use + // in-memory databases when in standalone mode anyway. + config->overwrite(SECTION_RELATIONAL_DB, "backend", backend); + return config; + } + +public: + RelationalDatabase_test() = default; + + void + testBasicInitialization( + std::string const& backend, + std::unique_ptr config) + { + testcase("Basic initialization and empty database - " + backend); + + using namespace test::jtx; + Env env(*this, std::move(config)); + auto& db = env.app().getRelationalDatabase(); + + // Test empty database state + BEAST_EXPECT(db.getMinLedgerSeq() == 2); + BEAST_EXPECT(db.getMaxLedgerSeq() == 2); + BEAST_EXPECT(db.getNewestLedgerInfo()->seq == 2); + + auto* sqliteDb = getInterface(db); + BEAST_EXPECT(sqliteDb != nullptr); + + if (sqliteDb) + { + BEAST_EXPECT(!sqliteDb->getTransactionsMinLedgerSeq().has_value()); + BEAST_EXPECT( + !sqliteDb->getAccountTransactionsMinLedgerSeq().has_value()); + + auto ledgerCount = sqliteDb->getLedgerCountMinMax(); + BEAST_EXPECT(ledgerCount.numberOfRows == 1); + BEAST_EXPECT(ledgerCount.minLedgerSequence == 2); + BEAST_EXPECT(ledgerCount.maxLedgerSequence == 2); + } + } + + void + testLedgerSequenceOperations( + std::string const& backend, + std::unique_ptr config) + { + testcase("Ledger sequence operations - " + backend); + + using namespace test::jtx; + config->LEDGER_HISTORY = 1000; + + Env env(*this, std::move(config)); + auto& db = env.app().getRelationalDatabase(); + + // Create initial ledger + Account alice("alice"); + env.fund(XRP(10000), alice); + env.close(); + + // Test basic sequence operations + auto minSeq = db.getMinLedgerSeq(); + auto maxSeq = db.getMaxLedgerSeq(); + + BEAST_EXPECT(minSeq.has_value()); + BEAST_EXPECT(maxSeq.has_value()); + BEAST_EXPECT(*minSeq == 2); + BEAST_EXPECT(*maxSeq == 3); + + // Create more ledgers + env(pay(alice, Account("bob"), XRP(1000))); + env.close(); + + env(pay(alice, Account("carol"), XRP(500))); + env.close(); + + // Verify sequence updates + minSeq = db.getMinLedgerSeq(); + maxSeq = db.getMaxLedgerSeq(); + + BEAST_EXPECT(*minSeq == 2); + BEAST_EXPECT(*maxSeq == 5); + + auto* sqliteDb = getInterface(db); + if (sqliteDb) + { + auto ledgerCount = sqliteDb->getLedgerCountMinMax(); + BEAST_EXPECT(ledgerCount.numberOfRows == 4); + BEAST_EXPECT(ledgerCount.minLedgerSequence == 2); + BEAST_EXPECT(ledgerCount.maxLedgerSequence == 5); + } + } + + void + testLedgerInfoOperations( + std::string const& backend, + std::unique_ptr config) + { + testcase("Ledger info retrieval operations - " + backend); + + using namespace test::jtx; + config->LEDGER_HISTORY = 1000; + + Env env(*this, std::move(config)); + auto* db = getInterface(env.app()); + + Account alice("alice"); + env.fund(XRP(10000), alice); + env.close(); + + // Test getNewestLedgerInfo + auto newestLedger = db->getNewestLedgerInfo(); + BEAST_EXPECT(newestLedger.has_value()); + BEAST_EXPECT(newestLedger->seq == 3); + + // Test getLedgerInfoByIndex + auto ledgerByIndex = db->getLedgerInfoByIndex(3); + BEAST_EXPECT(ledgerByIndex.has_value()); + BEAST_EXPECT(ledgerByIndex->seq == 3); + BEAST_EXPECT(ledgerByIndex->hash == newestLedger->hash); + + // Test getLedgerInfoByHash + auto ledgerByHash = db->getLedgerInfoByHash(newestLedger->hash); + BEAST_EXPECT(ledgerByHash.has_value()); + BEAST_EXPECT(ledgerByHash->seq == 3); + BEAST_EXPECT(ledgerByHash->hash == newestLedger->hash); + + // Test getLimitedOldestLedgerInfo + auto oldestLedger = db->getLimitedOldestLedgerInfo(2); + BEAST_EXPECT(oldestLedger.has_value()); + BEAST_EXPECT(oldestLedger->seq == 2); + + // Test getLimitedNewestLedgerInfo + auto limitedNewest = db->getLimitedNewestLedgerInfo(2); + BEAST_EXPECT(limitedNewest.has_value()); + BEAST_EXPECT(limitedNewest->seq == 3); + + // Test invalid queries + auto invalidLedger = db->getLedgerInfoByIndex(999); + BEAST_EXPECT(!invalidLedger.has_value()); + + uint256 invalidHash; + auto invalidHashLedger = db->getLedgerInfoByHash(invalidHash); + BEAST_EXPECT(!invalidHashLedger.has_value()); + } + + void + testHashOperations( + std::string const& backend, + std::unique_ptr config) + { + testcase("Hash retrieval operations - " + backend); + + using namespace test::jtx; + config->LEDGER_HISTORY = 1000; + + Env env(*this, std::move(config)); + auto& db = env.app().getRelationalDatabase(); + + Account alice("alice"); + env.fund(XRP(10000), alice); + env.close(); + + env(pay(alice, Account("bob"), XRP(1000))); + env.close(); + + // Test getHashByIndex + auto hash1 = db.getHashByIndex(3); + auto hash2 = db.getHashByIndex(4); + + BEAST_EXPECT(hash1 != uint256()); + BEAST_EXPECT(hash2 != uint256()); + BEAST_EXPECT(hash1 != hash2); + + // Test getHashesByIndex (single) + auto hashPair = db.getHashesByIndex(4); + BEAST_EXPECT(hashPair.has_value()); + BEAST_EXPECT(hashPair->ledgerHash == hash2); + BEAST_EXPECT(hashPair->parentHash == hash1); + + // Test getHashesByIndex (range) + auto hashRange = db.getHashesByIndex(3, 4); + BEAST_EXPECT(hashRange.size() == 2); + BEAST_EXPECT(hashRange[3].ledgerHash == hash1); + BEAST_EXPECT(hashRange[4].ledgerHash == hash2); + BEAST_EXPECT(hashRange[4].parentHash == hash1); + + // Test invalid hash queries + auto invalidHash = db.getHashByIndex(999); + BEAST_EXPECT(invalidHash == uint256()); + + auto invalidHashPair = db.getHashesByIndex(999); + BEAST_EXPECT(!invalidHashPair.has_value()); + + auto emptyRange = db.getHashesByIndex(10, 5); // max < min + BEAST_EXPECT(emptyRange.empty()); + } + + void + testTransactionOperations( + std::string const& backend, + std::unique_ptr config) + { + testcase("Transaction storage and retrieval - " + backend); + + using namespace test::jtx; + config->LEDGER_HISTORY = 1000; + + Env env(*this, std::move(config)); + auto& db = env.app().getRelationalDatabase(); + + Account alice("alice"); + Account bob("bob"); + + env.fund(XRP(10000), alice, bob); + env.close(); + + auto* sqliteDb = getInterface(db); + BEAST_EXPECT(sqliteDb != nullptr); + + if (!sqliteDb) + return; + + // Test initial transaction counts after funding + auto initialTxCount = sqliteDb->getTransactionCount(); + auto initialAcctTxCount = sqliteDb->getAccountTransactionCount(); + + BEAST_EXPECT(initialTxCount == 4); + BEAST_EXPECT(initialAcctTxCount == 6); + + // Create transactions + env(pay(alice, bob, XRP(1000))); + env.close(); + + env(pay(bob, alice, XRP(500))); + env.close(); + + // Test transaction counts after creation + auto txCount = sqliteDb->getTransactionCount(); + auto acctTxCount = sqliteDb->getAccountTransactionCount(); + + BEAST_EXPECT(txCount == 6); + BEAST_EXPECT(acctTxCount == 10); + + // Test transaction retrieval + uint256 invalidTxId; + error_code_i ec; + auto invalidTxResult = + sqliteDb->getTransaction(invalidTxId, std::nullopt, ec); + BEAST_EXPECT(std::holds_alternative(invalidTxResult)); + + // Test transaction history + auto txHistory = db.getTxHistory(0); + + BEAST_EXPECT(!txHistory.empty()); + BEAST_EXPECT(txHistory.size() == 6); + + // Test with valid transaction range + auto minSeq = sqliteDb->getTransactionsMinLedgerSeq(); + auto maxSeq = db.getMaxLedgerSeq(); + + if (minSeq && maxSeq) + { + ClosedInterval range(*minSeq, *maxSeq); + auto rangeResult = sqliteDb->getTransaction(invalidTxId, range, ec); + auto searched = std::get(rangeResult); + BEAST_EXPECT( + searched == TxSearched::all || searched == TxSearched::some); + } + } + + void + testAccountTransactionOperations( + std::string const& backend, + std::unique_ptr config) + { + testcase("Account transaction operations - " + backend); + + using namespace test::jtx; + config->LEDGER_HISTORY = 1000; + + Env env(*this, std::move(config)); + auto& db = env.app().getRelationalDatabase(); + + Account alice("alice"); + Account bob("bob"); + Account carol("carol"); + + env.fund(XRP(10000), alice, bob, carol); + env.close(); + + auto* sqliteDb = getInterface(db); + BEAST_EXPECT(sqliteDb != nullptr); + + if (!sqliteDb) + return; + + // Create multiple transactions involving alice + env(pay(alice, bob, XRP(1000))); + env.close(); + + env(pay(bob, alice, XRP(500))); + env.close(); + + env(pay(alice, carol, XRP(250))); + env.close(); + + auto minSeq = db.getMinLedgerSeq(); + auto maxSeq = db.getMaxLedgerSeq(); + + if (!minSeq || !maxSeq) + return; + + // Test getOldestAccountTxs + RelationalDatabase::AccountTxOptions options{ + alice.id(), *minSeq, *maxSeq, 0, 10, false}; + + auto oldestTxs = sqliteDb->getOldestAccountTxs(options); + BEAST_EXPECT(oldestTxs.size() == 5); + + // Test getNewestAccountTxs + auto newestTxs = sqliteDb->getNewestAccountTxs(options); + BEAST_EXPECT(newestTxs.size() == 5); + + // Test binary format versions + auto oldestTxsB = sqliteDb->getOldestAccountTxsB(options); + BEAST_EXPECT(oldestTxsB.size() == 5); + + auto newestTxsB = sqliteDb->getNewestAccountTxsB(options); + BEAST_EXPECT(newestTxsB.size() == 5); + + // Test with limit + options.limit = 1; + auto limitedTxs = sqliteDb->getOldestAccountTxs(options); + BEAST_EXPECT(limitedTxs.size() == 1); + + // Test with offset + options.limit = 10; + options.offset = 1; + auto offsetTxs = sqliteDb->getOldestAccountTxs(options); + BEAST_EXPECT(offsetTxs.size() == 4); + + // Test with invalid account + { + Account invalidAccount("invalid"); + RelationalDatabase::AccountTxOptions invalidOptions{ + invalidAccount.id(), *minSeq, *maxSeq, 0, 10, false}; + auto invalidAccountTxs = + sqliteDb->getOldestAccountTxs(invalidOptions); + BEAST_EXPECT(invalidAccountTxs.empty()); + } + } + + void + testAccountTransactionPaging( + std::string const& backend, + std::unique_ptr config) + { + testcase("Account transaction paging operations - " + backend); + + using namespace test::jtx; + config->LEDGER_HISTORY = 1000; + + Env env(*this, std::move(config)); + auto& db = env.app().getRelationalDatabase(); + + Account alice("alice"); + Account bob("bob"); + + env.fund(XRP(10000), alice, bob); + env.close(); + + auto* sqliteDb = getInterface(db); + BEAST_EXPECT(sqliteDb != nullptr); + if (!sqliteDb) + return; + + // Create multiple transactions for paging + for (int i = 0; i < 5; ++i) + { + env(pay(alice, bob, XRP(100 + i))); + env.close(); + } + + auto minSeq = db.getMinLedgerSeq(); + auto maxSeq = db.getMaxLedgerSeq(); + + if (!minSeq || !maxSeq) + return; + + RelationalDatabase::AccountTxPageOptions pageOptions{ + alice.id(), *minSeq, *maxSeq, std::nullopt, 2, false}; + + // Test oldestAccountTxPage + auto [oldestPage, oldestMarker] = + sqliteDb->oldestAccountTxPage(pageOptions); + + BEAST_EXPECT(oldestPage.size() == 2); + BEAST_EXPECT(oldestMarker.has_value() == true); + + // Test newestAccountTxPage + auto [newestPage, newestMarker] = + sqliteDb->newestAccountTxPage(pageOptions); + + BEAST_EXPECT(newestPage.size() == 2); + BEAST_EXPECT(newestMarker.has_value() == true); + + // Test binary versions + auto [oldestPageB, oldestMarkerB] = + sqliteDb->oldestAccountTxPageB(pageOptions); + BEAST_EXPECT(oldestPageB.size() == 2); + + auto [newestPageB, newestMarkerB] = + sqliteDb->newestAccountTxPageB(pageOptions); + BEAST_EXPECT(newestPageB.size() == 2); + + // Test with marker continuation + if (oldestMarker.has_value()) + { + pageOptions.marker = oldestMarker; + auto [continuedPage, continuedMarker] = + sqliteDb->oldestAccountTxPage(pageOptions); + BEAST_EXPECT(continuedPage.size() == 2); + } + } + + void + testDeletionOperations( + std::string const& backend, + std::unique_ptr config) + { + testcase("Deletion operations - " + backend); + + using namespace test::jtx; + config->LEDGER_HISTORY = 1000; + + Env env(*this, std::move(config)); + auto& db = env.app().getRelationalDatabase(); + + Account alice("alice"); + Account bob("bob"); + + env.fund(XRP(10000), alice, bob); + env.close(); + + auto* sqliteDb = getInterface(db); + BEAST_EXPECT(sqliteDb != nullptr); + if (!sqliteDb) + return; + + // Create multiple ledgers and transactions + for (int i = 0; i < 3; ++i) + { + env(pay(alice, bob, XRP(100 + i))); + env.close(); + } + + auto initialTxCount = sqliteDb->getTransactionCount(); + BEAST_EXPECT(initialTxCount == 7); + auto initialAcctTxCount = sqliteDb->getAccountTransactionCount(); + BEAST_EXPECT(initialAcctTxCount == 12); + auto initialLedgerCount = sqliteDb->getLedgerCountMinMax(); + BEAST_EXPECT(initialLedgerCount.numberOfRows == 5); + + auto maxSeq = db.getMaxLedgerSeq(); + if (!maxSeq || *maxSeq <= 2) + return; + + // Test deleteTransactionByLedgerSeq + sqliteDb->deleteTransactionByLedgerSeq(*maxSeq); + auto txCountAfterDelete = sqliteDb->getTransactionCount(); + BEAST_EXPECT(txCountAfterDelete == 6); + + // Test deleteTransactionsBeforeLedgerSeq + sqliteDb->deleteTransactionsBeforeLedgerSeq(*maxSeq - 1); + auto txCountAfterBulkDelete = sqliteDb->getTransactionCount(); + BEAST_EXPECT(txCountAfterBulkDelete == 1); + + // Test deleteAccountTransactionsBeforeLedgerSeq + sqliteDb->deleteAccountTransactionsBeforeLedgerSeq(*maxSeq - 1); + auto acctTxCountAfterDelete = sqliteDb->getAccountTransactionCount(); + BEAST_EXPECT(acctTxCountAfterDelete == 4); + + // Test deleteBeforeLedgerSeq + auto minSeq = db.getMinLedgerSeq(); + if (minSeq) + { + sqliteDb->deleteBeforeLedgerSeq(*minSeq + 1); + auto ledgerCountAfterDelete = sqliteDb->getLedgerCountMinMax(); + BEAST_EXPECT(ledgerCountAfterDelete.numberOfRows == 4); + } + } + + void + testDatabaseSpaceOperations( + std::string const& backend, + std::unique_ptr config) + { + testcase("Database space and size operations - " + backend); + + using namespace test::jtx; + Env env(*this, std::move(config)); + auto& db = env.app().getRelationalDatabase(); + + auto* sqliteDb = getInterface(db); + BEAST_EXPECT(sqliteDb != nullptr); + if (!sqliteDb) + return; + + // Test size queries + auto allKB = sqliteDb->getKBUsedAll(); + auto ledgerKB = sqliteDb->getKBUsedLedger(); + auto txKB = sqliteDb->getKBUsedTransaction(); + + if (backend == "rwdb") + { + // RWDB reports actual data memory (rounded down to KB) + // Initially should be < 1KB, so rounds down to 0 + // Note: These are 0 due to rounding, not because there's literally + // no data + BEAST_EXPECT(allKB == 0); // < 1024 bytes rounds to 0 KB + BEAST_EXPECT(ledgerKB == 0); // < 1024 bytes rounds to 0 KB + BEAST_EXPECT(txKB == 0); // < 1024 bytes rounds to 0 KB + } + else + { + // SQLite reports cache/engine memory which has overhead even when + // empty Just verify the functions return reasonable values + BEAST_EXPECT(allKB >= 0); + BEAST_EXPECT(ledgerKB >= 0); + BEAST_EXPECT(txKB >= 0); + } + + // Create some data and verify size increases + Account alice("alice"); + env.fund(XRP(10000), alice); + env.close(); + + auto newAllKB = sqliteDb->getKBUsedAll(); + auto newLedgerKB = sqliteDb->getKBUsedLedger(); + auto newTxKB = sqliteDb->getKBUsedTransaction(); + + if (backend == "rwdb") + { + // RWDB reports actual data memory + // After adding data, should see some increase + BEAST_EXPECT(newAllKB >= 1); // Should have at least 1KB total + BEAST_EXPECT( + newTxKB >= 0); // Transactions added (might still be < 1KB) + BEAST_EXPECT( + newLedgerKB >= 0); // Ledger data (might still be < 1KB) + + // Key relationships + BEAST_EXPECT(newAllKB >= newLedgerKB + newTxKB); // Total >= parts + BEAST_EXPECT(newAllKB >= allKB); // Should increase or stay same + BEAST_EXPECT(newTxKB >= txKB); // Should increase or stay same + } + else + { + // SQLite: Memory usage should not decrease after adding data + // Values might increase due to cache growth + BEAST_EXPECT(newAllKB >= allKB); + BEAST_EXPECT(newLedgerKB >= ledgerKB); + BEAST_EXPECT(newTxKB >= txKB); + + // SQLite's getKBUsedAll is global memory, should be >= parts + BEAST_EXPECT(newAllKB >= newLedgerKB); + BEAST_EXPECT(newAllKB >= newTxKB); + } + + // Test space availability + // Both SQLite and RWDB use in-memory databases in standalone mode, + // so file-based space checks don't apply to either backend. + // Skip these checks for both. + + // if (backend == "rwdb") + // { + // BEAST_EXPECT(db.ledgerDbHasSpace(env.app().config())); + // BEAST_EXPECT(db.transactionDbHasSpace(env.app().config())); + // } + + // Test database closure operations (should not throw) + try + { + sqliteDb->closeLedgerDB(); + sqliteDb->closeTransactionDB(); + } + catch (std::exception const& e) + { + BEAST_EXPECT(false); // Should not throw + } + } + + void + testTransactionMinLedgerSeq( + std::string const& backend, + std::unique_ptr config) + { + testcase("Transaction minimum ledger sequence tracking - " + backend); + + using namespace test::jtx; + config->LEDGER_HISTORY = 1000; + + Env env(*this, std::move(config)); + auto& db = env.app().getRelationalDatabase(); + + auto* sqliteDb = getInterface(db); + BEAST_EXPECT(sqliteDb != nullptr); + if (!sqliteDb) + return; + + // Initially should have no transactions + BEAST_EXPECT(!sqliteDb->getTransactionsMinLedgerSeq().has_value()); + BEAST_EXPECT( + !sqliteDb->getAccountTransactionsMinLedgerSeq().has_value()); + + Account alice("alice"); + Account bob("bob"); + + env.fund(XRP(10000), alice, bob); + env.close(); + + // Create first transaction + env(pay(alice, bob, XRP(1000))); + env.close(); + + auto txMinSeq = sqliteDb->getTransactionsMinLedgerSeq(); + auto acctTxMinSeq = sqliteDb->getAccountTransactionsMinLedgerSeq(); + BEAST_EXPECT(txMinSeq.has_value()); + BEAST_EXPECT(acctTxMinSeq.has_value()); + BEAST_EXPECT(*txMinSeq == 3); + BEAST_EXPECT(*acctTxMinSeq == 3); + + // Create more transactions + env(pay(bob, alice, XRP(500))); + env.close(); + + env(pay(alice, bob, XRP(250))); + env.close(); + + // Min sequences should remain the same (first transaction ledger) + auto newTxMinSeq = sqliteDb->getTransactionsMinLedgerSeq(); + auto newAcctTxMinSeq = sqliteDb->getAccountTransactionsMinLedgerSeq(); + BEAST_EXPECT(newTxMinSeq == txMinSeq); + BEAST_EXPECT(newAcctTxMinSeq == acctTxMinSeq); + } + + std::vector static getBackends(std::string const& unittest_arg) + { + // Valid backends + static const std::set validBackends = {"sqlite", "rwdb"}; + + // Default to all valid backends if no arg specified + if (unittest_arg.empty()) + return {validBackends.begin(), validBackends.end()}; + + std::set backends; // Use set to avoid duplicates + std::stringstream ss(unittest_arg); + std::string backend; + + while (std::getline(ss, backend, ',')) + { + if (!backend.empty()) + { + // Validate backend + if (validBackends.contains(backend)) + { + backends.insert(backend); + } + } + } + + // Return as vector (sorted due to set) + return {backends.begin(), backends.end()}; + } + + void + run() override + { + auto backends = getBackends(arg()); + + if (backends.empty()) + { + fail("no valid backend specified: '" + arg() + "'"); + } + + for (auto const& backend : backends) + { + testBasicInitialization(backend, makeConfig(backend)); + testLedgerSequenceOperations(backend, makeConfig(backend)); + testLedgerInfoOperations(backend, makeConfig(backend)); + testHashOperations(backend, makeConfig(backend)); + testTransactionOperations(backend, makeConfig(backend)); + testAccountTransactionOperations(backend, makeConfig(backend)); + testAccountTransactionPaging(backend, makeConfig(backend)); + testDeletionOperations(backend, makeConfig(backend)); + testDatabaseSpaceOperations(backend, makeConfig(backend)); + testTransactionMinLedgerSeq(backend, makeConfig(backend)); + } + } +}; + +BEAST_DEFINE_TESTSUITE(RelationalDatabase, rdb, ripple); + +} // namespace test +} // namespace ripple \ No newline at end of file