From 4239124750cc7cfa10a29cc7ab7233474d237373 Mon Sep 17 00:00:00 2001 From: tequ Date: Fri, 25 Oct 2024 08:10:45 +0900 Subject: [PATCH 01/11] Update amendments for rippled-standalone.cfg (#385) --- cfg/rippled-standalone.cfg | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cfg/rippled-standalone.cfg b/cfg/rippled-standalone.cfg index 79c5c9004..3301299ca 100755 --- a/cfg/rippled-standalone.cfg +++ b/cfg/rippled-standalone.cfg @@ -144,4 +144,12 @@ D686F2538F410C9D0D856788E98E3579595DAF7B38D38887F81ECAC934B06040 HooksUpdate1 86E83A7D2ECE3AD5FA87AB2195AE015C950469ABF0B72EAACED318F74886AE90 CryptoConditionsSuite 3C43D9A973AA4443EF3FC38E42DD306160FBFFDAB901CD8BAA15D09F2597EB87 NonFungibleTokensV1 0285B7E5E08E1A8E4C15636F0591D87F73CB6A7B6452A932AD72BBC8E5D1CBE3 fixNFTokenDirV1 -36799EA497B1369B170805C078AEFE6188345F9B3E324C21E9CA3FF574E3C3D6 fixNFTokenNegOffer \ No newline at end of file +36799EA497B1369B170805C078AEFE6188345F9B3E324C21E9CA3FF574E3C3D6 fixNFTokenNegOffer +4C499D17719BB365B69010A436B64FD1A82AAB199FC1CEB06962EBD01059FB09 fixXahauV1 +215181D23BF5C173314B5FDB9C872C92DE6CC918483727DE037C0C13E7E6EE9D fixXahauV2 +0D8BF22FF7570D58598D1EF19EBB6E142AD46E59A223FD3816262FBB69345BEA Remit +7CA0426E7F411D39BB014E57CD9E08F61DE1750F0D41FCD428D9FB80BB7596B0 ZeroB2M +4B8466415FAB32FFA89D9DCBE166A42340115771DF611A7160F8D7439C87ECD8 fixNSDelete +EDB4EE4C524E16BDD91D9A529332DED08DCAAA51CC6DC897ACFA1A0ED131C5B6 fix240819 +8063140E9260799D6716756B891CEC3E7006C4E4F277AB84670663A88F94B9C4 fixPageCap +88693F108C3CD8A967F3F4253A32DEF5E35F9406ACD2A11B88B11D90865763A9 fix240911 From 287c01ad0427410b81ede73946eee8e1dc1a9c67 Mon Sep 17 00:00:00 2001 From: Wietse Wind Date: Fri, 25 Oct 2024 02:10:14 +0200 Subject: [PATCH 02/11] Improve Admin command RPC Post (#384) * Improve ADMIN HTTP POST RPC notifications: no queue limit, shorter HTTP call TTL --- src/ripple/net/impl/RPCCall.cpp | 4 +++- src/ripple/net/impl/RPCSub.cpp | 17 ++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/ripple/net/impl/RPCCall.cpp b/src/ripple/net/impl/RPCCall.cpp index f11f831e0..cb63d997c 100644 --- a/src/ripple/net/impl/RPCCall.cpp +++ b/src/ripple/net/impl/RPCCall.cpp @@ -1894,7 +1894,9 @@ fromNetwork( constexpr auto RPC_REPLY_MAX_BYTES = megabytes(256); using namespace std::chrono_literals; - auto constexpr RPC_NOTIFY = 10min; + // auto constexpr RPC_NOTIFY = 10min; // Wietse: lolwut 10 minutes for one + // HTTP call? + auto constexpr RPC_NOTIFY = 30s; HTTPClient::request( bSSL, diff --git a/src/ripple/net/impl/RPCSub.cpp b/src/ripple/net/impl/RPCSub.cpp index 8b052e817..5423a1132 100644 --- a/src/ripple/net/impl/RPCSub.cpp +++ b/src/ripple/net/impl/RPCSub.cpp @@ -78,12 +78,14 @@ public: { std::lock_guard sl(mLock); - if (mDeque.size() >= eventQueueMax) - { - // Drop the previous event. - JLOG(j_.warn()) << "RPCCall::fromNetwork drop"; - mDeque.pop_back(); - } + // Wietse: we're not going to limit this, this is admin-port only, scale + // accordingly Dropping events just like this results in inconsistent + // data on the receiving end if (mDeque.size() >= eventQueueMax) + // { + // // Drop the previous event. + // JLOG(j_.warn()) << "RPCCall::fromNetwork drop"; + // mDeque.pop_back(); + // } auto jm = broadcast ? j_.debug() : j_.info(); JLOG(jm) << "RPCCall::fromNetwork push: " << jvObj; @@ -182,7 +184,8 @@ private: } private: - enum { eventQueueMax = 32 }; + // Wietse: we're not going to limit this, this is admin-port only, scale + // accordingly enum { eventQueueMax = 32 }; boost::asio::io_service& m_io_service; JobQueue& m_jobQueue; From 766f5d7ee1d04085823e7b8e30f30a8977ea7ec9 Mon Sep 17 00:00:00 2001 From: tequ Date: Fri, 25 Oct 2024 09:10:43 +0900 Subject: [PATCH 03/11] Update macro.h (#366) --- hook/macro.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hook/macro.h b/hook/macro.h index 38a534bfc..838c2b55c 100644 --- a/hook/macro.h +++ b/hook/macro.h @@ -1,5 +1,5 @@ /** - * These are helper macros for writing hooks, all of them are optional as is including hookmacro.h at all + * These are helper macros for writing hooks, all of them are optional as is including macro.h at all */ #include From 08f13b7cfeed4ccdf6d7aba1dbd08feac65c73ee Mon Sep 17 00:00:00 2001 From: Wietse Wind Date: Fri, 25 Oct 2024 03:13:42 +0200 Subject: [PATCH 04/11] Fix account_tx sluggishness as per https://github.com/XRPLF/rippled/commit/2e9261cb (#308) --- src/ripple/app/rdb/backend/detail/impl/Node.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/ripple/app/rdb/backend/detail/impl/Node.cpp b/src/ripple/app/rdb/backend/detail/impl/Node.cpp index 11c43191a..c80038ef7 100644 --- a/src/ripple/app/rdb/backend/detail/impl/Node.cpp +++ b/src/ripple/app/rdb/backend/detail/impl/Node.cpp @@ -1132,7 +1132,7 @@ accountTxPage( { sql = boost::str( boost::format( - prefix + (R"(AccountTransactions.LedgerSeq BETWEEN '%u' AND '%u' + prefix + (R"(AccountTransactions.LedgerSeq BETWEEN %u AND %u ORDER BY AccountTransactions.LedgerSeq %s, AccountTransactions.TxnSeq %s LIMIT %u;)")) % @@ -1155,12 +1155,14 @@ accountTxPage( FROM AccountTransactions, Transactions WHERE (AccountTransactions.TransID = Transactions.TransID AND AccountTransactions.Account = '%s' AND - AccountTransactions.LedgerSeq BETWEEN '%u' AND '%u') - OR + AccountTransactions.LedgerSeq BETWEEN %u AND %u) + UNION + SELECT AccountTransactions.LedgerSeq,AccountTransactions.TxnSeq,Status,RawTxn,TxnMeta + FROM AccountTransactions, Transactions WHERE (AccountTransactions.TransID = Transactions.TransID AND AccountTransactions.Account = '%s' AND - AccountTransactions.LedgerSeq = '%u' AND - AccountTransactions.TxnSeq %s '%u') + AccountTransactions.LedgerSeq = %u AND + AccountTransactions.TxnSeq %s %u) ORDER BY AccountTransactions.LedgerSeq %s, AccountTransactions.TxnSeq %s LIMIT %u; From 6b26045cbce373388c8cd7603968d5612f531c08 Mon Sep 17 00:00:00 2001 From: Denis Angell Date: Fri, 25 Oct 2024 03:56:16 +0200 Subject: [PATCH 05/11] Update settings.json (#342) --- .vscode/settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 9e4544373..1642d6324 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,7 +3,7 @@ "C_Cpp.clang_format_path": ".clang-format", "C_Cpp.clang_format_fallbackStyle": "{ ColumnLimit: 0 }", "[cpp]":{ - "editor.wordBasedSuggestions": false, + "editor.wordBasedSuggestions": "off", "editor.suggest.insertMode": "replace", "editor.semanticHighlighting.enabled": true, "editor.tabSize": 4, From 8cffd3054d1d78970c80faea5c6a425a6766e675 Mon Sep 17 00:00:00 2001 From: tequ Date: Sat, 9 Nov 2024 13:00:59 +0900 Subject: [PATCH 06/11] add trace message to exception on etxn_fee_base (#387) --- src/ripple/app/hook/impl/applyHook.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ripple/app/hook/impl/applyHook.cpp b/src/ripple/app/hook/impl/applyHook.cpp index bf5c9f160..2afd018e8 100644 --- a/src/ripple/app/hook/impl/applyHook.cpp +++ b/src/ripple/app/hook/impl/applyHook.cpp @@ -4617,6 +4617,8 @@ DEFINE_HOOK_FUNCTION( } catch (std::exception& e) { + JLOG(j.trace()) << "HookInfo[" << HC_ACC() + << "]: etxn_fee_base exception: " << e.what(); return INVALID_TXN; } From 8ba37a3138dc6ffabd813693900cd304031454e8 Mon Sep 17 00:00:00 2001 From: Denis Angell Date: Sat, 9 Nov 2024 05:17:49 +0100 Subject: [PATCH 07/11] Add Script for SfCode generation (#358) Co-authored-by: RichardAH Co-authored-by: tequ --- hook/generate_sfcodes.sh | 29 +++++++++++++++++++++++++++++ hook/sfcodes.h | 25 ++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100755 hook/generate_sfcodes.sh diff --git a/hook/generate_sfcodes.sh b/hook/generate_sfcodes.sh new file mode 100755 index 000000000..b651e4c65 --- /dev/null +++ b/hook/generate_sfcodes.sh @@ -0,0 +1,29 @@ +#/bin/bash +RIPPLED_ROOT="../src/ripple" +echo '// For documentation please see: https://xrpl-hooks.readme.io/reference/' +echo '// Generated using generate_sfcodes.sh' +cat $RIPPLED_ROOT/protocol/impl/SField.cpp | grep -E '^CONSTRUCT_' | + sed 's/UINT16/1/g' | + sed 's/UINT32/2/g' | + sed 's/UINT64/3/g' | + sed 's/HASH128/4/g' | + sed 's/HASH256/5/g' | + sed 's/UINT128/4/g' | + sed 's/UINT256/5/g' | + sed 's/AMOUNT/6/g' | + sed 's/VL/7/g' | sed 's/Import7/ImportVL/g' | + sed 's/ACCOUNT/8/g' | + sed 's/OBJECT/14/g' | + sed 's/ARRAY/15/g' | + sed 's/UINT8/16/g' | + sed 's/HASH160/17/g' | + sed 's/UINT160/17/g' | + sed 's/PATHSET/18/g' | + sed 's/VECTOR256/19/g' | + sed 's/UINT96/20/g' | + sed 's/UINT192/21/g' | + sed 's/UINT384/22/g' | + sed 's/UINT512/23/g' | + grep -Eo '"([^"]+)", *([0-9]+), *([0-9]+)' | + sed 's/"//g' | sed 's/ *//g' | sed 's/,/ /g' | + awk '{print ("#define sf"$1" (("$2"U << 16U) + "$3"U)")}' diff --git a/hook/sfcodes.h b/hook/sfcodes.h index 86972c236..a88278d23 100644 --- a/hook/sfcodes.h +++ b/hook/sfcodes.h @@ -60,7 +60,10 @@ #define sfBurnedNFTokens ((2U << 16U) + 44U) #define sfHookStateCount ((2U << 16U) + 45U) #define sfEmitGeneration ((2U << 16U) + 46U) -#define sfLockCount ((2U << 16U) + 47U) +#define sfLockCount ((2U << 16U) + 49U) +#define sfFirstNFTokenSequence ((2U << 16U) + 50U) +#define sfXahauActivationLgrSeq ((2U << 16U) + 96U) +#define sfImportSequence ((2U << 16U) + 97U) #define sfRewardTime ((2U << 16U) + 98U) #define sfRewardLgrFirst ((2U << 16U) + 99U) #define sfRewardLgrLast ((2U << 16U) + 100U) @@ -80,6 +83,8 @@ #define sfHookInstructionCount ((3U << 16U) + 17U) #define sfHookReturnCode ((3U << 16U) + 18U) #define sfReferenceCount ((3U << 16U) + 19U) +#define sfAccountIndex ((3U << 16U) + 98U) +#define sfAccountCount ((3U << 16U) + 99U) #define sfRewardAccumulator ((3U << 16U) + 100U) #define sfEmailHash ((4U << 16U) + 1U) #define sfTakerPaysCurrency ((10U << 16U) + 1U) @@ -120,6 +125,9 @@ #define sfOfferID ((5U << 16U) + 34U) #define sfEscrowID ((5U << 16U) + 35U) #define sfURITokenID ((5U << 16U) + 36U) +#define sfGovernanceFlags ((5U << 16U) + 99U) +#define sfGovernanceMarks ((5U << 16U) + 98U) +#define sfEmittedTxnID ((5U << 16U) + 97U) #define sfAmount ((6U << 16U) + 1U) #define sfBalance ((6U << 16U) + 2U) #define sfLimitAmount ((6U << 16U) + 3U) @@ -136,6 +144,9 @@ #define sfNFTokenBrokerFee ((6U << 16U) + 19U) #define sfHookCallbackFee ((6U << 16U) + 20U) #define sfLockedBalance ((6U << 16U) + 21U) +#define sfBaseFeeDrops ((6U << 16U) + 22U) +#define sfReserveBaseDrops ((6U << 16U) + 23U) +#define sfReserveIncrementDrops ((6U << 16U) + 24U) #define sfPublicKey ((7U << 16U) + 1U) #define sfMessageKey ((7U << 16U) + 2U) #define sfSigningPubKey ((7U << 16U) + 3U) @@ -171,11 +182,13 @@ #define sfNFTokenMinter ((8U << 16U) + 9U) #define sfEmitCallback ((8U << 16U) + 10U) #define sfHookAccount ((8U << 16U) + 16U) +#define sfInform ((8U << 16U) + 99U) #define sfIndexes ((19U << 16U) + 1U) #define sfHashes ((19U << 16U) + 2U) #define sfAmendments ((19U << 16U) + 3U) #define sfNFTokenOffers ((19U << 16U) + 4U) #define sfHookNamespaces ((19U << 16U) + 5U) +#define sfURITokenIDs ((19U << 16U) + 99U) #define sfPaths ((18U << 16U) + 1U) #define sfTransactionMetaData ((14U << 16U) + 2U) #define sfCreatedNode ((14U << 16U) + 3U) @@ -198,6 +211,12 @@ #define sfHookDefinition ((14U << 16U) + 22U) #define sfHookParameter ((14U << 16U) + 23U) #define sfHookGrant ((14U << 16U) + 24U) +#define sfGenesisMint ((14U << 16U) + 96U) +#define sfActiveValidator ((14U << 16U) + 95U) +#define sfImportVLKey ((14U << 16U) + 94U) +#define sfHookEmission ((14U << 16U) + 93U) +#define sfMintURIToken ((14U << 16U) + 92U) +#define sfAmountEntry ((14U << 16U) + 91U) #define sfSigners ((15U << 16U) + 3U) #define sfSignerEntries ((15U << 16U) + 4U) #define sfTemplate ((15U << 16U) + 5U) @@ -212,4 +231,8 @@ #define sfHookExecutions ((15U << 16U) + 18U) #define sfHookParameters ((15U << 16U) + 19U) #define sfHookGrants ((15U << 16U) + 20U) +#define sfGenesisMints ((15U << 16U) + 96U) #define sfActiveValidators ((15U << 16U) + 95U) +#define sfImportVLKeys ((15U << 16U) + 94U) +#define sfHookEmissions ((15U << 16U) + 93U) +#define sfAmounts ((15U << 16U) + 92U) \ No newline at end of file From 58b22901cb12e8442fe7023a6a6c7bd866456f34 Mon Sep 17 00:00:00 2001 From: Denis Angell Date: Sat, 9 Nov 2024 06:17:00 +0100 Subject: [PATCH 08/11] Fix: `float_divide` rounding error (#351) Co-authored-by: RichardAH --- src/ripple/app/hook/impl/applyHook.cpp | 21 ++++++++++++++++----- src/ripple/protocol/Feature.h | 3 ++- src/ripple/protocol/impl/Feature.cpp | 1 + 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/ripple/app/hook/impl/applyHook.cpp b/src/ripple/app/hook/impl/applyHook.cpp index 2afd018e8..abd7ef136 100644 --- a/src/ripple/app/hook/impl/applyHook.cpp +++ b/src/ripple/app/hook/impl/applyHook.cpp @@ -5404,7 +5404,7 @@ DEFINE_HOOK_FUNCTION( const int64_t float_one_internal = make_float(1000000000000000ull, -15, false); inline int64_t -float_divide_internal(int64_t float1, int64_t float2) +float_divide_internal(int64_t float1, int64_t float2, bool hasFix) { RETURN_IF_INVALID_FLOAT(float1); RETURN_IF_INVALID_FLOAT(float2); @@ -5457,8 +5457,16 @@ float_divide_internal(int64_t float1, int64_t float2) while (man2 > 0) { int i = 0; - for (; man1 > man2; man1 -= man2, ++i) - ; + if (hasFix) + { + for (; man1 >= man2; man1 -= man2, ++i) + ; + } + else + { + for (; man1 > man2; man1 -= man2, ++i) + ; + } man3 *= 10; man3 += i; @@ -5478,7 +5486,8 @@ DEFINE_HOOK_FUNCTION(int64_t, float_divide, int64_t float1, int64_t float2) HOOK_SETUP(); // populates memory_ctx, memory, memory_length, applyCtx, // hookCtx on current stack - return float_divide_internal(float1, float2); + bool const hasFix = view.rules().enabled(fixFloatDivide); + return float_divide_internal(float1, float2, hasFix); HOOK_TEARDOWN(); } @@ -5497,7 +5506,9 @@ DEFINE_HOOK_FUNCTION(int64_t, float_invert, int64_t float1) return DIVISION_BY_ZERO; if (float1 == float_one_internal) return float_one_internal; - return float_divide_internal(float_one_internal, float1); + + bool const fixV3 = view.rules().enabled(fixFloatDivide); + return float_divide_internal(float_one_internal, float1, fixV3); HOOK_TEARDOWN(); } diff --git a/src/ripple/protocol/Feature.h b/src/ripple/protocol/Feature.h index 54628c9cd..43d510c63 100644 --- a/src/ripple/protocol/Feature.h +++ b/src/ripple/protocol/Feature.h @@ -74,7 +74,7 @@ namespace detail { // Feature.cpp. Because it's only used to reserve storage, and determine how // large to make the FeatureBitset, it MAY be larger. It MUST NOT be less than // the actual number of amendments. A LogicError on startup will verify this. -static constexpr std::size_t numFeatures = 73; +static constexpr std::size_t numFeatures = 74; /** Amendments that this server supports and the default voting behavior. Whether they are enabled depends on the Rules defined in the validated @@ -361,6 +361,7 @@ extern uint256 const fixNSDelete; extern uint256 const fix240819; extern uint256 const fixPageCap; extern uint256 const fix240911; +extern uint256 const fixFloatDivide; } // namespace ripple diff --git a/src/ripple/protocol/impl/Feature.cpp b/src/ripple/protocol/impl/Feature.cpp index 27ec16cb7..23cbe236d 100644 --- a/src/ripple/protocol/impl/Feature.cpp +++ b/src/ripple/protocol/impl/Feature.cpp @@ -467,6 +467,7 @@ REGISTER_FIX (fixNSDelete, Supported::yes, VoteBehavior::De REGISTER_FIX (fix240819, Supported::yes, VoteBehavior::DefaultYes); REGISTER_FIX (fixPageCap, Supported::yes, VoteBehavior::DefaultYes); REGISTER_FIX (fix240911, Supported::yes, VoteBehavior::DefaultYes); +REGISTER_FIX (fixFloatDivide, Supported::yes, VoteBehavior::DefaultYes); // The following amendments are obsolete, but must remain supported // because they could potentially get enabled. From 2b225977e230fb2cbf945fe176efd90212caa738 Mon Sep 17 00:00:00 2001 From: RichardAH Date: Tue, 12 Nov 2024 08:55:56 +1000 Subject: [PATCH 09/11] Feature: RWDB (#378) Co-authored-by: Denis Angell --- Builds/CMake/RippledCore.cmake | 2 + cfg/rippled-example.cfg | 16 +- src/ripple/app/ledger/impl/LedgerCleaner.cpp | 13 +- src/ripple/app/main/Application.h | 1 - src/ripple/app/misc/SHAMapStoreImp.cpp | 17 +- src/ripple/app/rdb/backend/FlatmapDatabase.h | 853 ++++++++++++++++ src/ripple/app/rdb/backend/RWDBDatabase.h | 964 ++++++++++++++++++ .../app/rdb/impl/RelationalDatabase.cpp | 20 + src/ripple/core/Config.h | 16 + .../nodestore/backend/FlatmapFactory.cpp | 235 +++++ .../nodestore/backend/MemoryFactory.cpp | 2 +- src/ripple/nodestore/backend/RWDBFactory.cpp | 242 +++++ src/ripple/overlay/impl/OverlayImpl.cpp | 7 +- src/ripple/peerfinder/impl/InMemoryStore.h | 54 + .../peerfinder/impl/PeerfinderManager.cpp | 21 +- src/ripple/peerfinder/make_Manager.h | 3 +- src/ripple/shamap/impl/SHAMap.cpp | 44 + src/test/nodestore/Backend_test.cpp | 6 +- src/test/nodestore/Database_test.cpp | 2 + 19 files changed, 2495 insertions(+), 23 deletions(-) create mode 100644 src/ripple/app/rdb/backend/FlatmapDatabase.h create mode 100644 src/ripple/app/rdb/backend/RWDBDatabase.h create mode 100644 src/ripple/nodestore/backend/FlatmapFactory.cpp create mode 100644 src/ripple/nodestore/backend/RWDBFactory.cpp create mode 100644 src/ripple/peerfinder/impl/InMemoryStore.h diff --git a/Builds/CMake/RippledCore.cmake b/Builds/CMake/RippledCore.cmake index 9a211ef9b..bf24760b0 100644 --- a/Builds/CMake/RippledCore.cmake +++ b/Builds/CMake/RippledCore.cmake @@ -538,7 +538,9 @@ target_sources (rippled PRIVATE subdir: nodestore #]===============================] src/ripple/nodestore/backend/CassandraFactory.cpp + src/ripple/nodestore/backend/RWDBFactory.cpp src/ripple/nodestore/backend/MemoryFactory.cpp + src/ripple/nodestore/backend/FlatmapFactory.cpp src/ripple/nodestore/backend/NuDBFactory.cpp src/ripple/nodestore/backend/NullFactory.cpp src/ripple/nodestore/backend/RocksDBFactory.cpp diff --git a/cfg/rippled-example.cfg b/cfg/rippled-example.cfg index 0a6693130..41fe12158 100644 --- a/cfg/rippled-example.cfg +++ b/cfg/rippled-example.cfg @@ -1056,7 +1056,18 @@ # Cassandra is an alternative backend to be used only with Reporting Mode. # See the Reporting Mode section for more details about Reporting Mode. # -# Required keys for NuDB and RocksDB: +# type = RWDB +# +# RWDB is a high-performance memory store written by XRPL-Labs and optimized +# for xahaud. RWDB is NOT persistent and the data will be lost on restart. +# RWDB is recommended for Validator and Peer nodes that are not required to +# store history. +# +# RWDB maintains its high speed regardless of the amount of history +# stored. Online delete should NOT be used instead RWDB will use the +# ledger_history config value to determine how many ledgers to keep in memory. +# +# Required keys for NuDB, RWDB and RocksDB: # # path Location to store the database # @@ -1112,7 +1123,8 @@ # online_delete Minimum value of 256. Enable automatic purging # of older ledger information. Maintain at least this # number of ledger records online. Must be greater -# than or equal to ledger_history. +# than or equal to ledger_history. If using RWDB +# this value is ignored. # # These keys modify the behavior of online_delete, and thus are only # relevant if online_delete is defined and non-zero: diff --git a/src/ripple/app/ledger/impl/LedgerCleaner.cpp b/src/ripple/app/ledger/impl/LedgerCleaner.cpp index e5ee6409d..bbb3c709f 100644 --- a/src/ripple/app/ledger/impl/LedgerCleaner.cpp +++ b/src/ripple/app/ledger/impl/LedgerCleaner.cpp @@ -219,7 +219,7 @@ private: run() { beast::setCurrentThreadName("LedgerCleaner"); - JLOG(j_.debug()) << "Started"; + JLOG(j_.debug()) << "Started ledger cleaner"; while (true) { @@ -392,7 +392,8 @@ private: if (app_.getFeeTrack().isLoadedLocal()) { - JLOG(j_.debug()) << "Waiting for load to subside"; + JLOG(j_.debug()) + << "Ledger Cleaner: Waiting for load to subside"; std::this_thread::sleep_for(std::chrono::seconds(5)); continue; } @@ -415,13 +416,15 @@ private: bool fail = false; if (ledgerHash.isZero()) { - JLOG(j_.info()) - << "Unable to get hash for ledger " << ledgerIndex; + JLOG(j_.warn()) + << "Ledger Cleaner: Unable to get hash for ledger " + << ledgerIndex; fail = true; } else if (!doLedger(ledgerIndex, ledgerHash, doNodes, doTxns)) { - JLOG(j_.info()) << "Failed to process ledger " << ledgerIndex; + JLOG(j_.warn()) << "Ledger Cleaner: Failed to process ledger " + << ledgerIndex; fail = true; } diff --git a/src/ripple/app/main/Application.h b/src/ripple/app/main/Application.h index d8cb7d318..a6e58b19e 100644 --- a/src/ripple/app/main/Application.h +++ b/src/ripple/app/main/Application.h @@ -19,7 +19,6 @@ #ifndef RIPPLE_APP_MAIN_APPLICATION_H_INCLUDED #define RIPPLE_APP_MAIN_APPLICATION_H_INCLUDED - #include #include #include diff --git a/src/ripple/app/misc/SHAMapStoreImp.cpp b/src/ripple/app/misc/SHAMapStoreImp.cpp index d5cb07792..34cb86ed8 100644 --- a/src/ripple/app/misc/SHAMapStoreImp.cpp +++ b/src/ripple/app/misc/SHAMapStoreImp.cpp @@ -118,7 +118,9 @@ SHAMapStoreImp::SHAMapStoreImp( get_if_exists(section, "online_delete", deleteInterval_); - if (deleteInterval_) + bool const isMem = config.mem_backend(); + + if (deleteInterval_ || isMem) { if (app_.config().reporting()) { @@ -127,6 +129,9 @@ SHAMapStoreImp::SHAMapStoreImp( "online_delete info from config"); } + if (isMem) + deleteInterval_ = config.LEDGER_HISTORY; + // Configuration that affects the behavior of online delete get_if_exists(section, "delete_batch", deleteBatch_); std::uint32_t temp; @@ -162,7 +167,8 @@ SHAMapStoreImp::SHAMapStoreImp( } state_db_.init(config, dbName_); - dbPaths(); + if (!isMem) + dbPaths(); } } @@ -195,6 +201,7 @@ SHAMapStoreImp::makeNodeStore(int readThreads) "online_delete info from config"); } SavedState state = state_db_.getState(); + auto writableBackend = makeBackendRotating(state.writableDb); auto archiveBackend = makeBackendRotating(state.archiveDb); if (!state.writableDb.size()) @@ -293,6 +300,8 @@ SHAMapStoreImp::run() fullBelowCache_ = &(*app_.getNodeFamily().getFullBelowCache(0)); treeNodeCache_ = &(*app_.getNodeFamily().getTreeNodeCache(0)); + bool const isMem = app_.config().mem_backend(); + if (advisoryDelete_) canDelete_ = state_db_.getCanDelete(); @@ -351,7 +360,7 @@ SHAMapStoreImp::run() // will delete up to (not including) lastRotated if (readyToRotate && !waitForImport) { - JLOG(journal_.warn()) + JLOG(journal_.debug()) << "rotating validatedSeq " << validatedSeq << " lastRotated " << lastRotated << " deleteInterval " << deleteInterval_ << " canDelete_ " << canDelete_ << " state " @@ -395,7 +404,7 @@ SHAMapStoreImp::run() // Only log if we completed without a "health" abort JLOG(journal_.debug()) << validatedSeq << " freshened caches"; - JLOG(journal_.trace()) << "Making a new backend"; + JLOG(journal_.debug()) << "Making a new backend"; auto newBackend = makeBackendRotating(); JLOG(journal_.debug()) << validatedSeq << " new backend " << newBackend->getName(); diff --git a/src/ripple/app/rdb/backend/FlatmapDatabase.h b/src/ripple/app/rdb/backend/FlatmapDatabase.h new file mode 100644 index 000000000..34814655f --- /dev/null +++ b/src/ripple/app/rdb/backend/FlatmapDatabase.h @@ -0,0 +1,853 @@ +#ifndef RIPPLE_APP_RDB_BACKEND_FLATMAPDATABASE_H_INCLUDED +#define RIPPLE_APP_RDB_BACKEND_FLATMAPDATABASE_H_INCLUDED + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace ripple { + +struct base_uint_hasher +{ + using result_type = std::size_t; + + result_type + operator()(base_uint<256> const& value) const + { + return hardened_hash<>{}(value); + } + + result_type + operator()(AccountID const& value) const + { + return hardened_hash<>{}(value); + } +}; + +class FlatmapDatabase : public SQLiteDatabase +{ +private: + struct LedgerData + { + LedgerInfo info; + boost::unordered:: + concurrent_flat_map + transactions; + }; + + struct AccountTxData + { + boost::unordered:: + concurrent_flat_map, AccountTx> + transactions; + }; + + Application& app_; + Config const& config_; + JobQueue& jobQueue_; + + boost::unordered::concurrent_flat_map ledgers_; + boost::unordered:: + concurrent_flat_map + ledgerHashToSeq_; + boost::unordered::concurrent_flat_map + transactionMap_; + boost::unordered:: + concurrent_flat_map + accountTxMap_; + +public: + FlatmapDatabase(Application& app, Config const& config, JobQueue& jobQueue) + : app_(app), config_(config), jobQueue_(jobQueue) + { + } + + std::optional + getMinLedgerSeq() override + { + std::optional minSeq; + ledgers_.visit_all([&minSeq](auto const& pair) { + if (!minSeq || pair.first < *minSeq) + { + minSeq = pair.first; + } + }); + return minSeq; + } + + std::optional + getTransactionsMinLedgerSeq() override + { + std::optional minSeq; + transactionMap_.visit_all([&minSeq](auto const& pair) { + LedgerIndex seq = pair.second.second->getLgrSeq(); + if (!minSeq || seq < *minSeq) + { + minSeq = seq; + } + }); + return minSeq; + } + + std::optional + getAccountTransactionsMinLedgerSeq() override + { + std::optional minSeq; + accountTxMap_.visit_all([&minSeq](auto const& pair) { + pair.second.transactions.visit_all([&minSeq](auto const& tx) { + if (!minSeq || tx.first.first < *minSeq) + { + minSeq = tx.first.first; + } + }); + }); + return minSeq; + } + + std::optional + getMaxLedgerSeq() override + { + std::optional maxSeq; + ledgers_.visit_all([&maxSeq](auto const& pair) { + if (!maxSeq || pair.first > *maxSeq) + { + maxSeq = pair.first; + } + }); + return maxSeq; + } + void + deleteTransactionByLedgerSeq(LedgerIndex ledgerSeq) override + { + ledgers_.visit(ledgerSeq, [this](auto& item) { + item.second.transactions.visit_all([this](auto const& txPair) { + transactionMap_.erase(txPair.first); + }); + item.second.transactions.clear(); + }); + + accountTxMap_.visit_all([ledgerSeq](auto& item) { + item.second.transactions.erase_if([ledgerSeq](auto const& tx) { + return tx.first.first == ledgerSeq; + }); + }); + } + + void + deleteBeforeLedgerSeq(LedgerIndex ledgerSeq) override + { + ledgers_.erase_if([this, ledgerSeq](auto const& item) { + if (item.first < ledgerSeq) + { + item.second.transactions.visit_all([this](auto const& txPair) { + transactionMap_.erase(txPair.first); + }); + ledgerHashToSeq_.erase(item.second.info.hash); + return true; + } + return false; + }); + + accountTxMap_.visit_all([ledgerSeq](auto& item) { + item.second.transactions.erase_if([ledgerSeq](auto const& tx) { + return tx.first.first < ledgerSeq; + }); + }); + } + + void + deleteTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) override + { + ledgers_.visit_all([this, ledgerSeq](auto& item) { + if (item.first < ledgerSeq) + { + item.second.transactions.visit_all([this](auto const& txPair) { + transactionMap_.erase(txPair.first); + }); + item.second.transactions.clear(); + } + }); + + accountTxMap_.visit_all([ledgerSeq](auto& item) { + item.second.transactions.erase_if([ledgerSeq](auto const& tx) { + return tx.first.first < ledgerSeq; + }); + }); + } + + void + deleteAccountTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) override + { + accountTxMap_.visit_all([ledgerSeq](auto& item) { + item.second.transactions.erase_if([ledgerSeq](auto const& tx) { + return tx.first.first < ledgerSeq; + }); + }); + } + std::size_t + getTransactionCount() override + { + return transactionMap_.size(); + } + + std::size_t + getAccountTransactionCount() override + { + std::size_t count = 0; + accountTxMap_.visit_all([&count](auto const& item) { + count += item.second.transactions.size(); + }); + return count; + } + + CountMinMax + getLedgerCountMinMax() override + { + CountMinMax result{0, 0, 0}; + ledgers_.visit_all([&result](auto const& item) { + result.numberOfRows++; + if (result.minLedgerSequence == 0 || + item.first < result.minLedgerSequence) + { + result.minLedgerSequence = item.first; + } + if (item.first > result.maxLedgerSequence) + { + result.maxLedgerSequence = item.first; + } + }); + return result; + } + + bool + saveValidatedLedger( + std::shared_ptr const& ledger, + bool current) override + { + try + { + LedgerData ledgerData; + ledgerData.info = ledger->info(); + + auto aLedger = std::make_shared(ledger, app_); + for (auto const& acceptedLedgerTx : *aLedger) + { + auto const& txn = acceptedLedgerTx->getTxn(); + auto const& meta = acceptedLedgerTx->getMeta(); + auto const& id = txn->getTransactionID(); + + std::string reason; + auto accTx = std::make_pair( + std::make_shared(txn, reason, app_), + std::make_shared(meta)); + + ledgerData.transactions.emplace(id, accTx); + transactionMap_.emplace(id, accTx); + + for (auto const& account : meta.getAffectedAccounts()) + { + accountTxMap_.visit(account, [&](auto& data) { + data.second.transactions.emplace( + std::make_pair( + ledger->info().seq, + acceptedLedgerTx->getTxnSeq()), + accTx); + }); + } + } + + ledgers_.emplace(ledger->info().seq, std::move(ledgerData)); + ledgerHashToSeq_.emplace(ledger->info().hash, ledger->info().seq); + + if (current) + { + auto const cutoffSeq = + ledger->info().seq > app_.config().LEDGER_HISTORY + ? ledger->info().seq - app_.config().LEDGER_HISTORY + : 0; + + if (cutoffSeq > 0) + { + const std::size_t BATCH_SIZE = 128; + std::size_t deleted = 0; + + ledgers_.erase_if([&](auto const& item) { + if (deleted >= BATCH_SIZE) + return false; + + if (item.first < cutoffSeq) + { + item.second.transactions.visit_all( + [this](auto const& txPair) { + transactionMap_.erase(txPair.first); + }); + ledgerHashToSeq_.erase(item.second.info.hash); + deleted++; + return true; + } + return false; + }); + + if (deleted > 0) + { + accountTxMap_.visit_all([cutoffSeq](auto& item) { + item.second.transactions.erase_if( + [cutoffSeq](auto const& tx) { + return tx.first.first < cutoffSeq; + }); + }); + } + + app_.getLedgerMaster().clearPriorLedgers(cutoffSeq); + } + } + + return true; + } + catch (std::exception const&) + { + deleteTransactionByLedgerSeq(ledger->info().seq); + return false; + } + } + + std::optional + getLedgerInfoByIndex(LedgerIndex ledgerSeq) override + { + std::optional result; + ledgers_.visit(ledgerSeq, [&result](auto const& item) { + result = item.second.info; + }); + return result; + } + + std::optional + getNewestLedgerInfo() override + { + std::optional result; + ledgers_.visit_all([&result](auto const& item) { + if (!result || item.second.info.seq > result->seq) + { + result = item.second.info; + } + }); + return result; + } + + std::optional + getLimitedOldestLedgerInfo(LedgerIndex ledgerFirstIndex) override + { + std::optional result; + ledgers_.visit_all([&](auto const& item) { + if (item.first >= ledgerFirstIndex && + (!result || item.first < result->seq)) + { + result = item.second.info; + } + }); + return result; + } + + std::optional + getLimitedNewestLedgerInfo(LedgerIndex ledgerFirstIndex) override + { + std::optional result; + ledgers_.visit_all([&](auto const& item) { + if (item.first >= ledgerFirstIndex && + (!result || item.first > result->seq)) + { + result = item.second.info; + } + }); + return result; + } + + std::optional + getLedgerInfoByHash(uint256 const& ledgerHash) override + { + std::optional result; + ledgerHashToSeq_.visit(ledgerHash, [this, &result](auto const& item) { + ledgers_.visit(item.second, [&result](auto const& item) { + result = item.second.info; + }); + }); + return result; + } + uint256 + getHashByIndex(LedgerIndex ledgerIndex) override + { + uint256 result; + ledgers_.visit(ledgerIndex, [&result](auto const& item) { + result = item.second.info.hash; + }); + return result; + } + + std::optional + getHashesByIndex(LedgerIndex ledgerIndex) override + { + std::optional result; + ledgers_.visit(ledgerIndex, [&result](auto const& item) { + result = LedgerHashPair{ + item.second.info.hash, item.second.info.parentHash}; + }); + return result; + } + + std::map + getHashesByIndex(LedgerIndex minSeq, LedgerIndex maxSeq) override + { + std::map result; + ledgers_.visit_all([&](auto const& item) { + if (item.first >= minSeq && item.first <= maxSeq) + { + result[item.first] = LedgerHashPair{ + item.second.info.hash, item.second.info.parentHash}; + } + }); + return result; + } + + std::variant + getTransaction( + uint256 const& id, + std::optional> const& range, + error_code_i& ec) override + { + std::variant result = TxSearched::unknown; + transactionMap_.visit(id, [&](auto const& item) { + auto const& tx = item.second; + if (!range || + (range->lower() <= tx.second->getLgrSeq() && + tx.second->getLgrSeq() <= range->upper())) + { + result = tx; + } + else + { + result = TxSearched::all; + } + }); + return result; + } + + bool + ledgerDbHasSpace(Config const& config) override + { + return true; // In-memory database always has space + } + + bool + transactionDbHasSpace(Config const& config) override + { + return true; // In-memory database always has space + } + + std::uint32_t + getKBUsedAll() override + { + std::uint32_t size = sizeof(*this); + size += ledgers_.size() * (sizeof(LedgerIndex) + sizeof(LedgerData)); + size += + ledgerHashToSeq_.size() * (sizeof(uint256) + sizeof(LedgerIndex)); + size += transactionMap_.size() * (sizeof(uint256) + sizeof(AccountTx)); + accountTxMap_.visit_all([&size](auto const& item) { + size += sizeof(AccountID) + sizeof(AccountTxData); + size += item.second.transactions.size() * sizeof(AccountTx); + }); + return size / 1024; // Convert to KB + } + + std::uint32_t + getKBUsedLedger() override + { + std::uint32_t size = + ledgers_.size() * (sizeof(LedgerIndex) + sizeof(LedgerData)); + size += + ledgerHashToSeq_.size() * (sizeof(uint256) + sizeof(LedgerIndex)); + return size / 1024; + } + + std::uint32_t + getKBUsedTransaction() override + { + std::uint32_t size = + transactionMap_.size() * (sizeof(uint256) + sizeof(AccountTx)); + accountTxMap_.visit_all([&size](auto const& item) { + size += sizeof(AccountID) + sizeof(AccountTxData); + size += item.second.transactions.size() * sizeof(AccountTx); + }); + return size / 1024; + } + + void + closeLedgerDB() override + { + // No-op for in-memory database + } + + void + closeTransactionDB() override + { + // No-op for in-memory database + } + + ~FlatmapDatabase() + { + // Concurrent maps need visit_all + accountTxMap_.visit_all( + [](auto& pair) { pair.second.transactions.clear(); }); + accountTxMap_.clear(); + + transactionMap_.clear(); + + ledgers_.visit_all( + [](auto& pair) { pair.second.transactions.clear(); }); + ledgers_.clear(); + + ledgerHashToSeq_.clear(); + } + + std::vector> + getTxHistory(LedgerIndex startIndex) override + { + std::vector> result; + transactionMap_.visit_all([&](auto const& item) { + if (item.second.second->getLgrSeq() >= startIndex) + { + result.push_back(item.second.first); + } + }); + std::sort( + result.begin(), result.end(), [](auto const& a, auto const& b) { + return a->getLedger() > b->getLedger(); + }); + if (result.size() > 20) + { + result.resize(20); + } + return result; + } + // Helper function to handle limits + template + void + applyLimit(Container& container, std::size_t limit, bool bUnlimited) + { + if (!bUnlimited && limit > 0 && container.size() > limit) + { + container.resize(limit); + } + } + + AccountTxs + getOldestAccountTxs(AccountTxOptions const& options) override + { + AccountTxs result; + accountTxMap_.visit(options.account, [&](auto const& item) { + item.second.transactions.visit_all([&](auto const& tx) { + if (tx.first.first >= options.minLedger && + tx.first.first <= options.maxLedger) + { + result.push_back(tx.second); + } + }); + }); + std::sort( + result.begin(), result.end(), [](auto const& a, auto const& b) { + return a.second->getLgrSeq() < b.second->getLgrSeq(); + }); + applyLimit(result, options.limit, options.bUnlimited); + return result; + } + + AccountTxs + getNewestAccountTxs(AccountTxOptions const& options) override + { + AccountTxs result; + accountTxMap_.visit(options.account, [&](auto const& item) { + item.second.transactions.visit_all([&](auto const& tx) { + if (tx.first.first >= options.minLedger && + tx.first.first <= options.maxLedger) + { + result.push_back(tx.second); + } + }); + }); + std::sort( + result.begin(), result.end(), [](auto const& a, auto const& b) { + return a.second->getLgrSeq() > b.second->getLgrSeq(); + }); + applyLimit(result, options.limit, options.bUnlimited); + return result; + } + + MetaTxsList + getOldestAccountTxsB(AccountTxOptions const& options) override + { + MetaTxsList result; + accountTxMap_.visit(options.account, [&](auto const& item) { + item.second.transactions.visit_all([&](auto const& tx) { + if (tx.first.first >= options.minLedger && + tx.first.first <= options.maxLedger) + { + result.emplace_back( + tx.second.first->getSTransaction() + ->getSerializer() + .peekData(), + tx.second.second->getAsObject() + .getSerializer() + .peekData(), + tx.first.first); + } + }); + }); + std::sort( + result.begin(), result.end(), [](auto const& a, auto const& b) { + return std::get<2>(a) < std::get<2>(b); + }); + applyLimit(result, options.limit, options.bUnlimited); + return result; + } + + MetaTxsList + getNewestAccountTxsB(AccountTxOptions const& options) override + { + MetaTxsList result; + accountTxMap_.visit(options.account, [&](auto const& item) { + item.second.transactions.visit_all([&](auto const& tx) { + if (tx.first.first >= options.minLedger && + tx.first.first <= options.maxLedger) + { + result.emplace_back( + tx.second.first->getSTransaction() + ->getSerializer() + .peekData(), + tx.second.second->getAsObject() + .getSerializer() + .peekData(), + tx.first.first); + } + }); + }); + std::sort( + result.begin(), result.end(), [](auto const& a, auto const& b) { + return std::get<2>(a) > std::get<2>(b); + }); + applyLimit(result, options.limit, options.bUnlimited); + return result; + } + std::pair> + oldestAccountTxPage(AccountTxPageOptions const& options) override + { + AccountTxs result; + std::optional marker; + + accountTxMap_.visit(options.account, [&](auto const& item) { + std::vector, AccountTx>> + txs; + item.second.transactions.visit_all([&](auto const& tx) { + if (tx.first.first >= options.minLedger && + tx.first.first <= options.maxLedger) + { + txs.emplace_back(tx); + } + }); + + std::sort(txs.begin(), txs.end(), [](auto const& a, auto const& b) { + return a.first < b.first; + }); + + auto it = txs.begin(); + if (options.marker) + { + it = std::find_if(txs.begin(), txs.end(), [&](auto const& tx) { + return tx.first.first == options.marker->ledgerSeq && + tx.first.second == options.marker->txnSeq; + }); + if (it != txs.end()) + ++it; + } + + for (; it != txs.end() && + (options.limit == 0 || result.size() < options.limit); + ++it) + { + result.push_back(it->second); + } + + if (it != txs.end()) + { + marker = AccountTxMarker{it->first.first, it->first.second}; + } + }); + + return {result, marker}; + } + + std::pair> + newestAccountTxPage(AccountTxPageOptions const& options) override + { + AccountTxs result; + std::optional marker; + + accountTxMap_.visit(options.account, [&](auto const& item) { + std::vector, AccountTx>> + txs; + item.second.transactions.visit_all([&](auto const& tx) { + if (tx.first.first >= options.minLedger && + tx.first.first <= options.maxLedger) + { + txs.emplace_back(tx); + } + }); + + std::sort(txs.begin(), txs.end(), [](auto const& a, auto const& b) { + return a.first > b.first; + }); + + auto it = txs.begin(); + if (options.marker) + { + it = std::find_if(txs.begin(), txs.end(), [&](auto const& tx) { + return tx.first.first == options.marker->ledgerSeq && + tx.first.second == options.marker->txnSeq; + }); + if (it != txs.end()) + ++it; + } + + for (; it != txs.end() && + (options.limit == 0 || result.size() < options.limit); + ++it) + { + result.push_back(it->second); + } + + if (it != txs.end()) + { + marker = AccountTxMarker{it->first.first, it->first.second}; + } + }); + + return {result, marker}; + } + + std::pair> + oldestAccountTxPageB(AccountTxPageOptions const& options) override + { + MetaTxsList result; + std::optional marker; + + accountTxMap_.visit(options.account, [&](auto const& item) { + std::vector> txs; + item.second.transactions.visit_all([&](auto const& tx) { + if (tx.first.first >= options.minLedger && + tx.first.first <= options.maxLedger) + { + txs.emplace_back( + tx.first.first, tx.first.second, tx.second); + } + }); + + std::sort(txs.begin(), txs.end()); + + auto it = txs.begin(); + if (options.marker) + { + it = std::find_if(txs.begin(), txs.end(), [&](auto const& tx) { + return std::get<0>(tx) == options.marker->ledgerSeq && + std::get<1>(tx) == options.marker->txnSeq; + }); + if (it != txs.end()) + ++it; + } + + for (; it != txs.end() && + (options.limit == 0 || result.size() < options.limit); + ++it) + { + const auto& [_, __, tx] = *it; + result.emplace_back( + tx.first->getSTransaction()->getSerializer().peekData(), + tx.second->getAsObject().getSerializer().peekData(), + std::get<0>(*it)); + } + + if (it != txs.end()) + { + marker = AccountTxMarker{std::get<0>(*it), std::get<1>(*it)}; + } + }); + + return {result, marker}; + } + + std::pair> + newestAccountTxPageB(AccountTxPageOptions const& options) override + { + MetaTxsList result; + std::optional marker; + + accountTxMap_.visit(options.account, [&](auto const& item) { + std::vector> txs; + item.second.transactions.visit_all([&](auto const& tx) { + if (tx.first.first >= options.minLedger && + tx.first.first <= options.maxLedger) + { + txs.emplace_back( + tx.first.first, tx.first.second, tx.second); + } + }); + + std::sort(txs.begin(), txs.end(), std::greater<>()); + + auto it = txs.begin(); + if (options.marker) + { + it = std::find_if(txs.begin(), txs.end(), [&](auto const& tx) { + return std::get<0>(tx) == options.marker->ledgerSeq && + std::get<1>(tx) == options.marker->txnSeq; + }); + if (it != txs.end()) + ++it; + } + + for (; it != txs.end() && + (options.limit == 0 || result.size() < options.limit); + ++it) + { + const auto& [_, __, tx] = *it; + result.emplace_back( + tx.first->getSTransaction()->getSerializer().peekData(), + tx.second->getAsObject().getSerializer().peekData(), + std::get<0>(*it)); + } + + if (it != txs.end()) + { + marker = AccountTxMarker{std::get<0>(*it), std::get<1>(*it)}; + } + }); + + return {result, marker}; + } +}; + +// Factory function +std::unique_ptr +getFlatmapDatabase(Application& app, Config const& config, JobQueue& jobQueue) +{ + return std::make_unique(app, config, jobQueue); +} + +} // namespace ripple +#endif // RIPPLE_APP_RDB_BACKEND_FLATMAPDATABASE_H_INCLUDED diff --git a/src/ripple/app/rdb/backend/RWDBDatabase.h b/src/ripple/app/rdb/backend/RWDBDatabase.h new file mode 100644 index 000000000..0a84f48ce --- /dev/null +++ b/src/ripple/app/rdb/backend/RWDBDatabase.h @@ -0,0 +1,964 @@ +#ifndef RIPPLE_APP_RDB_BACKEND_MEMORYDATABASE_H_INCLUDED +#define RIPPLE_APP_RDB_BACKEND_MEMORYDATABASE_H_INCLUDED + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ripple { + +class RWDBDatabase : public SQLiteDatabase +{ +private: + struct LedgerData + { + LedgerInfo info; + std::map transactions; + }; + + struct AccountTxData + { + AccountTxs transactions; + std::map> + ledgerTxMap; // ledgerSeq -> txSeq -> index in transactions + }; + + Application& app_; + + mutable std::shared_mutex mutex_; + + std::map ledgers_; + std::map ledgerHashToSeq_; + std::map transactionMap_; + std::map accountTxMap_; + +public: + RWDBDatabase(Application& app, Config const& config, JobQueue& jobQueue) + : app_(app) + { + } + + std::optional + getMinLedgerSeq() override + { + std::shared_lock lock(mutex_); + if (ledgers_.empty()) + return std::nullopt; + return ledgers_.begin()->first; + } + + std::optional + getTransactionsMinLedgerSeq() override + { + std::shared_lock lock(mutex_); + if (transactionMap_.empty()) + return std::nullopt; + return transactionMap_.begin()->second.second->getLgrSeq(); + } + + std::optional + getAccountTransactionsMinLedgerSeq() override + { + std::shared_lock lock(mutex_); + if (accountTxMap_.empty()) + return std::nullopt; + LedgerIndex minSeq = std::numeric_limits::max(); + for (const auto& [_, accountData] : accountTxMap_) + { + if (!accountData.ledgerTxMap.empty()) + minSeq = + std::min(minSeq, accountData.ledgerTxMap.begin()->first); + } + return minSeq == std::numeric_limits::max() + ? std::nullopt + : std::optional(minSeq); + } + + std::optional + getMaxLedgerSeq() override + { + std::shared_lock lock(mutex_); + if (ledgers_.empty()) + return std::nullopt; + return ledgers_.rbegin()->first; + } + void + deleteTransactionByLedgerSeq(LedgerIndex ledgerSeq) override + { + std::unique_lock lock(mutex_); + auto it = ledgers_.find(ledgerSeq); + if (it != ledgers_.end()) + { + for (const auto& [txHash, _] : it->second.transactions) + { + transactionMap_.erase(txHash); + } + it->second.transactions.clear(); + } + for (auto& [_, accountData] : accountTxMap_) + { + accountData.ledgerTxMap.erase(ledgerSeq); + accountData.transactions.erase( + std::remove_if( + accountData.transactions.begin(), + accountData.transactions.end(), + [ledgerSeq](const AccountTx& tx) { + return tx.second->getLgrSeq() == ledgerSeq; + }), + accountData.transactions.end()); + } + } + + void + deleteBeforeLedgerSeq(LedgerIndex ledgerSeq) override + { + std::unique_lock lock(mutex_); + auto it = ledgers_.begin(); + while (it != ledgers_.end() && it->first < ledgerSeq) + { + for (const auto& [txHash, _] : it->second.transactions) + { + transactionMap_.erase(txHash); + } + ledgerHashToSeq_.erase(it->second.info.hash); + it = ledgers_.erase(it); + } + for (auto& [_, accountData] : accountTxMap_) + { + auto txIt = accountData.ledgerTxMap.begin(); + while (txIt != accountData.ledgerTxMap.end() && + txIt->first < ledgerSeq) + { + txIt = accountData.ledgerTxMap.erase(txIt); + } + accountData.transactions.erase( + std::remove_if( + accountData.transactions.begin(), + accountData.transactions.end(), + [ledgerSeq](const AccountTx& tx) { + return tx.second->getLgrSeq() < ledgerSeq; + }), + accountData.transactions.end()); + } + } + + void + deleteTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) override + { + std::unique_lock lock(mutex_); + for (auto& [seq, ledgerData] : ledgers_) + { + if (seq < ledgerSeq) + { + for (const auto& [txHash, _] : ledgerData.transactions) + { + transactionMap_.erase(txHash); + } + ledgerData.transactions.clear(); + } + } + for (auto& [_, accountData] : accountTxMap_) + { + auto txIt = accountData.ledgerTxMap.begin(); + while (txIt != accountData.ledgerTxMap.end() && + txIt->first < ledgerSeq) + { + txIt = accountData.ledgerTxMap.erase(txIt); + } + accountData.transactions.erase( + std::remove_if( + accountData.transactions.begin(), + accountData.transactions.end(), + [ledgerSeq](const AccountTx& tx) { + return tx.second->getLgrSeq() < ledgerSeq; + }), + accountData.transactions.end()); + } + } + + void + deleteAccountTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) override + { + std::unique_lock lock(mutex_); + for (auto& [_, accountData] : accountTxMap_) + { + auto txIt = accountData.ledgerTxMap.begin(); + while (txIt != accountData.ledgerTxMap.end() && + txIt->first < ledgerSeq) + { + txIt = accountData.ledgerTxMap.erase(txIt); + } + accountData.transactions.erase( + std::remove_if( + accountData.transactions.begin(), + accountData.transactions.end(), + [ledgerSeq](const AccountTx& tx) { + return tx.second->getLgrSeq() < ledgerSeq; + }), + accountData.transactions.end()); + } + } + std::size_t + getTransactionCount() override + { + std::shared_lock lock(mutex_); + return transactionMap_.size(); + } + + std::size_t + getAccountTransactionCount() override + { + std::shared_lock lock(mutex_); + std::size_t count = 0; + for (const auto& [_, accountData] : accountTxMap_) + { + count += accountData.transactions.size(); + } + return count; + } + + CountMinMax + getLedgerCountMinMax() override + { + std::shared_lock lock(mutex_); + if (ledgers_.empty()) + return {0, 0, 0}; + return { + ledgers_.size(), ledgers_.begin()->first, ledgers_.rbegin()->first}; + } + + bool + saveValidatedLedger( + std::shared_ptr const& ledger, + bool current) override + { + std::unique_lock lock(mutex_); + LedgerData ledgerData; + ledgerData.info = ledger->info(); + auto aLedger = std::make_shared(ledger, app_); + + for (auto const& acceptedLedgerTx : *aLedger) + { + auto const& txn = acceptedLedgerTx->getTxn(); + auto const& meta = acceptedLedgerTx->getMeta(); + auto const& id = txn->getTransactionID(); + std::string reason; + + auto accTx = std::make_pair( + std::make_shared(txn, reason, app_), + std::make_shared(meta)); + + ledgerData.transactions.emplace(id, accTx); + transactionMap_.emplace(id, accTx); + + for (auto const& account : meta.getAffectedAccounts()) + { + if (accountTxMap_.find(account) == accountTxMap_.end()) + accountTxMap_[account] = AccountTxData(); + auto& accountData = accountTxMap_[account]; + accountData.transactions.push_back(accTx); + accountData.ledgerTxMap[ledger->info().seq] + [acceptedLedgerTx->getTxnSeq()] = + accountData.transactions.size() - 1; + } + } + + ledgers_[ledger->info().seq] = std::move(ledgerData); + ledgerHashToSeq_[ledger->info().hash] = ledger->info().seq; + + if (current) + { + auto const cutoffSeq = + ledger->info().seq > app_.config().LEDGER_HISTORY + ? ledger->info().seq - app_.config().LEDGER_HISTORY + : 0; + + if (cutoffSeq > 0) + { + const std::size_t BATCH_SIZE = 128; + std::size_t deleted = 0; + + std::vector ledgersToDelete; + for (const auto& item : ledgers_) + { + if (deleted >= BATCH_SIZE) + break; + if (item.first < cutoffSeq) + { + ledgersToDelete.push_back(item.first); + deleted++; + } + } + + for (auto seq : ledgersToDelete) + { + auto& ledgerToDelete = ledgers_[seq]; + + for (const auto& txPair : ledgerToDelete.transactions) + { + transactionMap_.erase(txPair.first); + } + + ledgerHashToSeq_.erase(ledgerToDelete.info.hash); + ledgers_.erase(seq); + } + + if (deleted > 0) + { + for (auto& [account, data] : accountTxMap_) + { + auto it = data.ledgerTxMap.begin(); + while (it != data.ledgerTxMap.end()) + { + if (it->first < cutoffSeq) + { + for (const auto& seqPair : it->second) + { + if (seqPair.second < + data.transactions.size()) + { + auto& txPair = + data.transactions[seqPair.second]; + txPair.first.reset(); + txPair.second.reset(); + } + } + it = data.ledgerTxMap.erase(it); + } + else + { + ++it; + } + } + + data.transactions.erase( + std::remove_if( + data.transactions.begin(), + data.transactions.end(), + [](const auto& tx) { + return !tx.first && !tx.second; + }), + data.transactions.end()); + + for (auto& [ledgerSeq, txMap] : data.ledgerTxMap) + { + for (auto& [txSeq, index] : txMap) + { + auto newIndex = std::distance( + data.transactions.begin(), + std::find( + data.transactions.begin(), + data.transactions.end(), + data.transactions[index])); + index = newIndex; + } + } + } + + app_.getLedgerMaster().clearPriorLedgers(cutoffSeq); + } + } + } + + return true; + } + + std::optional + getLedgerInfoByIndex(LedgerIndex ledgerSeq) override + { + std::shared_lock lock(mutex_); + auto it = ledgers_.find(ledgerSeq); + if (it != ledgers_.end()) + return it->second.info; + return std::nullopt; + } + + std::optional + getNewestLedgerInfo() override + { + std::shared_lock lock(mutex_); + if (ledgers_.empty()) + return std::nullopt; + return ledgers_.rbegin()->second.info; + } + + std::optional + getLimitedOldestLedgerInfo(LedgerIndex ledgerFirstIndex) override + { + std::shared_lock lock(mutex_); + auto it = ledgers_.lower_bound(ledgerFirstIndex); + if (it != ledgers_.end()) + return it->second.info; + return std::nullopt; + } + + std::optional + getLimitedNewestLedgerInfo(LedgerIndex ledgerFirstIndex) override + { + std::shared_lock lock(mutex_); + auto it = ledgers_.lower_bound(ledgerFirstIndex); + if (it == ledgers_.end()) + return std::nullopt; + return ledgers_.rbegin()->second.info; + } + + std::optional + getLedgerInfoByHash(uint256 const& ledgerHash) override + { + std::shared_lock lock(mutex_); + auto it = ledgerHashToSeq_.find(ledgerHash); + if (it != ledgerHashToSeq_.end()) + return ledgers_.at(it->second).info; + return std::nullopt; + } + uint256 + getHashByIndex(LedgerIndex ledgerIndex) override + { + std::shared_lock lock(mutex_); + auto it = ledgers_.find(ledgerIndex); + if (it != ledgers_.end()) + return it->second.info.hash; + return uint256(); + } + + std::optional + getHashesByIndex(LedgerIndex ledgerIndex) override + { + std::shared_lock lock(mutex_); + auto it = ledgers_.find(ledgerIndex); + if (it != ledgers_.end()) + { + return LedgerHashPair{ + it->second.info.hash, it->second.info.parentHash}; + } + return std::nullopt; + } + + std::map + getHashesByIndex(LedgerIndex minSeq, LedgerIndex maxSeq) override + { + std::shared_lock lock(mutex_); + std::map result; + auto it = ledgers_.lower_bound(minSeq); + auto end = ledgers_.upper_bound(maxSeq); + for (; it != end; ++it) + { + result[it->first] = LedgerHashPair{ + it->second.info.hash, it->second.info.parentHash}; + } + return result; + } + + std::variant + getTransaction( + uint256 const& id, + std::optional> const& range, + error_code_i& ec) override + { + std::shared_lock lock(mutex_); + auto it = transactionMap_.find(id); + if (it != transactionMap_.end()) + { + const auto& [txn, txMeta] = it->second; + if (!range || + (range->lower() <= txMeta->getLgrSeq() && + txMeta->getLgrSeq() <= range->upper())) + return it->second; + } + + if (range) + { + bool allPresent = true; + for (LedgerIndex seq = range->lower(); seq <= range->upper(); ++seq) + { + if (ledgers_.find(seq) == ledgers_.end()) + { + allPresent = false; + break; + } + } + return allPresent ? TxSearched::all : TxSearched::some; + } + + return TxSearched::unknown; + } + + bool + ledgerDbHasSpace(Config const& config) override + { + return true; // In-memory database always has space + } + + bool + transactionDbHasSpace(Config const& config) override + { + return true; // In-memory database always has space + } + + std::uint32_t + getKBUsedAll() override + { + std::shared_lock lock(mutex_); + std::uint32_t size = sizeof(*this); + size += ledgers_.size() * (sizeof(LedgerIndex) + sizeof(LedgerData)); + size += + ledgerHashToSeq_.size() * (sizeof(uint256) + sizeof(LedgerIndex)); + size += transactionMap_.size() * (sizeof(uint256) + sizeof(AccountTx)); + for (const auto& [_, accountData] : accountTxMap_) + { + size += sizeof(AccountID) + sizeof(AccountTxData); + size += accountData.transactions.size() * sizeof(AccountTx); + for (const auto& [_, innerMap] : accountData.ledgerTxMap) + { + size += sizeof(uint32_t) + + innerMap.size() * (sizeof(uint32_t) + sizeof(size_t)); + } + } + return size / 1024; + } + + std::uint32_t + getKBUsedLedger() override + { + std::shared_lock lock(mutex_); + std::uint32_t size = 0; + size += ledgers_.size() * (sizeof(LedgerIndex) + sizeof(LedgerData)); + size += + ledgerHashToSeq_.size() * (sizeof(uint256) + sizeof(LedgerIndex)); + return size / 1024; + } + + std::uint32_t + getKBUsedTransaction() override + { + std::shared_lock lock(mutex_); + std::uint32_t size = 0; + size += transactionMap_.size() * (sizeof(uint256) + sizeof(AccountTx)); + for (const auto& [_, accountData] : accountTxMap_) + { + size += sizeof(AccountID) + sizeof(AccountTxData); + size += accountData.transactions.size() * sizeof(AccountTx); + for (const auto& [_, innerMap] : accountData.ledgerTxMap) + { + size += sizeof(uint32_t) + + innerMap.size() * (sizeof(uint32_t) + sizeof(size_t)); + } + } + return size / 1024; + } + + void + closeLedgerDB() override + { + // No-op for in-memory database + } + + void + closeTransactionDB() override + { + // No-op for in-memory database + } + + ~RWDBDatabase() + { + // Regular maps can use standard clear + accountTxMap_.clear(); + transactionMap_.clear(); + for (auto& ledger : ledgers_) + { + ledger.second.transactions.clear(); + } + ledgers_.clear(); + ledgerHashToSeq_.clear(); + } + + std::vector> + getTxHistory(LedgerIndex startIndex) override + { + std::shared_lock lock(mutex_); + std::vector> result; + auto it = ledgers_.lower_bound(startIndex); + int count = 0; + while (it != ledgers_.end() && count < 20) + { + for (const auto& [txHash, accountTx] : it->second.transactions) + { + result.push_back(accountTx.first); + if (++count >= 20) + break; + } + ++it; + } + return result; + } + // Helper function to handle limits + template + void + applyLimit(Container& container, std::size_t limit, bool bUnlimited) + { + if (!bUnlimited && limit > 0 && container.size() > limit) + { + container.resize(limit); + } + } + + AccountTxs + getOldestAccountTxs(AccountTxOptions const& options) override + { + std::shared_lock lock(mutex_); + auto it = accountTxMap_.find(options.account); + if (it == accountTxMap_.end()) + return {}; + + AccountTxs result; + const auto& accountData = it->second; + auto txIt = accountData.ledgerTxMap.lower_bound(options.minLedger); + auto txEnd = accountData.ledgerTxMap.upper_bound(options.maxLedger); + + std::size_t skipped = 0; + for (; txIt != txEnd && + (options.bUnlimited || result.size() < options.limit); + ++txIt) + { + for (const auto& [txSeq, txIndex] : txIt->second) + { + if (skipped < options.offset) + { + ++skipped; + continue; + } + result.push_back(accountData.transactions[txIndex]); + if (!options.bUnlimited && result.size() >= options.limit) + break; + } + } + + return result; + } + + AccountTxs + getNewestAccountTxs(AccountTxOptions const& options) override + { + std::shared_lock lock(mutex_); + auto it = accountTxMap_.find(options.account); + if (it == accountTxMap_.end()) + return {}; + + AccountTxs result; + const auto& accountData = it->second; + auto txIt = accountData.ledgerTxMap.lower_bound(options.minLedger); + auto txEnd = accountData.ledgerTxMap.upper_bound(options.maxLedger); + + std::size_t skipped = 0; + for (auto rIt = std::make_reverse_iterator(txEnd); + rIt != std::make_reverse_iterator(txIt) && + (options.bUnlimited || result.size() < options.limit); + ++rIt) + { + for (auto innerRIt = rIt->second.rbegin(); + innerRIt != rIt->second.rend(); + ++innerRIt) + { + if (skipped < options.offset) + { + ++skipped; + continue; + } + result.push_back(accountData.transactions[innerRIt->second]); + if (!options.bUnlimited && result.size() >= options.limit) + break; + } + } + + return result; + } + + MetaTxsList + getOldestAccountTxsB(AccountTxOptions const& options) override + { + std::shared_lock lock(mutex_); + auto it = accountTxMap_.find(options.account); + if (it == accountTxMap_.end()) + return {}; + + MetaTxsList result; + const auto& accountData = it->second; + auto txIt = accountData.ledgerTxMap.lower_bound(options.minLedger); + auto txEnd = accountData.ledgerTxMap.upper_bound(options.maxLedger); + + std::size_t skipped = 0; + for (; txIt != txEnd && + (options.bUnlimited || result.size() < options.limit); + ++txIt) + { + for (const auto& [txSeq, txIndex] : txIt->second) + { + if (skipped < options.offset) + { + ++skipped; + continue; + } + const auto& [txn, txMeta] = accountData.transactions[txIndex]; + result.emplace_back( + txn->getSTransaction()->getSerializer().peekData(), + txMeta->getAsObject().getSerializer().peekData(), + txIt->first); + if (!options.bUnlimited && result.size() >= options.limit) + break; + } + } + + return result; + } + + MetaTxsList + getNewestAccountTxsB(AccountTxOptions const& options) override + { + std::shared_lock lock(mutex_); + auto it = accountTxMap_.find(options.account); + if (it == accountTxMap_.end()) + return {}; + + MetaTxsList result; + const auto& accountData = it->second; + auto txIt = accountData.ledgerTxMap.lower_bound(options.minLedger); + auto txEnd = accountData.ledgerTxMap.upper_bound(options.maxLedger); + + std::size_t skipped = 0; + for (auto rIt = std::make_reverse_iterator(txEnd); + rIt != std::make_reverse_iterator(txIt) && + (options.bUnlimited || result.size() < options.limit); + ++rIt) + { + for (auto innerRIt = rIt->second.rbegin(); + innerRIt != rIt->second.rend(); + ++innerRIt) + { + if (skipped < options.offset) + { + ++skipped; + continue; + } + const auto& [txn, txMeta] = + accountData.transactions[innerRIt->second]; + result.emplace_back( + txn->getSTransaction()->getSerializer().peekData(), + txMeta->getAsObject().getSerializer().peekData(), + rIt->first); + if (!options.bUnlimited && result.size() >= options.limit) + break; + } + } + + return result; + } + std::pair> + oldestAccountTxPage(AccountTxPageOptions const& options) override + { + std::shared_lock lock(mutex_); + auto it = accountTxMap_.find(options.account); + if (it == accountTxMap_.end()) + return {{}, std::nullopt}; + + AccountTxs result; + std::optional marker; + const auto& accountData = it->second; + auto txIt = accountData.ledgerTxMap.lower_bound(options.minLedger); + auto txEnd = accountData.ledgerTxMap.upper_bound(options.maxLedger); + + bool lookingForMarker = options.marker.has_value(); + std::size_t count = 0; + + for (; txIt != txEnd && (options.limit == 0 || count < options.limit); + ++txIt) + { + for (const auto& [txSeq, txIndex] : txIt->second) + { + if (lookingForMarker) + { + if (txIt->first == options.marker->ledgerSeq && + txSeq == options.marker->txnSeq) + lookingForMarker = false; + continue; + } + + result.push_back(accountData.transactions[txIndex]); + ++count; + + if (options.limit > 0 && count >= options.limit) + { + marker = AccountTxMarker{txIt->first, txSeq}; + break; + } + } + } + + return {result, marker}; + } + + std::pair> + newestAccountTxPage(AccountTxPageOptions const& options) override + { + std::shared_lock lock(mutex_); + auto it = accountTxMap_.find(options.account); + if (it == accountTxMap_.end()) + return {{}, std::nullopt}; + + AccountTxs result; + std::optional marker; + const auto& accountData = it->second; + auto txIt = accountData.ledgerTxMap.lower_bound(options.minLedger); + auto txEnd = accountData.ledgerTxMap.upper_bound(options.maxLedger); + + bool lookingForMarker = options.marker.has_value(); + std::size_t count = 0; + + for (auto rIt = std::make_reverse_iterator(txEnd); + rIt != std::make_reverse_iterator(txIt) && + (options.limit == 0 || count < options.limit); + ++rIt) + { + for (auto innerRIt = rIt->second.rbegin(); + innerRIt != rIt->second.rend(); + ++innerRIt) + { + if (lookingForMarker) + { + if (rIt->first == options.marker->ledgerSeq && + innerRIt->first == options.marker->txnSeq) + lookingForMarker = false; + continue; + } + + result.push_back(accountData.transactions[innerRIt->second]); + ++count; + + if (options.limit > 0 && count >= options.limit) + { + marker = AccountTxMarker{rIt->first, innerRIt->first}; + break; + } + } + } + + return {result, marker}; + } + + std::pair> + oldestAccountTxPageB(AccountTxPageOptions const& options) override + { + std::shared_lock lock(mutex_); + auto it = accountTxMap_.find(options.account); + if (it == accountTxMap_.end()) + return {{}, std::nullopt}; + + MetaTxsList result; + std::optional marker; + const auto& accountData = it->second; + auto txIt = accountData.ledgerTxMap.lower_bound(options.minLedger); + auto txEnd = accountData.ledgerTxMap.upper_bound(options.maxLedger); + + bool lookingForMarker = options.marker.has_value(); + std::size_t count = 0; + + for (; txIt != txEnd && (options.limit == 0 || count < options.limit); + ++txIt) + { + for (const auto& [txSeq, txIndex] : txIt->second) + { + if (lookingForMarker) + { + if (txIt->first == options.marker->ledgerSeq && + txSeq == options.marker->txnSeq) + lookingForMarker = false; + continue; + } + + const auto& [txn, txMeta] = accountData.transactions[txIndex]; + result.emplace_back( + txn->getSTransaction()->getSerializer().peekData(), + txMeta->getAsObject().getSerializer().peekData(), + txIt->first); + ++count; + + if (options.limit > 0 && count >= options.limit) + { + marker = AccountTxMarker{txIt->first, txSeq}; + break; + } + } + } + + return {result, marker}; + } + + std::pair> + newestAccountTxPageB(AccountTxPageOptions const& options) override + { + std::shared_lock lock(mutex_); + auto it = accountTxMap_.find(options.account); + if (it == accountTxMap_.end()) + return {{}, std::nullopt}; + + MetaTxsList result; + std::optional marker; + const auto& accountData = it->second; + auto txIt = accountData.ledgerTxMap.lower_bound(options.minLedger); + auto txEnd = accountData.ledgerTxMap.upper_bound(options.maxLedger); + + bool lookingForMarker = options.marker.has_value(); + std::size_t count = 0; + + for (auto rIt = std::make_reverse_iterator(txEnd); + rIt != std::make_reverse_iterator(txIt) && + (options.limit == 0 || count < options.limit); + ++rIt) + { + for (auto innerRIt = rIt->second.rbegin(); + innerRIt != rIt->second.rend(); + ++innerRIt) + { + if (lookingForMarker) + { + if (rIt->first == options.marker->ledgerSeq && + innerRIt->first == options.marker->txnSeq) + lookingForMarker = false; + continue; + } + + const auto& [txn, txMeta] = + accountData.transactions[innerRIt->second]; + result.emplace_back( + txn->getSTransaction()->getSerializer().peekData(), + txMeta->getAsObject().getSerializer().peekData(), + rIt->first); + ++count; + + if (options.limit > 0 && count >= options.limit) + { + marker = AccountTxMarker{rIt->first, innerRIt->first}; + break; + } + } + } + + return {result, marker}; + } +}; + +// Factory function +std::unique_ptr +getRWDBDatabase(Application& app, Config const& config, JobQueue& jobQueue) +{ + return std::make_unique(app, config, jobQueue); +} + +} // namespace ripple +#endif // RIPPLE_APP_RDB_BACKEND_MEMORYDATABASE_H_INCLUDED diff --git a/src/ripple/app/rdb/impl/RelationalDatabase.cpp b/src/ripple/app/rdb/impl/RelationalDatabase.cpp index 8a3ce5b01..64161bd53 100644 --- a/src/ripple/app/rdb/impl/RelationalDatabase.cpp +++ b/src/ripple/app/rdb/impl/RelationalDatabase.cpp @@ -19,6 +19,8 @@ #include #include +#include +#include #include #include @@ -38,6 +40,8 @@ RelationalDatabase::init( { bool use_sqlite = false; bool use_postgres = false; + bool use_rwdb = false; + bool use_flatmap = false; if (config.reporting()) { @@ -52,6 +56,14 @@ RelationalDatabase::init( { use_sqlite = true; } + else if (boost::iequals(get(rdb_section, "backend"), "rwdb")) + { + use_rwdb = true; + } + else if (boost::iequals(get(rdb_section, "backend"), "flatmap")) + { + use_flatmap = true; + } else { Throw( @@ -73,6 +85,14 @@ RelationalDatabase::init( { return getPostgresDatabase(app, config, jobQueue); } + else if (use_rwdb) + { + return getRWDBDatabase(app, config, jobQueue); + } + else if (use_flatmap) + { + return getFlatmapDatabase(app, config, jobQueue); + } return std::unique_ptr(); } diff --git a/src/ripple/core/Config.h b/src/ripple/core/Config.h index d3d032b01..2779547e2 100644 --- a/src/ripple/core/Config.h +++ b/src/ripple/core/Config.h @@ -25,6 +25,7 @@ #include #include #include +#include #include #include // VFALCO Breaks levelization #include @@ -350,6 +351,21 @@ public: { return RUN_REPORTING; } + bool + mem_backend() const + { + static bool const isMem = + (!section(SECTION_RELATIONAL_DB).empty() && + boost::beast::iequals( + get(section(SECTION_RELATIONAL_DB), "backend"), "rwdb")) || + (!section("node_db").empty() && + (boost::beast::iequals(get(section("node_db"), "type"), "rwdb") || + boost::beast::iequals( + get(section("node_db"), "type"), "flatmap"))); + // RHNOTE: memory type is not selected for here because it breaks + // tests + return isMem; + } bool useTxTables() const diff --git a/src/ripple/nodestore/backend/FlatmapFactory.cpp b/src/ripple/nodestore/backend/FlatmapFactory.cpp new file mode 100644 index 000000000..4cec115ef --- /dev/null +++ b/src/ripple/nodestore/backend/FlatmapFactory.cpp @@ -0,0 +1,235 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ripple { +namespace NodeStore { + +class FlatmapBackend : public Backend +{ +private: + std::string name_; + beast::Journal journal_; + bool isOpen_{false}; + + struct base_uint_hasher + { + using result_type = std::size_t; + + result_type + operator()(base_uint<256> const& value) const + { + return hardened_hash<>{}(value); + } + }; + + using DataStore = boost::unordered::concurrent_flat_map< + uint256, + std::vector, // Store compressed blob data + base_uint_hasher>; + + DataStore table_; + +public: + FlatmapBackend( + size_t keyBytes, + Section const& keyValues, + beast::Journal journal) + : name_(get(keyValues, "path")), journal_(journal) + { + boost::ignore_unused(journal_); + if (name_.empty()) + name_ = "node_db"; + } + + ~FlatmapBackend() override + { + close(); + } + + std::string + getName() override + { + return name_; + } + + void + open(bool createIfMissing) override + { + if (isOpen_) + Throw("already open"); + isOpen_ = true; + } + + bool + isOpen() override + { + return isOpen_; + } + + void + close() override + { + table_.clear(); + isOpen_ = false; + } + + Status + fetch(void const* key, std::shared_ptr* pObject) override + { + if (!isOpen_) + return notFound; + + uint256 const hash(uint256::fromVoid(key)); + + bool found = table_.visit(hash, [&](const auto& key_value_pair) { + nudb::detail::buffer bf; + auto const result = nodeobject_decompress( + key_value_pair.second.data(), key_value_pair.second.size(), bf); + DecodedBlob decoded(hash.data(), result.first, result.second); + if (!decoded.wasOk()) + { + *pObject = nullptr; + return; + } + *pObject = decoded.createObject(); + }); + return found ? (*pObject ? ok : dataCorrupt) : notFound; + } + + std::pair>, Status> + fetchBatch(std::vector const& hashes) override + { + std::vector> results; + results.reserve(hashes.size()); + for (auto const& h : hashes) + { + std::shared_ptr nObj; + Status status = fetch(h->begin(), &nObj); + if (status != ok) + results.push_back({}); + else + results.push_back(nObj); + } + return {results, ok}; + } + + void + store(std::shared_ptr const& object) override + { + if (!isOpen_) + return; + + if (!object) + return; + + EncodedBlob encoded(object); + nudb::detail::buffer bf; + auto const result = + nodeobject_compress(encoded.getData(), encoded.getSize(), bf); + + std::vector compressed( + static_cast(result.first), + static_cast(result.first) + result.second); + + table_.insert_or_assign(object->getHash(), std::move(compressed)); + } + + void + storeBatch(Batch const& batch) override + { + for (auto const& e : batch) + store(e); + } + + void + sync() override + { + } + + void + for_each(std::function)> f) override + { + if (!isOpen_) + return; + + table_.visit_all([&f](const auto& entry) { + nudb::detail::buffer bf; + auto const result = nodeobject_decompress( + entry.second.data(), entry.second.size(), bf); + DecodedBlob decoded( + entry.first.data(), result.first, result.second); + if (decoded.wasOk()) + f(decoded.createObject()); + }); + } + + int + getWriteLoad() override + { + return 0; + } + + void + setDeletePath() override + { + close(); + } + + int + fdRequired() const override + { + return 0; + } + +private: + size_t + size() const + { + return table_.size(); + } +}; + +class FlatmapFactory : public Factory +{ +public: + FlatmapFactory() + { + Manager::instance().insert(*this); + } + + ~FlatmapFactory() override + { + Manager::instance().erase(*this); + } + + std::string + getName() const override + { + return "Flatmap"; + } + + std::unique_ptr + createInstance( + size_t keyBytes, + Section const& keyValues, + std::size_t burstSize, + Scheduler& scheduler, + beast::Journal journal) override + { + return std::make_unique(keyBytes, keyValues, journal); + } +}; + +static FlatmapFactory flatmapFactory; + +} // namespace NodeStore +} // namespace ripple diff --git a/src/ripple/nodestore/backend/MemoryFactory.cpp b/src/ripple/nodestore/backend/MemoryFactory.cpp index f82c14429..a1c7a4cfb 100644 --- a/src/ripple/nodestore/backend/MemoryFactory.cpp +++ b/src/ripple/nodestore/backend/MemoryFactory.cpp @@ -94,7 +94,7 @@ public: { boost::ignore_unused(journal_); // Keep unused journal_ just in case. if (name_.empty()) - Throw("Missing path in Memory backend"); + Throw("Missing path in TestMemory backend"); } ~MemoryBackend() override diff --git a/src/ripple/nodestore/backend/RWDBFactory.cpp b/src/ripple/nodestore/backend/RWDBFactory.cpp new file mode 100644 index 000000000..3f700038b --- /dev/null +++ b/src/ripple/nodestore/backend/RWDBFactory.cpp @@ -0,0 +1,242 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ripple { +namespace NodeStore { + +class RWDBBackend : public Backend +{ +private: + std::string name_; + beast::Journal journal_; + bool isOpen_{false}; + + struct base_uint_hasher + { + using result_type = std::size_t; + + result_type + operator()(base_uint<256> const& value) const + { + return hardened_hash<>{}(value); + } + }; + + using DataStore = + std::map>; // Store compressed blob + // data + mutable std::recursive_mutex + mutex_; // Only needed for std::map implementation + + DataStore table_; + +public: + RWDBBackend( + size_t keyBytes, + Section const& keyValues, + beast::Journal journal) + : name_(get(keyValues, "path")), journal_(journal) + { + boost::ignore_unused(journal_); + if (name_.empty()) + name_ = "node_db"; + } + + ~RWDBBackend() override + { + close(); + } + + std::string + getName() override + { + return name_; + } + + void + open(bool createIfMissing) override + { + std::lock_guard lock(mutex_); + if (isOpen_) + Throw("already open"); + isOpen_ = true; + } + + bool + isOpen() override + { + return isOpen_; + } + + void + close() override + { + std::lock_guard lock(mutex_); + table_.clear(); + isOpen_ = false; + } + + Status + fetch(void const* key, std::shared_ptr* pObject) override + { + if (!isOpen_) + return notFound; + + uint256 const hash(uint256::fromVoid(key)); + + std::lock_guard lock(mutex_); + auto it = table_.find(hash); + if (it == table_.end()) + return notFound; + + nudb::detail::buffer bf; + auto const result = + nodeobject_decompress(it->second.data(), it->second.size(), bf); + DecodedBlob decoded(hash.data(), result.first, result.second); + if (!decoded.wasOk()) + return dataCorrupt; + *pObject = decoded.createObject(); + return ok; + } + + std::pair>, Status> + fetchBatch(std::vector const& hashes) override + { + std::vector> results; + results.reserve(hashes.size()); + for (auto const& h : hashes) + { + std::shared_ptr nObj; + Status status = fetch(h->begin(), &nObj); + if (status != ok) + results.push_back({}); + else + results.push_back(nObj); + } + return {results, ok}; + } + + void + store(std::shared_ptr const& object) override + { + if (!isOpen_) + return; + + if (!object) + return; + + EncodedBlob encoded(object); + nudb::detail::buffer bf; + auto const result = + nodeobject_compress(encoded.getData(), encoded.getSize(), bf); + + std::vector compressed( + static_cast(result.first), + static_cast(result.first) + result.second); + + std::lock_guard lock(mutex_); + table_[object->getHash()] = std::move(compressed); + } + + void + storeBatch(Batch const& batch) override + { + for (auto const& e : batch) + store(e); + } + + void + sync() override + { + } + + void + for_each(std::function)> f) override + { + if (!isOpen_) + return; + + std::lock_guard lock(mutex_); + for (const auto& entry : table_) + { + nudb::detail::buffer bf; + auto const result = nodeobject_decompress( + entry.second.data(), entry.second.size(), bf); + DecodedBlob decoded( + entry.first.data(), result.first, result.second); + if (decoded.wasOk()) + f(decoded.createObject()); + } + } + + int + getWriteLoad() override + { + return 0; + } + + void + setDeletePath() override + { + close(); + } + + int + fdRequired() const override + { + return 0; + } + +private: + size_t + size() const + { + std::lock_guard lock(mutex_); + return table_.size(); + } +}; + +class RWDBFactory : public Factory +{ +public: + RWDBFactory() + { + Manager::instance().insert(*this); + } + + ~RWDBFactory() override + { + Manager::instance().erase(*this); + } + + std::string + getName() const override + { + return "RWDB"; + } + + std::unique_ptr + createInstance( + size_t keyBytes, + Section const& keyValues, + std::size_t burstSize, + Scheduler& scheduler, + beast::Journal journal) override + { + return std::make_unique(keyBytes, keyValues, journal); + } +}; + +static RWDBFactory rwDBFactory; + +} // namespace NodeStore +} // namespace ripple diff --git a/src/ripple/overlay/impl/OverlayImpl.cpp b/src/ripple/overlay/impl/OverlayImpl.cpp index 9b6ce2069..b1027826f 100644 --- a/src/ripple/overlay/impl/OverlayImpl.cpp +++ b/src/ripple/overlay/impl/OverlayImpl.cpp @@ -38,6 +38,7 @@ #include #include +#include #include #include @@ -136,7 +137,11 @@ OverlayImpl::OverlayImpl( stopwatch(), app_.journal("PeerFinder"), config, - collector)) + collector, + app.config().section(SECTION_RELATIONAL_DB).empty() || + !boost::iequals( + get(app.config().section(SECTION_RELATIONAL_DB), "backend"), + "rwdb"))) , m_resolver(resolver) , next_id_(1) , timer_count_(0) diff --git a/src/ripple/peerfinder/impl/InMemoryStore.h b/src/ripple/peerfinder/impl/InMemoryStore.h new file mode 100644 index 000000000..cae005622 --- /dev/null +++ b/src/ripple/peerfinder/impl/InMemoryStore.h @@ -0,0 +1,54 @@ +#ifndef RIPPLE_PEERFINDER_INMEMORYSTORE_H_INCLUDED +#define RIPPLE_PEERFINDER_INMEMORYSTORE_H_INCLUDED + +#include +#include +#include +#include + +namespace ripple { +namespace PeerFinder { + +struct EndpointHasher +{ + std::size_t + operator()(beast::IP::Endpoint const& endpoint) const + { + std::size_t seed = 0; + boost::hash_combine(seed, endpoint.address().to_string()); + boost::hash_combine(seed, endpoint.port()); + return seed; + } +}; + +class InMemoryStore : public Store +{ +private: + boost::concurrent_flat_map + entries; + +public: + std::size_t + load(load_callback const& cb) override + { + std::size_t count = 0; + entries.visit_all([&](auto const& entry) { + cb(entry.first, entry.second); + ++count; + }); + return count; + } + + void + save(std::vector const& v) override + { + entries.clear(); + for (auto const& entry : v) + entries.emplace(entry.endpoint, entry.valence); + } +}; + +} // namespace PeerFinder +} // namespace ripple + +#endif diff --git a/src/ripple/peerfinder/impl/PeerfinderManager.cpp b/src/ripple/peerfinder/impl/PeerfinderManager.cpp index e3743c047..258111a16 100644 --- a/src/ripple/peerfinder/impl/PeerfinderManager.cpp +++ b/src/ripple/peerfinder/impl/PeerfinderManager.cpp @@ -17,8 +17,10 @@ */ //============================================================================== +#include #include #include +#include #include #include #include @@ -38,7 +40,7 @@ public: std::optional work_; clock_type& m_clock; beast::Journal m_journal; - StoreSqdb m_store; + std::unique_ptr m_store; Checker checker_; Logic m_logic; BasicConfig const& m_config; @@ -50,15 +52,18 @@ public: clock_type& clock, beast::Journal journal, BasicConfig const& config, - beast::insight::Collector::ptr const& collector) + beast::insight::Collector::ptr const& collector, + bool useSqLiteStore) : Manager() , io_service_(io_service) , work_(std::in_place, std::ref(io_service_)) , m_clock(clock) , m_journal(journal) - , m_store(journal) + , m_store( + useSqLiteStore ? static_cast(new StoreSqdb(journal)) + : static_cast(new InMemoryStore())) , checker_(io_service_) - , m_logic(clock, m_store, checker_, journal) + , m_logic(clock, *m_store, checker_, journal) , m_config(config) , m_stats(std::bind(&ManagerImp::collect_metrics, this), collector) { @@ -215,7 +220,8 @@ public: void start() override { - m_store.open(m_config); + if (auto sqdb = dynamic_cast(m_store.get())) + sqdb->open(m_config); m_logic.load(); } @@ -275,10 +281,11 @@ make_Manager( clock_type& clock, beast::Journal journal, BasicConfig const& config, - beast::insight::Collector::ptr const& collector) + beast::insight::Collector::ptr const& collector, + bool useSqLiteStore) { return std::make_unique( - io_service, clock, journal, config, collector); + io_service, clock, journal, config, collector, useSqLiteStore); } } // namespace PeerFinder diff --git a/src/ripple/peerfinder/make_Manager.h b/src/ripple/peerfinder/make_Manager.h index 932fccb9a..0ce527125 100644 --- a/src/ripple/peerfinder/make_Manager.h +++ b/src/ripple/peerfinder/make_Manager.h @@ -34,7 +34,8 @@ make_Manager( clock_type& clock, beast::Journal journal, BasicConfig const& config, - beast::insight::Collector::ptr const& collector); + beast::insight::Collector::ptr const& collector, + bool useSqliteStore); } // namespace PeerFinder } // namespace ripple diff --git a/src/ripple/shamap/impl/SHAMap.cpp b/src/ripple/shamap/impl/SHAMap.cpp index ce031003c..7adee6f13 100644 --- a/src/ripple/shamap/impl/SHAMap.cpp +++ b/src/ripple/shamap/impl/SHAMap.cpp @@ -244,6 +244,7 @@ SHAMap::checkFilter(SHAMapHash const& hash, SHAMapSyncFilter* filter) const // Get a node without throwing // Used on maps where missing nodes are expected +/* std::shared_ptr SHAMap::fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter* filter) const { @@ -266,6 +267,49 @@ SHAMap::fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter* filter) const return node; } +*/ + +std::shared_ptr +SHAMap::fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter* filter) const +{ + using namespace std::chrono; + auto start = high_resolution_clock::now(); + auto timeout = nanoseconds(50); + + while (true) + { + // Try to fetch from cache first + auto node = cacheLookup(hash); + if (node) + return node; + + if (backed_) + { + node = fetchNodeFromDB(hash); + if (node) + { + canonicalize(hash, node); + return node; + } + } + + if (filter) + node = checkFilter(hash, filter); + + if (node) + return node; + + // Check if we've exceeded timeout + auto elapsed = high_resolution_clock::now() - start; + if (elapsed >= timeout) + break; + + // Short yield to avoid overwhelming CPU + std::this_thread::yield(); + } + + return nullptr; +} std::shared_ptr SHAMap::fetchNodeNT(SHAMapHash const& hash) const diff --git a/src/test/nodestore/Backend_test.cpp b/src/test/nodestore/Backend_test.cpp index 8c8432bb5..10ba4e4ad 100644 --- a/src/test/nodestore/Backend_test.cpp +++ b/src/test/nodestore/Backend_test.cpp @@ -82,6 +82,8 @@ public: } } + // rwdb backend does not keep table/data after close + if (type != "rwdb") { // Re-open the backend std::unique_ptr backend = Manager::instance().make_Backend( @@ -105,6 +107,8 @@ public: { std::uint64_t const seedValue = 50; + testBackend("memory", seedValue); + testBackend("rwdb", seedValue); testBackend("nudb", seedValue); #if RIPPLE_ROCKSDB_AVAILABLE @@ -117,7 +121,7 @@ public: } }; -BEAST_DEFINE_TESTSUITE(Backend, ripple_core, ripple); +BEAST_DEFINE_TESTSUITE(Backend, NodeStore, ripple); } // namespace NodeStore } // namespace ripple diff --git a/src/test/nodestore/Database_test.cpp b/src/test/nodestore/Database_test.cpp index 0cf2afb21..80c0808bb 100644 --- a/src/test/nodestore/Database_test.cpp +++ b/src/test/nodestore/Database_test.cpp @@ -661,6 +661,8 @@ public: testNodeStore("memory", false, seedValue); + testNodeStore("rwdb", false, seedValue); + // Persistent backend tests { testNodeStore("nudb", true, seedValue); From daf22b3b85b5555537a21cfe1bcbac5984677949 Mon Sep 17 00:00:00 2001 From: Denis Angell Date: Thu, 14 Nov 2024 22:31:55 +0100 Subject: [PATCH 10/11] Fix: RWDB (#389) --- src/ripple/app/misc/SHAMapStoreImp.cpp | 63 +- src/ripple/app/rdb/backend/RWDBDatabase.h | 766 ++++++++++++---------- src/test/app/LedgerLoad_test.cpp | 11 +- src/test/app/LedgerReplay_test.cpp | 8 +- src/test/jtx/impl/envconfig.cpp | 3 +- src/test/nodestore/Database_test.cpp | 2 +- src/test/rpc/GetCounts_test.cpp | 3 - src/test/rpc/NodeToShardRPC_test.cpp | 6 + src/test/shamap/common.h | 2 +- 9 files changed, 474 insertions(+), 390 deletions(-) diff --git a/src/ripple/app/misc/SHAMapStoreImp.cpp b/src/ripple/app/misc/SHAMapStoreImp.cpp index 34cb86ed8..48347537b 100644 --- a/src/ripple/app/misc/SHAMapStoreImp.cpp +++ b/src/ripple/app/misc/SHAMapStoreImp.cpp @@ -118,9 +118,7 @@ SHAMapStoreImp::SHAMapStoreImp( get_if_exists(section, "online_delete", deleteInterval_); - bool const isMem = config.mem_backend(); - - if (deleteInterval_ || isMem) + if (deleteInterval_) { if (app_.config().reporting()) { @@ -129,9 +127,6 @@ SHAMapStoreImp::SHAMapStoreImp( "online_delete info from config"); } - if (isMem) - deleteInterval_ = config.LEDGER_HISTORY; - // Configuration that affects the behavior of online delete get_if_exists(section, "delete_batch", deleteBatch_); std::uint32_t temp; @@ -167,7 +162,7 @@ SHAMapStoreImp::SHAMapStoreImp( } state_db_.init(config, dbName_); - if (!isMem) + if (!config.mem_backend()) dbPaths(); } } @@ -649,6 +644,33 @@ SHAMapStoreImp::clearPrior(LedgerIndex lastRotated) if (!db) Throw("Failed to get relational database"); + if (app_.config().useTxTables()) + { + clearSql( + lastRotated, + "Transactions", + [&db]() -> std::optional { + return db->getTransactionsMinLedgerSeq(); + }, + [&db](LedgerIndex min) -> void { + db->deleteTransactionsBeforeLedgerSeq(min); + }); + if (healthWait() == stopping) + return; + + clearSql( + lastRotated, + "AccountTransactions", + [&db]() -> std::optional { + return db->getAccountTransactionsMinLedgerSeq(); + }, + [&db](LedgerIndex min) -> void { + db->deleteAccountTransactionsBeforeLedgerSeq(min); + }); + if (healthWait() == stopping) + return; + } + clearSql( lastRotated, "Ledgers", @@ -656,33 +678,6 @@ SHAMapStoreImp::clearPrior(LedgerIndex lastRotated) [db](LedgerIndex min) -> void { db->deleteBeforeLedgerSeq(min); }); if (healthWait() == stopping) return; - - if (!app_.config().useTxTables()) - return; - - clearSql( - lastRotated, - "Transactions", - [&db]() -> std::optional { - return db->getTransactionsMinLedgerSeq(); - }, - [&db](LedgerIndex min) -> void { - db->deleteTransactionsBeforeLedgerSeq(min); - }); - if (healthWait() == stopping) - return; - - clearSql( - lastRotated, - "AccountTransactions", - [&db]() -> std::optional { - return db->getAccountTransactionsMinLedgerSeq(); - }, - [&db](LedgerIndex min) -> void { - db->deleteAccountTransactionsBeforeLedgerSeq(min); - }); - if (healthWait() == stopping) - return; } SHAMapStoreImp::HealthResult diff --git a/src/ripple/app/rdb/backend/RWDBDatabase.h b/src/ripple/app/rdb/backend/RWDBDatabase.h index 0a84f48ce..9c6d70e7e 100644 --- a/src/ripple/app/rdb/backend/RWDBDatabase.h +++ b/src/ripple/app/rdb/backend/RWDBDatabase.h @@ -3,7 +3,10 @@ #include #include +#include +#include #include +#include #include #include #include @@ -31,6 +34,7 @@ private: }; Application& app_; + bool const useTxTables_; mutable std::shared_mutex mutex_; @@ -41,7 +45,7 @@ private: public: RWDBDatabase(Application& app, Config const& config, JobQueue& jobQueue) - : app_(app) + : app_(app), useTxTables_(config.useTxTables()) { } @@ -57,6 +61,9 @@ public: std::optional getTransactionsMinLedgerSeq() override { + if (!useTxTables_) + return {}; + std::shared_lock lock(mutex_); if (transactionMap_.empty()) return std::nullopt; @@ -66,6 +73,9 @@ public: std::optional getAccountTransactionsMinLedgerSeq() override { + if (!useTxTables_) + return {}; + std::shared_lock lock(mutex_); if (accountTxMap_.empty()) return std::nullopt; @@ -92,6 +102,9 @@ public: void deleteTransactionByLedgerSeq(LedgerIndex ledgerSeq) override { + if (!useTxTables_) + return; + std::unique_lock lock(mutex_); auto it = ledgers_.find(ledgerSeq); if (it != ledgers_.end()) @@ -102,18 +115,6 @@ public: } it->second.transactions.clear(); } - for (auto& [_, accountData] : accountTxMap_) - { - accountData.ledgerTxMap.erase(ledgerSeq); - accountData.transactions.erase( - std::remove_if( - accountData.transactions.begin(), - accountData.transactions.end(), - [ledgerSeq](const AccountTx& tx) { - return tx.second->getLgrSeq() == ledgerSeq; - }), - accountData.transactions.end()); - } } void @@ -123,69 +124,36 @@ public: auto it = ledgers_.begin(); while (it != ledgers_.end() && it->first < ledgerSeq) { - for (const auto& [txHash, _] : it->second.transactions) - { - transactionMap_.erase(txHash); - } ledgerHashToSeq_.erase(it->second.info.hash); it = ledgers_.erase(it); } - for (auto& [_, accountData] : accountTxMap_) - { - auto txIt = accountData.ledgerTxMap.begin(); - while (txIt != accountData.ledgerTxMap.end() && - txIt->first < ledgerSeq) - { - txIt = accountData.ledgerTxMap.erase(txIt); - } - accountData.transactions.erase( - std::remove_if( - accountData.transactions.begin(), - accountData.transactions.end(), - [ledgerSeq](const AccountTx& tx) { - return tx.second->getLgrSeq() < ledgerSeq; - }), - accountData.transactions.end()); - } } void deleteTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) override { + if (!useTxTables_) + return; + std::unique_lock lock(mutex_); - for (auto& [seq, ledgerData] : ledgers_) + auto it = ledgers_.begin(); + while (it != ledgers_.end() && it->first < ledgerSeq) { - if (seq < ledgerSeq) + for (const auto& [txHash, _] : it->second.transactions) { - for (const auto& [txHash, _] : ledgerData.transactions) - { - transactionMap_.erase(txHash); - } - ledgerData.transactions.clear(); + transactionMap_.erase(txHash); } - } - for (auto& [_, accountData] : accountTxMap_) - { - auto txIt = accountData.ledgerTxMap.begin(); - while (txIt != accountData.ledgerTxMap.end() && - txIt->first < ledgerSeq) - { - txIt = accountData.ledgerTxMap.erase(txIt); - } - accountData.transactions.erase( - std::remove_if( - accountData.transactions.begin(), - accountData.transactions.end(), - [ledgerSeq](const AccountTx& tx) { - return tx.second->getLgrSeq() < ledgerSeq; - }), - accountData.transactions.end()); + it->second.transactions.clear(); + ++it; } } void deleteAccountTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) override { + if (!useTxTables_) + return; + std::unique_lock lock(mutex_); for (auto& [_, accountData] : accountTxMap_) { @@ -208,6 +176,9 @@ public: std::size_t getTransactionCount() override { + if (!useTxTables_) + return 0; + std::shared_lock lock(mutex_); return transactionMap_.size(); } @@ -215,6 +186,9 @@ public: std::size_t getAccountTransactionCount() override { + if (!useTxTables_) + return 0; + std::shared_lock lock(mutex_); std::size_t count = 0; for (const auto& [_, accountData] : accountTxMap_) @@ -242,131 +216,100 @@ public: std::unique_lock lock(mutex_); LedgerData ledgerData; ledgerData.info = ledger->info(); - auto aLedger = std::make_shared(ledger, app_); + auto j = app_.journal("Ledger"); + auto seq = ledger->info().seq; - for (auto const& acceptedLedgerTx : *aLedger) + JLOG(j.trace()) << "saveValidatedLedger " + << (current ? "" : "fromAcquire ") << seq; + + if (!ledger->info().accountHash.isNonZero()) { - auto const& txn = acceptedLedgerTx->getTxn(); - auto const& meta = acceptedLedgerTx->getMeta(); - auto const& id = txn->getTransactionID(); - std::string reason; + JLOG(j.fatal()) << "AH is zero: " << getJson({*ledger, {}}); + assert(false); + } - auto accTx = std::make_pair( - std::make_shared(txn, reason, app_), - std::make_shared(meta)); + if (ledger->info().accountHash != + ledger->stateMap().getHash().as_uint256()) + { + JLOG(j.fatal()) << "sAL: " << ledger->info().accountHash + << " != " << ledger->stateMap().getHash(); + JLOG(j.fatal()) + << "saveAcceptedLedger: seq=" << seq << ", current=" << current; + assert(false); + } - ledgerData.transactions.emplace(id, accTx); - transactionMap_.emplace(id, accTx); + assert(ledger->info().txHash == ledger->txMap().getHash().as_uint256()); - for (auto const& account : meta.getAffectedAccounts()) + // Save the ledger header in the hashed object store + { + Serializer s(128); + s.add32(HashPrefix::ledgerMaster); + addRaw(ledger->info(), s); + app_.getNodeStore().store( + hotLEDGER, std::move(s.modData()), ledger->info().hash, seq); + } + + std::shared_ptr aLedger; + try + { + aLedger = app_.getAcceptedLedgerCache().fetch(ledger->info().hash); + if (!aLedger) { - if (accountTxMap_.find(account) == accountTxMap_.end()) - accountTxMap_[account] = AccountTxData(); - auto& accountData = accountTxMap_[account]; - accountData.transactions.push_back(accTx); - accountData.ledgerTxMap[ledger->info().seq] - [acceptedLedgerTx->getTxnSeq()] = - accountData.transactions.size() - 1; - } - } - - ledgers_[ledger->info().seq] = std::move(ledgerData); - ledgerHashToSeq_[ledger->info().hash] = ledger->info().seq; - - if (current) - { - auto const cutoffSeq = - ledger->info().seq > app_.config().LEDGER_HISTORY - ? ledger->info().seq - app_.config().LEDGER_HISTORY - : 0; - - if (cutoffSeq > 0) - { - const std::size_t BATCH_SIZE = 128; - std::size_t deleted = 0; - - std::vector ledgersToDelete; - for (const auto& item : ledgers_) - { - if (deleted >= BATCH_SIZE) - break; - if (item.first < cutoffSeq) - { - ledgersToDelete.push_back(item.first); - deleted++; - } - } - - for (auto seq : ledgersToDelete) - { - auto& ledgerToDelete = ledgers_[seq]; - - for (const auto& txPair : ledgerToDelete.transactions) - { - transactionMap_.erase(txPair.first); - } - - ledgerHashToSeq_.erase(ledgerToDelete.info.hash); - ledgers_.erase(seq); - } - - if (deleted > 0) - { - for (auto& [account, data] : accountTxMap_) - { - auto it = data.ledgerTxMap.begin(); - while (it != data.ledgerTxMap.end()) - { - if (it->first < cutoffSeq) - { - for (const auto& seqPair : it->second) - { - if (seqPair.second < - data.transactions.size()) - { - auto& txPair = - data.transactions[seqPair.second]; - txPair.first.reset(); - txPair.second.reset(); - } - } - it = data.ledgerTxMap.erase(it); - } - else - { - ++it; - } - } - - data.transactions.erase( - std::remove_if( - data.transactions.begin(), - data.transactions.end(), - [](const auto& tx) { - return !tx.first && !tx.second; - }), - data.transactions.end()); - - for (auto& [ledgerSeq, txMap] : data.ledgerTxMap) - { - for (auto& [txSeq, index] : txMap) - { - auto newIndex = std::distance( - data.transactions.begin(), - std::find( - data.transactions.begin(), - data.transactions.end(), - data.transactions[index])); - index = newIndex; - } - } - } - - app_.getLedgerMaster().clearPriorLedgers(cutoffSeq); - } + aLedger = std::make_shared(ledger, app_); + app_.getAcceptedLedgerCache().canonicalize_replace_client( + ledger->info().hash, aLedger); + } + } + catch (std::exception const&) + { + JLOG(j.warn()) << "An accepted ledger was missing nodes"; + app_.getLedgerMaster().failedSave(seq, ledger->info().hash); + // Clients can now trust the database for information about this + // ledger sequence. + app_.pendingSaves().finishWork(seq); + return false; + } + + // Overwrite Current Ledger Transactions + if (useTxTables_) + { + for (auto const& acceptedLedgerTx : *aLedger) + { + auto const& txn = acceptedLedgerTx->getTxn(); + auto const& meta = acceptedLedgerTx->getMeta(); + auto const& id = txn->getTransactionID(); + std::string reason; + + auto accTx = std::make_pair( + std::make_shared(txn, reason, app_), + std::make_shared(meta)); + + ledgerData.transactions.emplace(id, accTx); + transactionMap_.emplace(id, accTx); + + for (auto const& account : meta.getAffectedAccounts()) + { + if (accountTxMap_.find(account) == accountTxMap_.end()) + accountTxMap_[account] = AccountTxData(); + + auto& accountData = accountTxMap_[account]; + accountData.transactions.push_back(accTx); + accountData + .ledgerTxMap[seq][acceptedLedgerTx->getTxnSeq()] = + accountData.transactions.size() - 1; + } + + app_.getMasterTransaction().inLedger( + id, + seq, + acceptedLedgerTx->getTxnSeq(), + app_.config().NETWORK_ID); } } + // Overwrite Current Ledger + ledgers_[seq] = std::move(ledgerData); + ledgerHashToSeq_[ledger->info().hash] = seq; return true; } @@ -462,29 +405,35 @@ public: std::optional> const& range, error_code_i& ec) override { + if (!useTxTables_) + return TxSearched::unknown; + std::shared_lock lock(mutex_); auto it = transactionMap_.find(id); if (it != transactionMap_.end()) { const auto& [txn, txMeta] = it->second; - if (!range || - (range->lower() <= txMeta->getLgrSeq() && - txMeta->getLgrSeq() <= range->upper())) - return it->second; + std::uint32_t inLedger = + rangeCheckedCast(txMeta->getLgrSeq()); + it->second.first->setStatus(COMMITTED); + it->second.first->setLedger(inLedger); + return it->second; } if (range) { - bool allPresent = true; - for (LedgerIndex seq = range->lower(); seq <= range->upper(); ++seq) + std::size_t count = 0; + for (LedgerIndex seq = range->first(); seq <= range->last(); ++seq) { - if (ledgers_.find(seq) == ledgers_.end()) + if (ledgers_.find(seq) != ledgers_.end()) { - allPresent = false; - break; + if (ledgers_[seq].transactions.size() > 0) + ++count; } } - return allPresent ? TxSearched::all : TxSearched::some; + return (count == (range->last() - range->first() + 1)) + ? TxSearched::all + : TxSearched::some; } return TxSearched::unknown; @@ -538,6 +487,9 @@ public: std::uint32_t getKBUsedTransaction() override { + if (!useTxTables_) + return 0; + std::shared_lock lock(mutex_); std::uint32_t size = 0; size += transactionMap_.size() * (sizeof(uint256) + sizeof(AccountTx)); @@ -582,22 +534,45 @@ public: std::vector> getTxHistory(LedgerIndex startIndex) override { + if (!useTxTables_) + return {}; + std::shared_lock lock(mutex_); std::vector> result; - auto it = ledgers_.lower_bound(startIndex); - int count = 0; - while (it != ledgers_.end() && count < 20) + + int skipped = 0; + int collected = 0; + + for (auto it = ledgers_.rbegin(); it != ledgers_.rend(); ++it) { - for (const auto& [txHash, accountTx] : it->second.transactions) + const auto& transactions = it->second.transactions; + for (const auto& [txHash, accountTx] : transactions) { - result.push_back(accountTx.first); - if (++count >= 20) + if (skipped < startIndex) + { + ++skipped; + continue; + } + + if (collected >= 20) + { break; + } + + std::uint32_t const inLedger = rangeCheckedCast( + accountTx.second->getLgrSeq()); + accountTx.first->setStatus(COMMITTED); + accountTx.first->setLedger(inLedger); + result.push_back(accountTx.first); + ++collected; } - ++it; + + if (collected >= 20) + break; } return result; } + // Helper function to handle limits template void @@ -612,6 +587,9 @@ public: AccountTxs getOldestAccountTxs(AccountTxOptions const& options) override { + if (!useTxTables_) + return {}; + std::shared_lock lock(mutex_); auto it = accountTxMap_.find(options.account); if (it == accountTxMap_.end()) @@ -634,7 +612,12 @@ public: ++skipped; continue; } - result.push_back(accountData.transactions[txIndex]); + AccountTx const accountTx = accountData.transactions[txIndex]; + std::uint32_t const inLedger = rangeCheckedCast( + accountTx.second->getLgrSeq()); + accountTx.first->setStatus(COMMITTED); + accountTx.first->setLedger(inLedger); + result.push_back(accountTx); if (!options.bUnlimited && result.size() >= options.limit) break; } @@ -646,6 +629,9 @@ public: AccountTxs getNewestAccountTxs(AccountTxOptions const& options) override { + if (!useTxTables_) + return {}; + std::shared_lock lock(mutex_); auto it = accountTxMap_.find(options.account); if (it == accountTxMap_.end()) @@ -671,7 +657,12 @@ public: ++skipped; continue; } - result.push_back(accountData.transactions[innerRIt->second]); + AccountTx const accountTx = + accountData.transactions[innerRIt->second]; + std::uint32_t const inLedger = rangeCheckedCast( + accountTx.second->getLgrSeq()); + accountTx.first->setLedger(inLedger); + result.push_back(accountTx); if (!options.bUnlimited && result.size() >= options.limit) break; } @@ -683,6 +674,9 @@ public: MetaTxsList getOldestAccountTxsB(AccountTxOptions const& options) override { + if (!useTxTables_) + return {}; + std::shared_lock lock(mutex_); auto it = accountTxMap_.find(options.account); if (it == accountTxMap_.end()) @@ -721,6 +715,9 @@ public: MetaTxsList getNewestAccountTxsB(AccountTxOptions const& options) override { + if (!useTxTables_) + return {}; + std::shared_lock lock(mutex_); auto it = accountTxMap_.find(options.account); if (it == accountTxMap_.end()) @@ -759,197 +756,270 @@ public: return result; } - std::pair> - oldestAccountTxPage(AccountTxPageOptions const& options) override + + std::pair, int> + accountTxPage( + std::function const& onUnsavedLedger, + std::function< + void(std::uint32_t, std::string const&, Blob&&, Blob&&)> const& + onTransaction, + RelationalDatabase::AccountTxPageOptions const& options, + int limit_used, + std::uint32_t page_length, + bool forward) { std::shared_lock lock(mutex_); auto it = accountTxMap_.find(options.account); if (it == accountTxMap_.end()) - return {{}, std::nullopt}; + return {std::nullopt, 0}; - AccountTxs result; - std::optional marker; - const auto& accountData = it->second; - auto txIt = accountData.ledgerTxMap.lower_bound(options.minLedger); - auto txEnd = accountData.ledgerTxMap.upper_bound(options.maxLedger); + int total = 0; bool lookingForMarker = options.marker.has_value(); - std::size_t count = 0; - for (; txIt != txEnd && (options.limit == 0 || count < options.limit); - ++txIt) + std::uint32_t numberOfResults; + + if (options.limit == 0 || options.limit == UINT32_MAX || + (options.limit > page_length && !options.bAdmin)) + numberOfResults = page_length; + else + numberOfResults = options.limit; + + if (numberOfResults < limit_used) + return {options.marker, -1}; + numberOfResults -= limit_used; + + // As an account can have many thousands of transactions, there is a + // limit placed on the amount of transactions returned. If the limit is + // reached before the result set has been exhausted (we always query for + // one more than the limit), then we return an opaque marker that can be + // supplied in a subsequent query. + std::uint32_t queryLimit = numberOfResults + 1; + std::uint32_t findLedger = 0, findSeq = 0; + + if (lookingForMarker) { - for (const auto& [txSeq, txIndex] : txIt->second) + findLedger = options.marker->ledgerSeq; + findSeq = options.marker->txnSeq; + } + + std::optional newmarker; + if (limit_used > 0) + newmarker = options.marker; + + if (forward) + { + // Oldest (forward = true) + const auto& accountData = it->second; + auto txIt = accountData.ledgerTxMap.lower_bound( + findLedger == 0 ? options.minLedger : findLedger); + auto txEnd = accountData.ledgerTxMap.upper_bound(options.maxLedger); + for (; txIt != txEnd; ++txIt) { - if (lookingForMarker) + std::uint32_t const ledgerSeq = txIt->first; + for (auto seqIt = txIt->second.begin(); + seqIt != txIt->second.end(); + ++seqIt) { - if (txIt->first == options.marker->ledgerSeq && - txSeq == options.marker->txnSeq) - lookingForMarker = false; - continue; - } + const auto& [txnSeq, index] = *seqIt; + if (lookingForMarker) + { + if (findLedger == ledgerSeq && findSeq == txnSeq) + { + lookingForMarker = false; + } + else + continue; + } + else if (numberOfResults == 0) + { + newmarker = { + rangeCheckedCast(ledgerSeq), txnSeq}; + return {newmarker, total}; + } - result.push_back(accountData.transactions[txIndex]); - ++count; + Blob rawTxn = accountData.transactions[index] + .first->getSTransaction() + ->getSerializer() + .peekData(); + Blob rawMeta = accountData.transactions[index] + .second->getAsObject() + .getSerializer() + .peekData(); - if (options.limit > 0 && count >= options.limit) - { - marker = AccountTxMarker{txIt->first, txSeq}; - break; + if (rawMeta.size() == 0) + onUnsavedLedger(ledgerSeq); + + onTransaction( + rangeCheckedCast(ledgerSeq), + "COMMITTED", + std::move(rawTxn), + std::move(rawMeta)); + --numberOfResults; + ++total; } } } + else + { + // Newest (forward = false) + const auto& accountData = it->second; + auto txIt = accountData.ledgerTxMap.lower_bound(options.minLedger); + auto txEnd = accountData.ledgerTxMap.upper_bound( + findLedger == 0 ? options.maxLedger : findLedger); + auto rtxIt = std::make_reverse_iterator(txEnd); + auto rtxEnd = std::make_reverse_iterator(txIt); + for (; rtxIt != rtxEnd; ++rtxIt) + { + std::uint32_t const ledgerSeq = rtxIt->first; + for (auto innerRIt = rtxIt->second.rbegin(); + innerRIt != rtxIt->second.rend(); + ++innerRIt) + { + const auto& [txnSeq, index] = *innerRIt; + if (lookingForMarker) + { + if (findLedger == ledgerSeq && findSeq == txnSeq) + { + lookingForMarker = false; + } + else + continue; + } + else if (numberOfResults == 0) + { + newmarker = { + rangeCheckedCast(ledgerSeq), txnSeq}; + return {newmarker, total}; + } - return {result, marker}; + Blob rawTxn = accountData.transactions[index] + .first->getSTransaction() + ->getSerializer() + .peekData(); + Blob rawMeta = accountData.transactions[index] + .second->getAsObject() + .getSerializer() + .peekData(); + + if (rawMeta.size() == 0) + onUnsavedLedger(ledgerSeq); + + onTransaction( + rangeCheckedCast(ledgerSeq), + "COMMITTED", + std::move(rawTxn), + std::move(rawMeta)); + --numberOfResults; + ++total; + } + } + } + return {newmarker, total}; + } + + std::pair> + oldestAccountTxPage(AccountTxPageOptions const& options) override + { + if (!useTxTables_) + return {}; + + static std::uint32_t const page_length(200); + auto onUnsavedLedger = + std::bind(saveLedgerAsync, std::ref(app_), std::placeholders::_1); + AccountTxs ret; + Application& app = app_; + auto onTransaction = [&ret, &app]( + std::uint32_t ledger_index, + std::string const& status, + Blob&& rawTxn, + Blob&& rawMeta) { + convertBlobsToTxResult( + ret, ledger_index, status, rawTxn, rawMeta, app); + }; + + auto newmarker = + accountTxPage( + onUnsavedLedger, onTransaction, options, 0, page_length, true) + .first; + return {ret, newmarker}; } std::pair> newestAccountTxPage(AccountTxPageOptions const& options) override { - std::shared_lock lock(mutex_); - auto it = accountTxMap_.find(options.account); - if (it == accountTxMap_.end()) - return {{}, std::nullopt}; + if (!useTxTables_) + return {}; - AccountTxs result; - std::optional marker; - const auto& accountData = it->second; - auto txIt = accountData.ledgerTxMap.lower_bound(options.minLedger); - auto txEnd = accountData.ledgerTxMap.upper_bound(options.maxLedger); + static std::uint32_t const page_length(200); + auto onUnsavedLedger = + std::bind(saveLedgerAsync, std::ref(app_), std::placeholders::_1); + AccountTxs ret; + Application& app = app_; + auto onTransaction = [&ret, &app]( + std::uint32_t ledger_index, + std::string const& status, + Blob&& rawTxn, + Blob&& rawMeta) { + convertBlobsToTxResult( + ret, ledger_index, status, rawTxn, rawMeta, app); + }; - bool lookingForMarker = options.marker.has_value(); - std::size_t count = 0; - - for (auto rIt = std::make_reverse_iterator(txEnd); - rIt != std::make_reverse_iterator(txIt) && - (options.limit == 0 || count < options.limit); - ++rIt) - { - for (auto innerRIt = rIt->second.rbegin(); - innerRIt != rIt->second.rend(); - ++innerRIt) - { - if (lookingForMarker) - { - if (rIt->first == options.marker->ledgerSeq && - innerRIt->first == options.marker->txnSeq) - lookingForMarker = false; - continue; - } - - result.push_back(accountData.transactions[innerRIt->second]); - ++count; - - if (options.limit > 0 && count >= options.limit) - { - marker = AccountTxMarker{rIt->first, innerRIt->first}; - break; - } - } - } - - return {result, marker}; + auto newmarker = + accountTxPage( + onUnsavedLedger, onTransaction, options, 0, page_length, false) + .first; + return {ret, newmarker}; } std::pair> oldestAccountTxPageB(AccountTxPageOptions const& options) override { - std::shared_lock lock(mutex_); - auto it = accountTxMap_.find(options.account); - if (it == accountTxMap_.end()) - return {{}, std::nullopt}; + if (!useTxTables_) + return {}; - MetaTxsList result; - std::optional marker; - const auto& accountData = it->second; - auto txIt = accountData.ledgerTxMap.lower_bound(options.minLedger); - auto txEnd = accountData.ledgerTxMap.upper_bound(options.maxLedger); - - bool lookingForMarker = options.marker.has_value(); - std::size_t count = 0; - - for (; txIt != txEnd && (options.limit == 0 || count < options.limit); - ++txIt) - { - for (const auto& [txSeq, txIndex] : txIt->second) - { - if (lookingForMarker) - { - if (txIt->first == options.marker->ledgerSeq && - txSeq == options.marker->txnSeq) - lookingForMarker = false; - continue; - } - - const auto& [txn, txMeta] = accountData.transactions[txIndex]; - result.emplace_back( - txn->getSTransaction()->getSerializer().peekData(), - txMeta->getAsObject().getSerializer().peekData(), - txIt->first); - ++count; - - if (options.limit > 0 && count >= options.limit) - { - marker = AccountTxMarker{txIt->first, txSeq}; - break; - } - } - } - - return {result, marker}; + static std::uint32_t const page_length(500); + auto onUnsavedLedger = + std::bind(saveLedgerAsync, std::ref(app_), std::placeholders::_1); + MetaTxsList ret; + auto onTransaction = [&ret]( + std::uint32_t ledgerIndex, + std::string const& status, + Blob&& rawTxn, + Blob&& rawMeta) { + ret.emplace_back( + std::move(rawTxn), std::move(rawMeta), ledgerIndex); + }; + auto newmarker = + accountTxPage( + onUnsavedLedger, onTransaction, options, 0, page_length, true) + .first; + return {ret, newmarker}; } std::pair> newestAccountTxPageB(AccountTxPageOptions const& options) override { - std::shared_lock lock(mutex_); - auto it = accountTxMap_.find(options.account); - if (it == accountTxMap_.end()) - return {{}, std::nullopt}; + if (!useTxTables_) + return {}; - MetaTxsList result; - std::optional marker; - const auto& accountData = it->second; - auto txIt = accountData.ledgerTxMap.lower_bound(options.minLedger); - auto txEnd = accountData.ledgerTxMap.upper_bound(options.maxLedger); - - bool lookingForMarker = options.marker.has_value(); - std::size_t count = 0; - - for (auto rIt = std::make_reverse_iterator(txEnd); - rIt != std::make_reverse_iterator(txIt) && - (options.limit == 0 || count < options.limit); - ++rIt) - { - for (auto innerRIt = rIt->second.rbegin(); - innerRIt != rIt->second.rend(); - ++innerRIt) - { - if (lookingForMarker) - { - if (rIt->first == options.marker->ledgerSeq && - innerRIt->first == options.marker->txnSeq) - lookingForMarker = false; - continue; - } - - const auto& [txn, txMeta] = - accountData.transactions[innerRIt->second]; - result.emplace_back( - txn->getSTransaction()->getSerializer().peekData(), - txMeta->getAsObject().getSerializer().peekData(), - rIt->first); - ++count; - - if (options.limit > 0 && count >= options.limit) - { - marker = AccountTxMarker{rIt->first, innerRIt->first}; - break; - } - } - } - - return {result, marker}; + static std::uint32_t const page_length(500); + auto onUnsavedLedger = + std::bind(saveLedgerAsync, std::ref(app_), std::placeholders::_1); + MetaTxsList ret; + auto onTransaction = [&ret]( + std::uint32_t ledgerIndex, + std::string const& status, + Blob&& rawTxn, + Blob&& rawMeta) { + ret.emplace_back( + std::move(rawTxn), std::move(rawMeta), ledgerIndex); + }; + auto newmarker = + accountTxPage( + onUnsavedLedger, onTransaction, options, 0, page_length, false) + .first; + return {ret, newmarker}; } }; diff --git a/src/test/app/LedgerLoad_test.cpp b/src/test/app/LedgerLoad_test.cpp index 93fc002ae..71b237e79 100644 --- a/src/test/app/LedgerLoad_test.cpp +++ b/src/test/app/LedgerLoad_test.cpp @@ -42,6 +42,9 @@ class LedgerLoad_test : public beast::unit_test::suite cfg->START_UP = type; assert(!dbPath.empty()); cfg->legacy("database_path", dbPath); + auto& sectionNode = cfg->section(ConfigSection::nodeDatabase()); + sectionNode.set("type", "memory"); + cfg->overwrite(SECTION_RELATIONAL_DB, "backend", "sqlite"); return cfg; } @@ -62,7 +65,13 @@ class LedgerLoad_test : public beast::unit_test::suite retval.ledgerFile = td.file("ledgerdata.json"); - Env env{*this}; + Env env{*this, envconfig([](std::unique_ptr cfg) { + auto& sectionNode = + cfg->section(ConfigSection::nodeDatabase()); + sectionNode.set("type", "memory"); + cfg->overwrite(SECTION_RELATIONAL_DB, "backend", "sqlite"); + return cfg; + })}; Account prev; for (auto i = 0; i < 20; ++i) diff --git a/src/test/app/LedgerReplay_test.cpp b/src/test/app/LedgerReplay_test.cpp index 7619a0f25..ab10ed5be 100644 --- a/src/test/app/LedgerReplay_test.cpp +++ b/src/test/app/LedgerReplay_test.cpp @@ -50,7 +50,13 @@ struct LedgerReplay_test : public beast::unit_test::suite auto const alice = Account("alice"); auto const bob = Account("bob"); - Env env(*this); + Env env = [&] { + auto c = jtx::envconfig(); + auto& sectionNode = c->section(ConfigSection::nodeDatabase()); + sectionNode.set("type", "memory"); + c->overwrite(SECTION_RELATIONAL_DB, "backend", "sqlite"); + return jtx::Env(*this, std::move(c)); + }(); env.fund(XRP(100000), alice, bob); env.close(); diff --git a/src/test/jtx/impl/envconfig.cpp b/src/test/jtx/impl/envconfig.cpp index 13999922a..ce2c2ae60 100644 --- a/src/test/jtx/impl/envconfig.cpp +++ b/src/test/jtx/impl/envconfig.cpp @@ -49,8 +49,9 @@ setupConfigForUnitTests(Config& cfg) cfg.FEES.account_reserve = XRP(200).value().xrp().drops(); cfg.FEES.owner_reserve = XRP(50).value().xrp().drops(); - cfg.overwrite(ConfigSection::nodeDatabase(), "type", "memory"); + cfg.overwrite(ConfigSection::nodeDatabase(), "type", "rwdb"); cfg.overwrite(ConfigSection::nodeDatabase(), "path", "main"); + cfg.overwrite(SECTION_RELATIONAL_DB, "backend", "rwdb"); cfg.deprecatedClearSection(ConfigSection::importNodeDatabase()); cfg.legacy("database_path", ""); cfg.setupControl(true, true, true); diff --git a/src/test/nodestore/Database_test.cpp b/src/test/nodestore/Database_test.cpp index 80c0808bb..fabd01cc8 100644 --- a/src/test/nodestore/Database_test.cpp +++ b/src/test/nodestore/Database_test.cpp @@ -564,7 +564,7 @@ public: BEAST_EXPECT(areBatchesEqual(batch, copy)); } - if (type == "memory") + if (type == "memory" || type == "rwdb") { // Verify default earliest ledger sequence { diff --git a/src/test/rpc/GetCounts_test.cpp b/src/test/rpc/GetCounts_test.cpp index 52b645ed7..82a8ad628 100644 --- a/src/test/rpc/GetCounts_test.cpp +++ b/src/test/rpc/GetCounts_test.cpp @@ -47,9 +47,6 @@ class GetCounts_test : public beast::unit_test::suite BEAST_EXPECT( result.isMember(jss::uptime) && !result[jss::uptime].asString().empty()); - BEAST_EXPECT( - result.isMember(jss::dbKBTotal) && - result[jss::dbKBTotal].asInt() > 0); } // create some transactions diff --git a/src/test/rpc/NodeToShardRPC_test.cpp b/src/test/rpc/NodeToShardRPC_test.cpp index 867f2cfe3..dfa8e3152 100644 --- a/src/test/rpc/NodeToShardRPC_test.cpp +++ b/src/test/rpc/NodeToShardRPC_test.cpp @@ -63,9 +63,11 @@ public: jtx::Env env = [&] { auto c = jtx::envconfig(); auto& sectionNode = c->section(ConfigSection::nodeDatabase()); + sectionNode.set("type", "memory"); sectionNode.set("earliest_seq", "257"); sectionNode.set("ledgers_per_shard", "256"); c->setupControl(true, true, true); + c->overwrite(SECTION_RELATIONAL_DB, "backend", "sqlite"); return jtx::Env(*this, std::move(c)); }(); @@ -138,9 +140,11 @@ public: section.set("ledgers_per_shard", "256"); section.set("earliest_seq", "257"); auto& sectionNode = c->section(ConfigSection::nodeDatabase()); + sectionNode.set("type", "memory"); sectionNode.set("earliest_seq", "257"); sectionNode.set("ledgers_per_shard", "256"); c->setupControl(true, true, true); + c->overwrite(SECTION_RELATIONAL_DB, "backend", "sqlite"); return jtx::Env(*this, std::move(c)); }(); @@ -282,9 +286,11 @@ public: section.set("ledgers_per_shard", "256"); section.set("earliest_seq", "257"); auto& sectionNode = c->section(ConfigSection::nodeDatabase()); + sectionNode.set("type", "memory"); sectionNode.set("earliest_seq", "257"); sectionNode.set("ledgers_per_shard", "256"); c->setupControl(true, true, true); + c->overwrite(SECTION_RELATIONAL_DB, "backend", "sqlite"); return jtx::Env( *this, std::move(c), nullptr, beast::severities::kDisabled); diff --git a/src/test/shamap/common.h b/src/test/shamap/common.h index d89acb988..591e95803 100644 --- a/src/test/shamap/common.h +++ b/src/test/shamap/common.h @@ -57,7 +57,7 @@ public: , j_(j) { Section testSection; - testSection.set("type", "memory"); + testSection.set("type", "rwdb"); testSection.set("path", "SHAMap_test"); db_ = NodeStore::Manager::instance().make_Database( megabytes(4), scheduler_, 1, testSection, j); From 57a1329bff462a6b1baf5ba0f9d49ce8d943dfa8 Mon Sep 17 00:00:00 2001 From: tequ Date: Fri, 15 Nov 2024 07:33:55 +0900 Subject: [PATCH 11/11] Fix lexicographical_compare_three_way build error at macos (#391) Co-authored-by: Denis Angell --- src/ripple/basics/base_uint.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/ripple/basics/base_uint.h b/src/ripple/basics/base_uint.h index f0e7acf89..93c5df8d6 100644 --- a/src/ripple/basics/base_uint.h +++ b/src/ripple/basics/base_uint.h @@ -549,8 +549,8 @@ using uint128 = base_uint<128>; using uint160 = base_uint<160>; using uint256 = base_uint<256>; -/* - * template +#ifdef __APPLE__ +template [[nodiscard]] inline constexpr std::strong_ordering operator<=>(base_uint const& lhs, base_uint const& rhs) { @@ -561,7 +561,6 @@ operator<=>(base_uint const& lhs, base_uint const& rhs) // // FIXME: use std::lexicographical_compare_three_way once support is // added to MacOS. - auto const ret = std::mismatch(lhs.cbegin(), lhs.cend(), rhs.cbegin()); // a == b @@ -571,8 +570,7 @@ operator<=>(base_uint const& lhs, base_uint const& rhs) return (*ret.first > *ret.second) ? std::strong_ordering::greater : std::strong_ordering::less; } -*/ - +#else template [[nodiscard]] inline constexpr std::strong_ordering operator<=>(base_uint const& lhs, base_uint const& rhs) @@ -584,6 +582,7 @@ operator<=>(base_uint const& lhs, base_uint const& rhs) rhs.cend(), std::compare_three_way{}); } +#endif template [[nodiscard]] inline constexpr bool