diff --git a/Builds/levelization/results/ordering.txt b/Builds/levelization/results/ordering.txt index 27ff477d8..9c5bf4b36 100644 --- a/Builds/levelization/results/ordering.txt +++ b/Builds/levelization/results/ordering.txt @@ -85,6 +85,7 @@ test.nodestore > xrpl.basics test.nodestore > xrpld.core test.nodestore > xrpld.nodestore test.nodestore > xrpld.unity +test.nodestore > xrpl.protocol test.overlay > test.jtx test.overlay > test.toplevel test.overlay > test.unit_test @@ -119,6 +120,7 @@ test.rpc > xrpld.core test.rpc > xrpld.net test.rpc > xrpld.overlay test.rpc > xrpld.rpc +test.rpc > xrpld.shamap test.rpc > xrpl.hook test.rpc > xrpl.json test.rpc > xrpl.protocol diff --git a/build-full.sh b/build-full.sh index 29b953ac3..c0e79b93b 100644 --- a/build-full.sh +++ b/build-full.sh @@ -12,17 +12,16 @@ echo "-- GITHUB_REPOSITORY: $1" echo "-- GITHUB_SHA: $2" echo "-- GITHUB_RUN_NUMBER: $4" -umask 0000; +umask 0000 #### -cd /io; -mkdir -p src/certs; -curl --silent -k https://raw.githubusercontent.com/RichardAH/rippled-release-builder/main/ca-bundle/certbundle.h -o src/certs/certbundle.h; -if [ "`grep certbundle.h src/xrpld/net/detail/RegisterSSLCerts.cpp | wc -l`" -eq "0" ] -then - cp src/xrpld/net/detail/RegisterSSLCerts.cpp src/xrpld/net/detail/RegisterSSLCerts.cpp.old - perl -i -pe "s/^{/{ +cd /io +mkdir -p src/certs +curl --silent -k https://raw.githubusercontent.com/RichardAH/rippled-release-builder/main/ca-bundle/certbundle.h -o src/certs/certbundle.h +if [ "$(grep certbundle.h src/xrpld/net/detail/RegisterSSLCerts.cpp | wc -l)" -eq "0" ]; then + cp src/xrpld/net/detail/RegisterSSLCerts.cpp src/xrpld/net/detail/RegisterSSLCerts.cpp.old + perl -i -pe "s/^{/{ #ifdef EMBEDDED_CA_BUNDLE BIO *cbio = BIO_new_mem_buf(ca_bundle.data(), ca_bundle.size()); X509_STORE *cts = SSL_CTX_get_cert_store(ctx.native_handle()); @@ -68,15 +67,14 @@ fi source /opt/rh/gcc-toolset-11/enable export PATH=/usr/local/bin:$PATH export CC='/usr/lib64/ccache/gcc' && -export CXX='/usr/lib64/ccache/g++' && -echo "-- Build Rippled --" && -pwd && + export CXX='/usr/lib64/ccache/g++' && + echo "-- Build Rippled --" && + pwd && + echo "MOVING TO [ build-core.sh ]" -echo "MOVING TO [ build-core.sh ]"; - -printenv > .env.temp; -cat .env.temp | grep '=' | sed s/\\\(^[^=]\\+=\\\)/\\1\\\"/g|sed s/\$/\\\"/g > .env; -rm .env.temp; +printenv >.env.temp +cat .env.temp | grep '=' | sed s/\\\(^[^=]\\+=\\\)/\\1\\\"/g | sed s/\$/\\\"/g >.env +rm .env.temp echo "Persisting ENV:" cat .env diff --git a/include/xrpl/protocol/Feature.h b/include/xrpl/protocol/Feature.h index 46e4c3556..617ae4709 100644 --- a/include/xrpl/protocol/Feature.h +++ b/include/xrpl/protocol/Feature.h @@ -33,35 +33,39 @@ * * Steps required to add new features to the code: * - * 1) In this file, increment `numFeatures` and add a uint256 declaration - * for the feature at the bottom - * 2) Add a uint256 definition for the feature to the corresponding source - * file (Feature.cpp). Use `registerFeature` to create the feature with - * the feature's name, `Supported::no`, and `VoteBehavior::DefaultNo`. This - * should be the only place the feature's name appears in code as a string. - * 3) Use the uint256 as the parameter to `view.rules.enabled()` to - * control flow into new code that this feature limits. - * 4) If the feature development is COMPLETE, and the feature is ready to be - * SUPPORTED, change the `registerFeature` parameter to Supported::yes. - * 5) When the feature is ready to be ENABLED, change the `registerFeature` - * parameter to `VoteBehavior::DefaultYes`. - * In general, any newly supported amendments (`Supported::yes`) should have - * a `VoteBehavior::DefaultNo` for at least one full release cycle. High - * priority bug fixes can be an exception to this rule of thumb. + * 1) Add the appropriate XRPL_FEATURE or XRPL_FIX macro definition for the + * feature to features.macro with the feature's name, `Supported::no`, and + * `VoteBehavior::DefaultNo`. + * + * 2) Use the generated variable name as the parameter to `view.rules.enabled()` + * to control flow into new code that this feature limits. (featureName or + * fixName) + * + * 3) If the feature development is COMPLETE, and the feature is ready to be + * SUPPORTED, change the macro parameter in features.macro to Supported::yes. + * + * 4) In general, any newly supported amendments (`Supported::yes`) should have + * a `VoteBehavior::DefaultNo` indefinitely so that external governance can + * make the decision on when to activate it. High priority bug fixes can be + * an exception to this rule. In such cases, ensure the fix has been + * clearly communicated to the community using appropriate channels, + * then change the macro parameter in features.macro to + * `VoteBehavior::DefaultYes`. The communication process is beyond + * the scope of these instructions. + * * * When a feature has been enabled for several years, the conditional code * may be removed, and the feature "retired". To retire a feature: - * 1) Remove the uint256 declaration from this file. - * 2) MOVE the uint256 definition in Feature.cpp to the "retired features" - * section at the end of the file. - * 3) CHANGE the name of the variable to start with "retired". - * 4) CHANGE the parameters of the `registerFeature` call to `Supported::yes` - * and `VoteBehavior::DefaultNo`. + * + * 1) MOVE the macro definition in features.macro to the "retired features" + * section at the end of the file, and change the macro to XRPL_RETIRE. + * * The feature must remain registered and supported indefinitely because it - * still exists in the ledger, but there is no need to vote for it because - * there's nothing to vote for. If it is removed completely from the code, any - * instances running that code will get amendment blocked. Removing the - * feature from the ledger is beyond the scope of these instructions. + * may exist in the Amendments object on ledger. There is no need to vote + * for it because there's nothing to vote for. If the feature definition is + * removed completely from the code, any instances running that code will get + * amendment blocked. Removing the feature from the ledger is beyond the scope + * of these instructions. * */ @@ -76,11 +80,32 @@ allAmendments(); namespace detail { +#pragma push_macro("XRPL_FEATURE") +#undef XRPL_FEATURE +#pragma push_macro("XRPL_FIX") +#undef XRPL_FIX +#pragma push_macro("XRPL_RETIRE") +#undef XRPL_RETIRE + +#define XRPL_FEATURE(name, supported, vote) +1 +#define XRPL_FIX(name, supported, vote) +1 +#define XRPL_RETIRE(name) +1 + // This value SHOULD be equal to the number of amendments registered in // Feature.cpp. Because it's only used to reserve storage, and determine how // large to make the FeatureBitset, it MAY be larger. It MUST NOT be less than // the actual number of amendments. A LogicError on startup will verify this. -static constexpr std::size_t numFeatures = 118; +static constexpr std::size_t numFeatures = + (0 + +#include + ); + +#undef XRPL_RETIRE +#pragma pop_macro("XRPL_RETIRE") +#undef XRPL_FIX +#pragma pop_macro("XRPL_FIX") +#undef XRPL_FEATURE +#pragma pop_macro("XRPL_FEATURE") /** Amendments that this server supports and the default voting behavior. Whether they are enabled depends on the Rules defined in the validated @@ -320,12 +345,17 @@ foreachFeature(FeatureBitset bs, F&& f) #undef XRPL_FEATURE #pragma push_macro("XRPL_FIX") #undef XRPL_FIX +#pragma push_macro("XRPL_RETIRE") +#undef XRPL_RETIRE #define XRPL_FEATURE(name, supported, vote) extern uint256 const feature##name; #define XRPL_FIX(name, supported, vote) extern uint256 const fix##name; +#define XRPL_RETIRE(name) #include +#undef XRPL_RETIRE +#pragma pop_macro("XRPL_RETIRE") #undef XRPL_FIX #pragma pop_macro("XRPL_FIX") #undef XRPL_FEATURE diff --git a/include/xrpl/protocol/detail/features.macro b/include/xrpl/protocol/detail/features.macro index 1bae5a18b..1f447c96a 100644 --- a/include/xrpl/protocol/detail/features.macro +++ b/include/xrpl/protocol/detail/features.macro @@ -23,6 +23,9 @@ #if !defined(XRPL_FIX) #error "undefined macro: XRPL_FIX" #endif +#if !defined(XRPL_RETIRE) +#error "undefined macro: XRPL_RETIRE" +#endif // clang-format off @@ -57,18 +60,18 @@ XRPL_FIX (DisallowIncomingV1, Supported::yes, VoteBehavior::DefaultYe XRPL_FEATURE(XChainBridge, Supported::no, VoteBehavior::DefaultNo) XRPL_FEATURE(AMM, Supported::yes, VoteBehavior::DefaultNo) XRPL_FIX (ReducedOffersV1, Supported::yes, VoteBehavior::DefaultYes) -XRPL_FEATURE(HooksUpdate2, Supported::yes, VoteBehavior::DefaultNo); -XRPL_FEATURE(HookOnV2, Supported::yes, VoteBehavior::DefaultNo); -XRPL_FEATURE(Export, Supported::yes, VoteBehavior::DefaultNo); -XRPL_FEATURE(ConsensusEntropy, Supported::yes, VoteBehavior::DefaultNo); -XRPL_FIX (HookAPI20251128, Supported::yes, VoteBehavior::DefaultYes); -XRPL_FIX (CronStacking, Supported::yes, VoteBehavior::DefaultYes); -XRPL_FEATURE(ExtendedHookState, Supported::yes, VoteBehavior::DefaultNo); -XRPL_FIX (InvalidTxFlags, Supported::yes, VoteBehavior::DefaultYes); -XRPL_FEATURE(Cron, Supported::yes, VoteBehavior::DefaultNo); -XRPL_FEATURE(IOUIssuerWeakTSH, Supported::yes, VoteBehavior::DefaultNo); -XRPL_FEATURE(DeepFreeze, Supported::yes, VoteBehavior::DefaultNo); -XRPL_FIX (ProvisionalDoubleThreading, Supported::yes, VoteBehavior::DefaultYes); +XRPL_FEATURE(HooksUpdate2, Supported::yes, VoteBehavior::DefaultNo) +XRPL_FEATURE(HookOnV2, Supported::yes, VoteBehavior::DefaultNo) +XRPL_FEATURE(Export, Supported::yes, VoteBehavior::DefaultNo) +XRPL_FEATURE(ConsensusEntropy, Supported::yes, VoteBehavior::DefaultNo) +XRPL_FIX (HookAPI20251128, Supported::yes, VoteBehavior::DefaultYes) +XRPL_FIX (CronStacking, Supported::yes, VoteBehavior::DefaultYes) +XRPL_FEATURE(ExtendedHookState, Supported::yes, VoteBehavior::DefaultNo) +XRPL_FIX (InvalidTxFlags, Supported::yes, VoteBehavior::DefaultYes) +XRPL_FEATURE(Cron, Supported::yes, VoteBehavior::DefaultNo) +XRPL_FEATURE(IOUIssuerWeakTSH, Supported::yes, VoteBehavior::DefaultNo) +XRPL_FEATURE(DeepFreeze, Supported::yes, VoteBehavior::DefaultNo) +XRPL_FIX (ProvisionalDoubleThreading, Supported::yes, VoteBehavior::DefaultYes) XRPL_FEATURE(Clawback, Supported::yes, VoteBehavior::DefaultNo) XRPL_FIX (RewardClaimFlags, Supported::yes, VoteBehavior::DefaultYes) XRPL_FEATURE(HookCanEmit, Supported::yes, VoteBehavior::DefaultNo) @@ -152,4 +155,24 @@ XRPL_FIX (NFTokenDirV1, Supported::yes, VoteBehavior::Obsolete) XRPL_FEATURE(NonFungibleTokensV1, Supported::yes, VoteBehavior::Obsolete) XRPL_FEATURE(CryptoConditionsSuite, Supported::yes, VoteBehavior::Obsolete) +// The following amendments have been active for at least two years. Their +// pre-amendment code has been removed and the identifiers are deprecated. +// All known amendments and amendments that may appear in a validated +// ledger must be registered either here or above with the "active" amendments +XRPL_RETIRE(MultiSign) +XRPL_RETIRE(TrustSetAuth) +XRPL_RETIRE(FeeEscalation) +XRPL_RETIRE(PayChan) +XRPL_RETIRE(CryptoConditions) +XRPL_RETIRE(TickSize) +XRPL_RETIRE(fix1368) +XRPL_RETIRE(Escrow) +XRPL_RETIRE(fix1373) +XRPL_RETIRE(EnforceInvariants) +XRPL_RETIRE(SortedDirectories) +XRPL_RETIRE(fix1201) +XRPL_RETIRE(fix1512) +XRPL_RETIRE(fix1523) +XRPL_RETIRE(fix1528) + // clang-format on diff --git a/release-builder.sh b/release-builder.sh index ac9c46981..712da3790 100755 --- a/release-builder.sh +++ b/release-builder.sh @@ -11,11 +11,11 @@ echo "START BUILDING (HOST)" echo "Cleaning previously built binary" rm -f release-build/xahaud -BUILD_CORES=$(echo "scale=0 ; `nproc` / 1.337" | bc) +BUILD_CORES=$(echo "scale=0 ; $(nproc) / 1.337" | bc) if [[ "$GITHUB_REPOSITORY" == "" ]]; then #Default - BUILD_CORES=${BUILD_CORES:-8} + BUILD_CORES=${BUILD_CORES:-8} fi # Ensure still works outside of GH Actions by setting these to /dev/null @@ -31,21 +31,19 @@ echo "-- GITHUB_SHA: $GITHUB_SHA" echo "-- GITHUB_RUN_NUMBER: $GITHUB_RUN_NUMBER" echo "-- CONTAINER_NAME: $CONTAINER_NAME" -which docker 2> /dev/null 2> /dev/null -if [ "$?" -eq "1" ] -then +which docker 2>/dev/null 2>/dev/null +if [ "$?" -eq "1" ]; then echo 'Docker not found. Install it first.' exit 1 fi -stat .git 2> /dev/null 2> /dev/null -if [ "$?" -eq "1" ] -then +stat .git 2>/dev/null 2>/dev/null +if [ "$?" -eq "1" ]; then echo 'Run this inside the source directory. (.git dir not found).' exit 1 fi -STATIC_CONTAINER=$(docker ps -a | grep $CONTAINER_NAME |wc -l) +STATIC_CONTAINER=$(docker ps -a | grep $CONTAINER_NAME | wc -l) CACHE_VOLUME_NAME="xahau-release-builder-cache" @@ -57,13 +55,14 @@ if false; then docker stop $CONTAINER_NAME else echo "No static container, build on temp container" - rm -rf release-build; - mkdir -p release-build; + rm -rf release-build + mkdir -p release-build docker volume create $CACHE_VOLUME_NAME # Create inline Dockerfile with environment setup for build-full.sh - DOCKERFILE_CONTENT=$(cat <<'DOCKERFILE_EOF' + DOCKERFILE_CONTENT=$( + cat <<'DOCKERFILE_EOF' FROM ghcr.io/phusion/holy-build-box:4.0.1-amd64 ARG BUILD_CORES=8 @@ -218,7 +217,7 @@ RUN /hbb_exe/activate-exec bash -c "ccache -M 100G && \ ln -s ../../bin/ccache /usr/lib64/ccache/c++" DOCKERFILE_EOF -) + ) # Build custom Docker image IMAGE_NAME="xahaud-builder:latest" @@ -228,14 +227,14 @@ DOCKERFILE_EOF if [[ "$GITHUB_REPOSITORY" == "" ]]; then # Non GH, local building echo "Non-GH runner, local building, temp container" - docker run -i --user 0:$(id -g) --rm -v /data/builds:/data/builds -v `pwd`:/io -v "$CACHE_VOLUME_NAME":/cache --network host "$IMAGE_NAME" /hbb_exe/activate-exec bash -c "source /opt/rh/gcc-toolset-11/enable && bash -x /io/build-full.sh '$GITHUB_REPOSITORY' '$GITHUB_SHA' '$BUILD_CORES' '$GITHUB_RUN_NUMBER'" + docker run -i --user 0:$(id -g) --rm -v /data/builds:/data/builds -v $(pwd):/io -v "$CACHE_VOLUME_NAME":/cache --network host "$IMAGE_NAME" /hbb_exe/activate-exec bash -c "source /opt/rh/gcc-toolset-11/enable && bash -x /io/build-full.sh '$GITHUB_REPOSITORY' '$GITHUB_SHA' '$BUILD_CORES' '$GITHUB_RUN_NUMBER'" else # GH Action, runner echo "GH Action, runner, clean & re-create create persistent container" docker rm -f $CONTAINER_NAME - echo "echo 'Stopping container: $CONTAINER_NAME'" >> "$JOB_CLEANUP_SCRIPT" - echo "docker stop --time=15 \"$CONTAINER_NAME\" || echo 'Failed to stop container or container not running'" >> "$JOB_CLEANUP_SCRIPT" - docker run -di --user 0:$(id -g) --name $CONTAINER_NAME -v /data/builds:/data/builds -v `pwd`:/io -v "$CACHE_VOLUME_NAME":/cache --network host "$IMAGE_NAME" /hbb_exe/activate-exec bash + echo "echo 'Stopping container: $CONTAINER_NAME'" >>"$JOB_CLEANUP_SCRIPT" + echo "docker stop --time=15 \"$CONTAINER_NAME\" || echo 'Failed to stop container or container not running'" >>"$JOB_CLEANUP_SCRIPT" + docker run -di --user 0:$(id -g) --name $CONTAINER_NAME -v /data/builds:/data/builds -v $(pwd):/io -v "$CACHE_VOLUME_NAME":/cache --network host "$IMAGE_NAME" /hbb_exe/activate-exec bash docker exec -i $CONTAINER_NAME /hbb_exe/activate-exec bash -c "source /opt/rh/gcc-toolset-11/enable && bash -x /io/build-full.sh '$GITHUB_REPOSITORY' '$GITHUB_SHA' '$BUILD_CORES' '$GITHUB_RUN_NUMBER'" docker stop $CONTAINER_NAME fi diff --git a/src/libxrpl/protocol/AMMCore.cpp b/src/libxrpl/protocol/AMMCore.cpp index 3bebfc465..1ea83afc2 100644 --- a/src/libxrpl/protocol/AMMCore.cpp +++ b/src/libxrpl/protocol/AMMCore.cpp @@ -66,7 +66,7 @@ invalidAMMAsset( Issue const& issue, std::optional> const& pair) { - if (badCurrency() == issue.currency) + if (isBadCurrency(issue.currency)) return temBAD_CURRENCY; if (isXRP(issue) && issue.account.isNonZero()) return temBAD_ISSUER; diff --git a/src/libxrpl/protocol/Feature.cpp b/src/libxrpl/protocol/Feature.cpp index 05164489e..52e1d7217 100644 --- a/src/libxrpl/protocol/Feature.cpp +++ b/src/libxrpl/protocol/Feature.cpp @@ -250,12 +250,9 @@ FeatureCollections::registerFeature( Feature const* i = getByName(name); if (!i) { - // If this check fails, and you just added a feature, increase the - // numFeatures value in Feature.h check( features.size() < detail::numFeatures, - "More features defined than allocated. Adjust numFeatures in " - "Feature.h."); + "More features defined than allocated."); auto const f = sha512Half(Slice(name.data(), name.size())); @@ -424,45 +421,26 @@ featureToName(uint256 const& f) #undef XRPL_FEATURE #pragma push_macro("XRPL_FIX") #undef XRPL_FIX +#pragma push_macro("XRPL_RETIRE") +#undef XRPL_RETIRE #define XRPL_FEATURE(name, supported, vote) \ uint256 const feature##name = registerFeature(#name, supported, vote); #define XRPL_FIX(name, supported, vote) \ uint256 const fix##name = registerFeature("fix" #name, supported, vote); +#define XRPL_RETIRE(name) \ + [[deprecated("The referenced amendment has been retired"), maybe_unused]] \ + uint256 const retired##name = retireFeature(#name); #include +#undef XRPL_RETIRE +#pragma pop_macro("XRPL_RETIRE") #undef XRPL_FIX #pragma pop_macro("XRPL_FIX") #undef XRPL_FEATURE #pragma pop_macro("XRPL_FEATURE") -// clang-format off - -// The following amendments have been active for at least two years. Their -// pre-amendment code has been removed and the identifiers are deprecated. -// All known amendments and amendments that may appear in a validated -// ledger must be registered either here or above with the "active" amendments -[[deprecated("The referenced amendment has been retired"), maybe_unused]] -uint256 const - retiredMultiSign = retireFeature("MultiSign"), - retiredTrustSetAuth = retireFeature("TrustSetAuth"), - retiredFeeEscalation = retireFeature("FeeEscalation"), - retiredPayChan = retireFeature("PayChan"), - retiredCryptoConditions = retireFeature("CryptoConditions"), - retiredTickSize = retireFeature("TickSize"), - retiredFix1368 = retireFeature("fix1368"), - retiredEscrow = retireFeature("Escrow"), - retiredFix1373 = retireFeature("fix1373"), - retiredEnforceInvariants = retireFeature("EnforceInvariants"), - retiredSortedDirectories = retireFeature("SortedDirectories"), - retiredFix1201 = retireFeature("fix1201"), - retiredFix1512 = retireFeature("fix1512"), - retiredFix1523 = retireFeature("fix1523"), - retiredFix1528 = retireFeature("fix1528"); - -// clang-format on - // All of the features should now be registered, since variables in a cpp file // are initialized from top to bottom. // diff --git a/src/libxrpl/protocol/Issue.cpp b/src/libxrpl/protocol/Issue.cpp index 179cb1eb1..a42baf8ad 100644 --- a/src/libxrpl/protocol/Issue.cpp +++ b/src/libxrpl/protocol/Issue.cpp @@ -111,7 +111,7 @@ issueFromJson(Json::Value const& v) } auto const currency = to_currency(curStr.asString()); - if (currency == badCurrency() || currency == noCurrency()) + if (isBadCurrency(currency) || currency == noCurrency()) { Throw("issueFromJson currency must be a valid currency"); } diff --git a/src/libxrpl/protocol/STCurrency.cpp b/src/libxrpl/protocol/STCurrency.cpp index 56ac19da1..55726fef5 100644 --- a/src/libxrpl/protocol/STCurrency.cpp +++ b/src/libxrpl/protocol/STCurrency.cpp @@ -103,7 +103,7 @@ currencyFromJson(SField const& name, Json::Value const& v) } auto const currency = to_currency(v.asString()); - if (currency == badCurrency() || currency == noCurrency()) + if (isBadCurrency(currency) || currency == noCurrency()) { Throw( "currencyFromJson currency must be a valid currency"); diff --git a/src/test/app/LedgerMaster_test.cpp b/src/test/app/LedgerMaster_test.cpp index 8a1f90857..eaa2100c0 100644 --- a/src/test/app/LedgerMaster_test.cpp +++ b/src/test/app/LedgerMaster_test.cpp @@ -20,6 +20,8 @@ #include #include #include +#include +#include #include namespace ripple { @@ -117,13 +119,224 @@ class LedgerMaster_test : public beast::unit_test::suite } } + void + testCanSkipPinnedGap() + { + // detail::canSkipPinnedGap is the lifted decision used by + // findNewLedgersToPublish to decide whether pubSeq can leap + // across a gap in the publish-target set without violating + // contiguous publication of non-pinned ledgers. In production + // this guard rarely fires (pinned ranges are old historical + // catalogue data, pubSeq is at the live tip) — it matters in + // test/standalone catalogue-load scenarios where pinned seqs + // can sit near pubSeq. The invariant is small and crisp; + // unit-testing it directly is far cheaper than reconstructing + // a believable findNewLedgersToPublish state through Env. + testcase("detail::canSkipPinnedGap"); + + using detail::canSkipPinnedGap; + + // No gap (pubSeq is already at intervalStart) — always OK. + { + RangeSet empty; + BEAST_EXPECT(canSkipPinnedGap(10, 10, empty)); + BEAST_EXPECT(canSkipPinnedGap(10, 5, empty)); // past + } + + // One-element gap, exactly pinned: can skip. + { + RangeSet pinned; + pinned.insert(range(10u, 10u)); + BEAST_EXPECT(canSkipPinnedGap(10, 11, pinned)); + } + + // One-element gap, NOT pinned: must halt. + { + RangeSet empty; + BEAST_EXPECT(!canSkipPinnedGap(10, 11, empty)); + } + + // Multi-element gap entirely covered by pinned: can skip. + { + RangeSet pinned; + pinned.insert(range(11u, 15u)); + BEAST_EXPECT(canSkipPinnedGap(11, 16, pinned)); + } + + // Multi-element gap, pinned covers most but leaves one + // non-pinned seq exposed: must halt. This is the bug the + // guard exists to prevent — silently publishing seq 16 + // while seq 15 (non-pinned) is unfetched. + { + RangeSet pinned; + pinned.insert(range(11u, 14u)); // missing 15 + BEAST_EXPECT(!canSkipPinnedGap(11, 16, pinned)); + } + + // Pinned set extends beyond the gap — only the gap window + // matters for the decision. + { + RangeSet pinned; + pinned.insert(range(0u, 1000u)); + BEAST_EXPECT(canSkipPinnedGap(50, 100, pinned)); + } + + // Empty pinned, with a gap: must halt. + { + RangeSet empty; + BEAST_EXPECT(!canSkipPinnedGap(50, 100, empty)); + } + + // Disjoint pinned that doesn't intersect the gap: must halt. + { + RangeSet pinned; + pinned.insert(range(200u, 300u)); + BEAST_EXPECT(!canSkipPinnedGap(50, 100, pinned)); + } + + // Pinned partially intersects gap from the left only. + { + RangeSet pinned; + pinned.insert(range(50u, 75u)); + BEAST_EXPECT(!canSkipPinnedGap(50, 100, pinned)); + } + + // Pinned partially intersects gap from the right only. + { + RangeSet pinned; + pinned.insert(range(75u, 99u)); + BEAST_EXPECT(!canSkipPinnedGap(50, 100, pinned)); + } + } + + void + testPinUnpinSymmetry() + { + testcase("pin/unpin symmetry"); + + using namespace test::jtx; + Env env{*this, makeNetworkConfig(11111)}; + + auto const alice = Account("alice"); + env.fund(XRP(1000), alice); + env.close(); + + auto& lm = env.app().getLedgerMaster(); + + // The most recently closed ledger is the one we'll pin. + auto const ledger = lm.getClosedLedger(); + BEAST_EXPECT(ledger); + if (!ledger) + return; + auto const seq = ledger->info().seq; + BEAST_EXPECT(seq > 0); + + // Baseline — not pinned. + BEAST_EXPECT(!lm.isPinned(seq)); + + // Pin via storeLedger(ledger, pin=true). This is the same path + // catalogue_load uses just before saveValidatedLedger, which is why + // a save failure must roll back via unpinLedger() to keep the + // in-memory pinned-set consistent with state.db. + lm.storeLedger(ledger, /*pin=*/true); + BEAST_EXPECT(lm.isPinned(seq)); + BEAST_EXPECT(boost::icl::contains(lm.getPinnedLedgersRangeSet(), seq)); + + // Unpin reverses the pin (the rollback contract used by + // catalogue_load's RAII guard on save failure). + lm.unpinLedger(seq); + BEAST_EXPECT(!lm.isPinned(seq)); + BEAST_EXPECT(!boost::icl::contains(lm.getPinnedLedgersRangeSet(), seq)); + + // Unpinning a seq that isn't pinned is a no-op (idempotent). + lm.unpinLedger(seq); + BEAST_EXPECT(!lm.isPinned(seq)); + } + + void + testSetPinnedRangesImmediateMerge() + { + // Regression test: when setPinnedLedgersRangeSet runs and + // mCompleteLedgers is already non-empty (LOAD/standalone startup + // already called setFullLedger), the deferred merge in + // setFullLedger never fires because the one-shot guard relies on + // a flag, not on emptiness. Without the immediate-merge branch + // in setPinnedLedgersRangeSet, pinned ranges restored from + // state.db would never appear in mCompleteLedgers and historical + // pinned ledgers would not be queryable until the next + // validation. This test asserts the immediate-merge behavior. + testcase("setPinnedLedgersRangeSet immediate merge"); + + using namespace test::jtx; + Env env{*this, makeNetworkConfig(11111)}; + + auto const alice = Account("alice"); + env.fund(XRP(1000), alice); + env.close(); + + auto& lm = env.app().getLedgerMaster(); + + // Why mCompleteLedgers is non-empty here: + // env.close() above runs synchronous standalone consensus. + // That hits LedgerMaster::switchLCL, which (in standalone) + // calls setFullLedger directly on the calling thread, which + // inserts the closed ledger's seq into mCompleteLedgers. + // So by the time we get here, mCompleteLedgers has the + // genesis + funded seqs in it. + // + // This is the precondition that triggers the immediate-merge + // branch in setPinnedLedgersRangeSet (the LOAD/standalone + // startup case). On a NORMAL/NETWORK production startup, + // mCompleteLedgers would still be empty when state.db is read + // and setPinnedLedgersRangeSet is called, and the merge would + // be deferred to the first validated ledger via setFullLedger. + // We're specifically testing the *other* branch here. + auto const completeBefore = lm.getCompleteLedgersRangeSet(); + BEAST_EXPECT(!completeBefore.empty()); + + // Pinned ranges are far outside the existing range so we can + // unambiguously detect them being merged in. + std::uint32_t const pinLo = 100000; + std::uint32_t const pinHi = 100099; + RangeSet pinned; + pinned.insert(range(pinLo, pinHi)); + + // Restore pinned ranges (mirrors what SHAMapStoreImp::start does + // after reading state.db's PinnedLedgers row). + lm.setPinnedLedgersRangeSet(pinned); + + // Pinned set is restored. + BEAST_EXPECT(lm.isPinned(pinLo)); + BEAST_EXPECT(lm.isPinned(pinHi)); + BEAST_EXPECT(lm.isPinned((pinLo + pinHi) / 2)); + + // Critical invariant: pinned ranges are now in mCompleteLedgers + // immediately, not deferred to the first validated ledger. + // This is what makes haveLedger() return true for pinned seqs + // after a LOAD/standalone restart, without which the historical + // pinned data is on disk but the system reports "we don't have + // it". + BEAST_EXPECT(lm.haveLedger(pinLo)); + BEAST_EXPECT(lm.haveLedger(pinHi)); + BEAST_EXPECT(lm.haveLedger((pinLo + pinHi) / 2)); + + // The pre-existing complete range is still complete (merge, + // not replace). + for (auto const& interval : completeBefore) + for (auto s = interval.lower(); s <= interval.upper(); ++s) + BEAST_EXPECT(lm.haveLedger(s)); + } + public: void run() override { using namespace test::jtx; FeatureBitset const all{supported_amendments() - featureXahauGenesis}; + testCanSkipPinnedGap(); testWithFeats(all); + testPinUnpinSymmetry(); + testSetPinnedRangesImmediateMerge(); } void diff --git a/src/test/app/SHAMapStore_test.cpp b/src/test/app/SHAMapStore_test.cpp index 134f72a00..2ea3d2412 100644 --- a/src/test/app/SHAMapStore_test.cpp +++ b/src/test/app/SHAMapStore_test.cpp @@ -22,10 +22,14 @@ #include #include #include +#include +#include #include #include #include +#include #include +#include namespace ripple { namespace test { @@ -51,6 +55,25 @@ class SHAMapStore_test : public beast::unit_test::suite return cfg; } + static auto + pinnedPersistent( + std::unique_ptr cfg, + std::string const& dbPath, + std::string const& nodePath, + std::string const& pinnedPath) + { + cfg = onlineDelete(std::move(cfg)); + cfg->legacy("database_path", dbPath); + cfg->overwrite(SECTION_RELATIONAL_DB, "backend", "sqlite"); + + auto& section = cfg->section(ConfigSection::nodeDatabase()); + section.set("type", "rwdb"); + section.set("path", nodePath); + section.set("pinned_type", "nudb"); + section.set("pinned_path", pinnedPath); + return cfg; + } + bool goodLedger( jtx::Env& env, @@ -648,13 +671,191 @@ public: BEAST_EXPECT(dbr->getName() == "3"); } + void + testPinnedRangeRestoreRequiresPinnedData() + { + testcase("pinned range restore requires pinned data"); + + using namespace jtx; + + beast::temp_dir tempDir; + auto const tempPath = boost::filesystem::path(tempDir.path()); + auto const dbPath = (tempPath / "db").string(); + auto const nodePath = (tempPath / "node").string(); + auto const pinnedPath = (tempPath / "pinned").string(); + + boost::filesystem::create_directories(dbPath); + boost::filesystem::create_directories(nodePath); + boost::filesystem::create_directories(pinnedPath); + + { + Env env( + *this, + envconfig(pinnedPersistent, dbPath, nodePath, pinnedPath)); + + RangeSet ranges; + ranges.insert(range(100u, 200u)); + env.app().getSHAMapStore().setPinnedRanges(ranges); + + NodeStoreScheduler scheduler(env.app().getJobQueue()); + auto const journal = env.app().journal("SHAMapStoreTest"); + + try + { + SHAMapStoreImp restoreCheck(env.app(), scheduler, journal); + restoreCheck.start(); + fail( + "Expected startup failure for stale persisted pinned " + "ranges"); + } + catch (std::runtime_error const& e) + { + BEAST_EXPECT( + std::string(e.what()).find( + "Persisted pinned interval 100-200") != + std::string::npos); + } + } + } + + // Helper: compare two RangeSet for set-equality. The + // test cares about which sequences are in the result, not the + // particular interval representation. + static bool + sameRanges( + RangeSet const& a, + RangeSet const& b) + { + return a == b; + } + + void + testComputeOnlineDeleteTargets() + { + // detail::computeOnlineDeleteTargets is the lifted interval + // arithmetic used by SHAMapStoreImp::clearSqlRanges to decide + // which ledger sequences to actually delete during online + // delete. The base window is [minSeq, lastRotated - 1] and + // pinned ranges are subtracted from it. The result must: + // (a) be empty when the base is empty or fully pinned, + // (b) preserve all pinned seqs (they survive rotation), + // (c) cover every non-pinned seq in the base. + // Lifting the math out keeps the contract testable without + // standing up a full SHAMapStoreImp + database fixture. + testcase("detail::computeOnlineDeleteTargets"); + + using detail::computeOnlineDeleteTargets; + + // Empty base window (minSeq >= lastRotated): no work. + { + RangeSet empty; + BEAST_EXPECT(computeOnlineDeleteTargets(100, 100, empty).empty()); + BEAST_EXPECT(computeOnlineDeleteTargets(101, 100, empty).empty()); + } + + // No pins: the entire base window is the target. + { + RangeSet empty; + RangeSet expected; + expected.insert(range(50u, 99u)); // [50, 100-1] + BEAST_EXPECT(sameRanges( + computeOnlineDeleteTargets(50, 100, empty), expected)); + } + + // Pins fully cover the base window: nothing to delete. This + // is the bug the pinning feature exists to prevent — pinned + // ranges getting silently rotated away. + { + RangeSet pinned; + pinned.insert(range(50u, 99u)); + BEAST_EXPECT(computeOnlineDeleteTargets(50, 100, pinned).empty()); + } + + // Pins extend beyond the base window: still nothing to delete. + { + RangeSet pinned; + pinned.insert(range(0u, 1000u)); + BEAST_EXPECT(computeOnlineDeleteTargets(50, 100, pinned).empty()); + } + + // Pins cover only the leading section of the base. Result is + // the trailing remainder. + { + RangeSet pinned; + pinned.insert(range(50u, 70u)); + RangeSet expected; + expected.insert(range(71u, 99u)); + BEAST_EXPECT(sameRanges( + computeOnlineDeleteTargets(50, 100, pinned), expected)); + } + + // Pins cover only the trailing section. Result is leading. + { + RangeSet pinned; + pinned.insert(range(80u, 99u)); + RangeSet expected; + expected.insert(range(50u, 79u)); + BEAST_EXPECT(sameRanges( + computeOnlineDeleteTargets(50, 100, pinned), expected)); + } + + // Pins cut a hole in the middle. Result is two disjoint + // intervals — both get deleted, the hole is preserved. + { + RangeSet pinned; + pinned.insert(range(60u, 80u)); + RangeSet expected; + expected.insert(range(50u, 59u)); + expected.insert(range(81u, 99u)); + BEAST_EXPECT(sameRanges( + computeOnlineDeleteTargets(50, 100, pinned), expected)); + } + + // Multiple disjoint pinned ranges all inside the base. Result + // is the base minus all pinned holes. + { + RangeSet pinned; + pinned.insert(range(55u, 60u)); + pinned.insert(range(70u, 75u)); + pinned.insert(range(90u, 95u)); + RangeSet expected; + expected.insert(range(50u, 54u)); + expected.insert(range(61u, 69u)); + expected.insert(range(76u, 89u)); + expected.insert(range(96u, 99u)); + BEAST_EXPECT(sameRanges( + computeOnlineDeleteTargets(50, 100, pinned), expected)); + } + + // Pinned ranges entirely outside the base window: no effect. + { + RangeSet pinned; + pinned.insert(range(0u, 49u)); + pinned.insert(range(100u, 200u)); + RangeSet expected; + expected.insert(range(50u, 99u)); + BEAST_EXPECT(sameRanges( + computeOnlineDeleteTargets(50, 100, pinned), expected)); + } + + // Single-sequence base window with a single-sequence pin + // matching exactly: empty result. + { + RangeSet pinned; + pinned.insert(range(99u, 99u)); + BEAST_EXPECT(computeOnlineDeleteTargets(99, 100, pinned).empty()); + } + } + void run() override { + testComputeOnlineDeleteTargets(); testClear(); testAutomatic(); testCanDelete(); testRotate(); + testPinnedRangeRestoreRequiresPinnedData(); } }; diff --git a/src/test/core/Config_test.cpp b/src/test/core/Config_test.cpp index a99d48f3a..7214d7764 100644 --- a/src/test/core/Config_test.cpp +++ b/src/test/core/Config_test.cpp @@ -1513,6 +1513,133 @@ r.ripple.com:51235 } } + void + testDatabasePinnedValidation() + { + testcase("DatabasePinned config validation"); + + // Test 1: pinned_type=rwdb in non-standalone mode (should throw) + { + Config c; + std::string toLoad = + "[node_db]\n" + "type=NuDB\n" + "path=main\n" + "online_delete=256\n" + "pinned_type=rwdb\n" + "pinned_path=pinned\n"; + c.setupControl(true, true, false); // standalone = false + try + { + c.loadFromString(toLoad); + fail("Expected exception for non-durable pinned_type"); + } + catch (std::runtime_error const& e) + { + BEAST_EXPECT( + std::string(e.what()).find("durable backend") != + std::string::npos); + pass(); + } + } + + // Test 2: pinned_type=none in non-standalone mode (should throw) + { + Config c; + std::string toLoad = + "[node_db]\n" + "type=NuDB\n" + "path=main\n" + "online_delete=256\n" + "pinned_type=none\n" + "pinned_path=pinned\n"; + c.setupControl(true, true, false); // standalone = false + try + { + c.loadFromString(toLoad); + fail("Expected exception for non-durable pinned_type"); + } + catch (std::runtime_error const& e) + { + BEAST_EXPECT( + std::string(e.what()).find("durable backend") != + std::string::npos); + pass(); + } + } + + // Test 3: pinned_type=rwdb in standalone mode (should succeed + // — standalone is unrestricted) + { + Config c; + std::string toLoad = + "[node_db]\n" + "type=NuDB\n" + "path=main\n" + "online_delete=256\n" + "pinned_type=rwdb\n" + "pinned_path=pinned\n"; + c.setupControl(true, true, true); // standalone = true + try + { + c.loadFromString(toLoad); + pass(); + } + catch (std::runtime_error const& e) + { + fail("Should not throw in standalone mode"); + } + } + + // Test 4: pinned_type=NuDB in non-standalone mode (should succeed) + { + Config c; + std::string toLoad = + "[node_db]\n" + "type=NuDB\n" + "path=main\n" + "online_delete=256\n" + "pinned_type=NuDB\n" + "pinned_path=pinned\n"; + c.setupControl(true, true, false); // standalone = false + try + { + c.loadFromString(toLoad); + pass(); + } + catch (std::runtime_error const& e) + { + fail("Should not throw for durable pinned_type"); + } + } + + // Test 5: pinned_type without online_delete (should throw) + { + Config c; + std::string toLoad = + "[node_db]\n" + "type=NuDB\n" + "path=main\n" + "pinned_type=NuDB\n" + "pinned_path=pinned\n"; + c.setupControl(true, true, false); + try + { + c.loadFromString(toLoad); + fail( + "Expected exception for pinned_type without " + "online_delete"); + } + catch (std::runtime_error const& e) + { + BEAST_EXPECT( + std::string(e.what()).find("online_delete") != + std::string::npos); + pass(); + } + } + } + void testOverlay() { @@ -1604,6 +1731,7 @@ r.ripple.com:51235 testGetters(); testAmendment(); testRWDBOnlineDelete(); + testDatabasePinnedValidation(); testOverlay(); testNetworkID(); } diff --git a/src/test/nodestore/DatabasePinned_test.cpp b/src/test/nodestore/DatabasePinned_test.cpp new file mode 100644 index 000000000..c937241b5 --- /dev/null +++ b/src/test/nodestore/DatabasePinned_test.cpp @@ -0,0 +1,195 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2026 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ripple { +namespace test { + +class DatabasePinned_test : public beast::unit_test::suite +{ + // Build an Env whose nodestore is wired up as DatabasePinnedImp. + // pinned_type=rwdb is allowed in standalone mode (the catalogue + // tests do the same). + static std::unique_ptr + pinnedEnvconfig() + { + auto cfg = jtx::envconfig(); + auto& nodeDb = cfg->section(ConfigSection::nodeDatabase()); + nodeDb.set("pinned_type", "rwdb"); + nodeDb.set("online_delete", "256"); + return cfg; + } + + // Make a NodeObject with deterministic but unique-per-call hash so + // we can store it and fetch it back. + static std::shared_ptr + makeObject(NodeObjectType type, std::uint8_t marker) + { + Blob data(64, marker); + // Hash is just a stable function of (type, marker) so each + // (type, marker) pair has a distinct key. + Serializer s; + s.add32(static_cast(type)); + s.add8(marker); + auto const hash = sha512Half(makeSlice(s.peekData())); + return NodeObject::createObject(type, std::move(data), hash); + } + + // Borrow the running app's nodestore. With pinned_type configured + // it must be a DatabasePinnedImp; otherwise the test environment + // is set up wrong and we can't proceed. + NodeStore::DatabasePinnedImp* + getPinnedDb(jtx::Env& env) + { + auto* db = dynamic_cast( + &env.app().getNodeStore()); + BEAST_EXPECT(db != nullptr); + return db; + } + + void + testStoreRoutingByType() + { + // The architectural core of DatabasePinnedImp: store() routes + // by NodeObjectType. pinned* types go to the persistent + // backend (which never rotates); everything else goes to the + // rotating backend (which is rotated by online_delete). If + // routing breaks, pinned data ends up rotating away and the + // whole feature fails silently. + testcase("store routing by NodeObjectType"); + + using namespace jtx; + Env env{*this, pinnedEnvconfig()}; + auto* db = getPinnedDb(env); + if (!db) + return; + + auto const pinnedBefore = db->pinnedStoreCount(); + auto const hotBefore = db->hotStoreCount(); + + // Pinned types: should each increment pinnedStoreCount. + // The store() implementation also reduces them to their hot + // equivalents before serialization (single-byte on-disk type + // field), but the routing decision happens first. + for (auto const type : + {pinnedLEDGER, pinnedACCOUNT_NODE, pinnedTRANSACTION_NODE}) + { + db->store( + type, + Blob(64, 0xAB), + sha512Half( + makeSlice(std::string{"pin-"} + std::to_string(type))), + /*ledgerSeq=*/100); + } + + // Hot types: should each increment hotStoreCount. + for (auto const type : + {hotLEDGER, hotACCOUNT_NODE, hotTRANSACTION_NODE}) + { + db->store( + type, + Blob(64, 0xCD), + sha512Half( + makeSlice(std::string{"hot-"} + std::to_string(type))), + /*ledgerSeq=*/100); + } + + // 3 pinned stores routed to persistent_, 3 hot to rotating_. + BEAST_EXPECT(db->pinnedStoreCount() - pinnedBefore == 3u); + BEAST_EXPECT(db->hotStoreCount() - hotBefore == 3u); + } + + void + testStoreFetchRoundTrip() + { + // Verifies that data routed to either backend is retrievable + // through the unified Database interface (fetchNodeObject + // returns the right object regardless of which backend it + // landed in). This is the read-side counterpart to the + // routing test — confirms callers don't need to know about + // the dual-backend split. + testcase("store/fetch round-trip across both backends"); + + using namespace jtx; + Env env{*this, pinnedEnvconfig()}; + auto& nodeStore = env.app().getNodeStore(); + + // Store one object of each type with distinct content so the + // fetched data is unambiguously the one we stored. + struct Item + { + NodeObjectType type; + std::uint8_t marker; + std::shared_ptr obj; + }; + + std::vector items{ + {pinnedLEDGER, 0x01, nullptr}, + {pinnedACCOUNT_NODE, 0x02, nullptr}, + {pinnedTRANSACTION_NODE, 0x03, nullptr}, + {hotLEDGER, 0x04, nullptr}, + {hotACCOUNT_NODE, 0x05, nullptr}, + {hotTRANSACTION_NODE, 0x06, nullptr}, + }; + + for (auto& it : items) + { + it.obj = makeObject(it.type, it.marker); + // Database::store takes (type, data, hash, ledgerSeq). + // Pass a copy of the data because store() takes Blob&&. + nodeStore.store( + it.type, + Blob(it.obj->getData()), + it.obj->getHash(), + /*ledgerSeq=*/100); + } + + for (auto const& it : items) + { + auto const fetched = + nodeStore.fetchNodeObject(it.obj->getHash(), 100); + BEAST_EXPECT(fetched != nullptr); + if (!fetched) + continue; + BEAST_EXPECT(fetched->getData() == it.obj->getData()); + } + } + +public: + void + run() override + { + testStoreRoutingByType(); + testStoreFetchRoundTrip(); + } +}; + +BEAST_DEFINE_TESTSUITE(DatabasePinned, nodestore, ripple); + +} // namespace test +} // namespace ripple diff --git a/src/test/rdb/RelationalDatabase_test.cpp b/src/test/rdb/RelationalDatabase_test.cpp index bfacf2142..9edf1b2ea 100644 --- a/src/test/rdb/RelationalDatabase_test.cpp +++ b/src/test/rdb/RelationalDatabase_test.cpp @@ -541,6 +541,124 @@ public: } } + void + testRangeDeletionOperations( + std::string const& backend, + std::unique_ptr config) + { + testcase("Range deletion operations - " + backend); + + using namespace test::jtx; + config->LEDGER_HISTORY = 1000; + + Env env(*this, std::move(config)); + auto& db = env.app().getRelationalDatabase(); + + Account alice("alice"); + Account bob("bob"); + + env.fund(XRP(10000), alice, bob); + env.close(); + + auto* sqliteDb = getInterface(db); + BEAST_EXPECT(sqliteDb != nullptr); + if (!sqliteDb) + return; + + // Create 5 ledgers with transactions + for (int i = 0; i < 5; ++i) + { + env(pay(alice, bob, XRP(100 + i))); + env.close(); + } + + auto minSeq = sqliteDb->getMinLedgerSeq(); + auto maxSeq = sqliteDb->getMaxLedgerSeq(); + BEAST_EXPECT(minSeq && maxSeq); + if (!minSeq || !maxSeq) + return; + + auto initialTxCount = sqliteDb->getTransactionCount(); + auto initialAcctTxCount = sqliteDb->getAccountTransactionCount(); + + // Use a range that covers ledgers with transactions + // (funded ledger + 5 payment ledgers = sequences 3..7 typically) + LedgerIndex rangeStart = *minSeq; + LedgerIndex rangeEnd = *maxSeq; + + // Test deleteTransactionsInRange with limit — ensures forward + // progress (no infinite loop on partial deletes) + { + std::size_t totalDeleted = 0; + std::size_t iterations = 0; + std::size_t const limit = 2; + + while (true) + { + auto deleted = sqliteDb->deleteTransactionsInRange( + rangeStart, rangeEnd, limit); + if (deleted == 0) + break; + totalDeleted += deleted; + ++iterations; + // Safety: prevent infinite loop in case of bug + if (iterations > initialTxCount + 1) + { + fail( + "deleteTransactionsInRange: too many iterations, " + "possible infinite loop"); + break; + } + } + BEAST_EXPECT(totalDeleted > 0); + BEAST_EXPECT(totalDeleted <= initialTxCount); + auto afterTxCount = sqliteDb->getTransactionCount(); + BEAST_EXPECT(afterTxCount == initialTxCount - totalDeleted); + } + + // Test deleteAccountTransactionsInRange with limit + { + std::size_t totalDeleted = 0; + std::size_t iterations = 0; + std::size_t const limit = 2; + + while (true) + { + auto deleted = sqliteDb->deleteAccountTransactionsInRange( + rangeStart, rangeEnd, limit); + if (deleted == 0) + break; + totalDeleted += deleted; + ++iterations; + if (iterations > initialAcctTxCount + 1) + { + fail( + "deleteAccountTransactionsInRange: too many " + "iterations, possible infinite loop"); + break; + } + } + BEAST_EXPECT(totalDeleted > 0); + BEAST_EXPECT(totalDeleted <= initialAcctTxCount); + auto afterAcctTxCount = sqliteDb->getAccountTransactionCount(); + BEAST_EXPECT(afterAcctTxCount == initialAcctTxCount - totalDeleted); + } + + // Test deleteLedgersInRange + auto initialLedgerCount = sqliteDb->getLedgerCountMinMax(); + auto ledgersDeleted = + sqliteDb->deleteLedgersInRange(rangeStart + 1, rangeEnd - 1); + BEAST_EXPECT(ledgersDeleted > 0); + auto afterLedgerCount = sqliteDb->getLedgerCountMinMax(); + BEAST_EXPECT( + afterLedgerCount.numberOfRows == + initialLedgerCount.numberOfRows - ledgersDeleted); + + // Test range deletion with no matching range + auto emptyDeleted = sqliteDb->deleteLedgersInRange(999999, 999999); + BEAST_EXPECT(emptyDeleted == 0); + } + void testDatabaseSpaceOperations( std::string const& backend, @@ -744,6 +862,7 @@ public: testAccountTransactionOperations(backend, makeConfig(backend)); testAccountTransactionPaging(backend, makeConfig(backend)); testDeletionOperations(backend, makeConfig(backend)); + testRangeDeletionOperations(backend, makeConfig(backend)); testDatabaseSpaceOperations(backend, makeConfig(backend)); testTransactionMinLedgerSeq(backend, makeConfig(backend)); } diff --git a/src/test/rpc/CatalogueStream_test.cpp b/src/test/rpc/CatalogueStream_test.cpp new file mode 100644 index 000000000..27e358bd4 --- /dev/null +++ b/src/test/rpc/CatalogueStream_test.cpp @@ -0,0 +1,322 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2025 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +namespace ripple { +namespace test { + +class CatalogueStream_test : public beast::unit_test::suite +{ + // The deserialize* public API takes `CatalogueInputStream&` which is just + // `boost::iostreams::filtering_istream`. So no production-code change is + // needed for fault-injection — we hand-build a byte vector that violates + // the wire format, push it through an `array_source`, and feed it to + // deserialize. Each test below targets one error branch in + // CatalogueStream.cpp. + // + // Wire format reminder (from serializeSHAMapToStream): + // leaf : [type:u8 ][key:32][size:u32][data:size] + // remove: [254 :u8 ][key:32] + // end : [255 :u8 ] + + using Bytes = std::vector; + + static void + appendByte(Bytes& b, std::uint8_t v) + { + b.push_back(v); + } + + static void + appendBytes(Bytes& b, void const* p, std::size_t n) + { + auto const* src = reinterpret_cast(p); + b.insert(b.end(), src, src + n); + } + + static void + appendKey(Bytes& b, std::uint8_t fillByte = 0xAA) + { + b.insert(b.end(), 32, fillByte); + } + + static void + appendU32(Bytes& b, std::uint32_t v) + { + // CatalogueStream writes raw bytes — no endian conversion. Match. + appendBytes(b, &v, sizeof(v)); + } + + // Build a fresh filtering_istream over `bytes`. The array_source is a + // value type stored by the filter chain, so `bytes` must outlive it. + static void + makeStream(Bytes const& bytes, RPC::CatalogueInputStream& out) + { + boost::iostreams::array_source src( + reinterpret_cast(bytes.data()), bytes.size()); + out.push(src); + } + + // Construct an empty SHAMap of the requested type using the test app's + // node family. Same pattern used in src/test/consensus/UNLReport_test.cpp. + static std::shared_ptr + makeMap(jtx::Env& env, SHAMapType type) + { + return std::make_shared(type, env.app().getNodeFamily()); + } + +public: + void + testTruncatedKey() + { + testcase("deserialize: truncated read of key"); + + jtx::Env env(*this); + auto map = makeMap(env, SHAMapType::STATE); + + // 1 byte (the node type) and nothing after — read of the 32-byte key + // hits EOF inside deserializeLeaf, triggers the "stream stopped while + // trying to read key" branch. + Bytes b; + appendByte(b, tnACCOUNT_STATE); + + RPC::CatalogueInputStream stream; + makeStream(b, stream); + + BEAST_EXPECT(!RPC::deserializeStateMapFromStream( + *map, stream, hotACCOUNT_NODE, env.app().journal("Test"))); + } + + void + testTruncatedSize() + { + testcase("deserialize: truncated read of size"); + + jtx::Env env(*this); + auto map = makeMap(env, SHAMapType::TRANSACTION); + + // type + key but no 4-byte size — triggers the "stream stopped while + // trying to read size" branch. Use tx map so the type byte isn't a + // remove (which would short-circuit before the size read). + Bytes b; + appendByte(b, tnTRANSACTION_MD); + appendKey(b); + + RPC::CatalogueInputStream stream; + makeStream(b, stream); + + BEAST_EXPECT(!RPC::deserializeTxMapFromStream( + *map, stream, hotTRANSACTION_NODE, env.app().journal("Test"))); + } + + void + testSizeTooLarge() + { + testcase("deserialize: size > 1 GiB rejected"); + + jtx::Env env(*this); + auto map = makeMap(env, SHAMapType::STATE); + + // type + key + size of 2 GiB — bails before any data read attempt. + Bytes b; + appendByte(b, tnACCOUNT_STATE); + appendKey(b); + appendU32(b, 2u * 1024u * 1024u * 1024u); + + RPC::CatalogueInputStream stream; + makeStream(b, stream); + + BEAST_EXPECT(!RPC::deserializeStateMapFromStream( + *map, stream, hotACCOUNT_NODE, env.app().journal("Test"))); + } + + void + testTruncatedData() + { + testcase("deserialize: truncated read of data payload"); + + jtx::Env env(*this); + auto map = makeMap(env, SHAMapType::STATE); + + // type + key + size=10 + only 5 bytes — hits the "Unexpected EOF + // while reading data" branch. + Bytes b; + appendByte(b, tnACCOUNT_STATE); + appendKey(b); + appendU32(b, 10); + for (int i = 0; i < 5; ++i) + appendByte(b, 0x42); + + RPC::CatalogueInputStream stream; + makeStream(b, stream); + + BEAST_EXPECT(!RPC::deserializeStateMapFromStream( + *map, stream, hotACCOUNT_NODE, env.app().journal("Test"))); + } + + void + testRemoveDisallowedInTxMap() + { + testcase( + "deserialize: remove sentinel rejected when allowRemoval=false"); + + jtx::Env env(*this); + auto map = makeMap(env, SHAMapType::TRANSACTION); + + // tx maps don't accept tnREMOVE — the deserialize for tx maps calls + // the inner helper with allowRemoval=false. We send a remove record + // and expect rejection at "unexpected removal in this map type". + Bytes b; + appendByte(b, tnREMOVE); + appendKey(b); + + RPC::CatalogueInputStream stream; + makeStream(b, stream); + + BEAST_EXPECT(!RPC::deserializeTxMapFromStream( + *map, stream, hotTRANSACTION_NODE, env.app().journal("Test"))); + } + + void + testRemoveOfAbsentKey() + { + testcase("deserialize: remove of absent key rejected"); + + jtx::Env env(*this); + auto map = makeMap(env, SHAMapType::STATE); + + // State map allows tnREMOVE, but the key has to be present first. + // Empty map + remove of any key → "key is already absent" branch. + Bytes b; + appendByte(b, tnREMOVE); + appendKey(b, 0x77); + + RPC::CatalogueInputStream stream; + makeStream(b, stream); + + BEAST_EXPECT(!RPC::deserializeStateMapFromStream( + *map, stream, hotACCOUNT_NODE, env.app().journal("Test"))); + } + + void + testMissingTerminator() + { + testcase("deserialize: EOF before terminal sentinel"); + + jtx::Env env(*this); + auto map = makeMap(env, SHAMapType::STATE); + + // One valid leaf, then EOF (no tnTERMINAL). The outer loop exits via + // stream.eof() with lastParsed = tnACCOUNT_STATE (not tnTERMINAL), + // so the "Unexpected EOF, terminal node not found" branch fires. + // Payload must be >= 12 bytes — SHAMapLeafNode asserts that for + // state-tree leaves. + Bytes b; + appendByte(b, tnACCOUNT_STATE); + appendKey(b, 0x11); + appendU32(b, 16); + for (int i = 0; i < 16; ++i) + appendByte(b, 0xCC); + + RPC::CatalogueInputStream stream; + makeStream(b, stream); + + BEAST_EXPECT(!RPC::deserializeStateMapFromStream( + *map, stream, hotACCOUNT_NODE, env.app().journal("Test"))); + } + + void + testEmptyStreamAcceptsTerminator() + { + testcase("deserialize: bare terminator → empty map, success"); + + jtx::Env env(*this); + auto map = makeMap(env, SHAMapType::STATE); + + Bytes b; + appendByte(b, tnTERMINAL); + + RPC::CatalogueInputStream stream; + makeStream(b, stream); + + BEAST_EXPECT(RPC::deserializeStateMapFromStream( + *map, stream, hotACCOUNT_NODE, env.app().journal("Test"))); + } + + void + testHappyPathRoundTrip() + { + testcase("deserialize: valid leaf + terminator → success"); + + jtx::Env env(*this); + auto map = makeMap(env, SHAMapType::STATE); + + // type + key + size=16 + 16 data bytes + terminator. Confirms the + // happy path through addGiveItem and flushDirty. Size must be >= 12 + // — SHAMapLeafNode asserts that for state-tree leaves. + Bytes b; + appendByte(b, tnACCOUNT_STATE); + appendKey(b, 0x55); + appendU32(b, 16); + for (int i = 0; i < 16; ++i) + appendByte(b, static_cast(i)); + appendByte(b, tnTERMINAL); + + RPC::CatalogueInputStream stream; + makeStream(b, stream); + + BEAST_EXPECT(RPC::deserializeStateMapFromStream( + *map, stream, hotACCOUNT_NODE, env.app().journal("Test"))); + + uint256 key; + std::memset(key.data(), 0x55, 32); + BEAST_EXPECT(map->hasItem(key)); + } + + void + run() override + { + testTruncatedKey(); + testTruncatedSize(); + testSizeTooLarge(); + testTruncatedData(); + testRemoveDisallowedInTxMap(); + testRemoveOfAbsentKey(); + testMissingTerminator(); + testEmptyStreamAcceptsTerminator(); + testHappyPathRoundTrip(); + } +}; + +BEAST_DEFINE_TESTSUITE(CatalogueStream, rpc, ripple); + +} // namespace test +} // namespace ripple diff --git a/src/test/rpc/Catalogue_test.cpp b/src/test/rpc/Catalogue_test.cpp index d2283f234..421f35564 100644 --- a/src/test/rpc/Catalogue_test.cpp +++ b/src/test/rpc/Catalogue_test.cpp @@ -19,15 +19,103 @@ #include #include +#include #include +#include #include #include #include +#include #include +#include #include namespace ripple { +namespace { + +// Parameters for polling ledger retrieval +struct LedgerRetryParams +{ + test::jtx::Env& env; + std::uint32_t seq = 0; + std::chrono::milliseconds pollInterval = std::chrono::milliseconds(500); + std::chrono::milliseconds maxWaitTime = std::chrono::milliseconds(10000); + bool requireValidated = false; +}; + +// Poll for ledger availability with retry logic +// Returns nullptr if ledger cannot be retrieved within the timeout period +std::shared_ptr +getLedgerWithRetry(const LedgerRetryParams& params) +{ + auto start = std::chrono::steady_clock::now(); + + while (true) + { + // First try to get the hash for this sequence + auto hash = params.env.app().getLedgerMaster().getHashBySeq(params.seq); + + if (hash.isNonZero()) + { + // Hash found, now try to get the ledger + auto ledger = + params.env.app().getLedgerMaster().getLedgerByHash(hash); + + if (ledger) + { + // If we don't require validation, or if it's already validated, + // return it + if (!params.requireValidated || ledger->info().validated) + { + return ledger; + } + // Otherwise continue polling until it becomes validated + } + } + + // Check if we've exceeded the timeout + auto elapsed = std::chrono::steady_clock::now() - start; + if (elapsed >= params.maxWaitTime) + { + return nullptr; + } + + // Wait before next attempt + std::this_thread::sleep_for(params.pollInterval); + } +} + +// Options for catalogueEnvconfig. Use designated initializers at call sites +// so each test reads as a small diff against the default — easy to scan and +// trivially extensible (add fields here, no call-site churn). +// catalogueEnvconfig() // happy path +// catalogueEnvconfig({.withPinnedType = false}) // exercise guard +struct CatalogueEnvOpts +{ + // Set [node_db] pinned_type=rwdb so catalogue_load passes its guard. + // false → omit pinned_type entirely (drives the rejection path). + bool withPinnedType = true; + + // [node_db] online_delete value (lifetime of rotating store, in ledgers). + std::string onlineDelete = "256"; +}; + +// envconfig with pinned_type set so catalogue_load passes the config guard. +// Uses rwdb for in-process speed (allowed in standalone). +inline std::unique_ptr +catalogueEnvconfig(CatalogueEnvOpts const& opts = {}) +{ + auto cfg = test::jtx::envconfig(); + auto& nodeDb = cfg->section(ConfigSection::nodeDatabase()); + if (opts.withPinnedType) + nodeDb.set("pinned_type", "rwdb"); + nodeDb.set("online_delete", opts.onlineDelete); + return cfg; +} + +} // anonymous namespace + #pragma pack(push, 1) // pack the struct tightly struct TestCATLHeader { @@ -215,7 +303,24 @@ class Catalogue_test : public beast::unit_test::suite { testcase("catalogue_load: Invalid parameters"); using namespace test::jtx; - Env env{*this, envconfig(), features}; + Env env{*this, catalogueEnvconfig(), features}; + + // Missing [node_db] pinned_type — catalogue_load rejects up front so + // loaded data isn't lost on the next rotation. Use a separate Env + // because this guard fires before any input_file parsing. + { + Env envNoPin{ + *this, catalogueEnvconfig({.withPinnedType = false}), features}; + Json::Value params{Json::objectValue}; + params[jss::input_file] = "/tmp/anything.catl"; + auto const result = + envNoPin.client().invoke("catalogue_load", params)[jss::result]; + BEAST_EXPECT(result[jss::error] == "invalidParams"); + BEAST_EXPECT(result[jss::status] == "error"); + BEAST_EXPECT( + result[jss::error_message].asString().find("pinned_type") != + std::string::npos); + } // No parameters { @@ -253,6 +358,42 @@ class Catalogue_test : public beast::unit_test::suite BEAST_EXPECT(result[jss::error] == "internal"); BEAST_EXPECT(result[jss::status] == "error"); } + + // Header has min_ledger > max_ledger + { + boost::filesystem::path tempDir = + boost::filesystem::temp_directory_path() / + boost::filesystem::unique_path(); + boost::filesystem::create_directories(tempDir); + + auto cataloguePath = (tempDir / "invalid-range.catl").string(); + + TestCATLHeader header; + header.min_ledger = 20; + header.max_ledger = 10; + header.version = 1; + header.network_id = env.app().config().NETWORK_ID; + header.filesize = sizeof(TestCATLHeader); + + std::ofstream outfile( + cataloguePath.c_str(), std::ios::out | std::ios::binary); + BEAST_EXPECT(outfile.good()); + outfile.write( + reinterpret_cast(&header), sizeof(header)); + outfile.close(); + + Json::Value params{Json::objectValue}; + params[jss::input_file] = cataloguePath; + auto const result = + env.client().invoke("catalogue_load", params)[jss::result]; + BEAST_EXPECT(result[jss::error] == "invalidParams"); + BEAST_EXPECT(result[jss::status] == "error"); + BEAST_EXPECT( + result[jss::error_message].asString().find( + "min_ledger must be <= max_ledger") != std::string::npos); + + boost::filesystem::remove_all(tempDir); + } } void @@ -262,9 +403,25 @@ class Catalogue_test : public beast::unit_test::suite using namespace test::jtx; // Create environment and test data - Env env{*this, envconfig(), features}; + Env env{*this, catalogueEnvconfig(), nullptr, beast::severities::kNone}; prepareLedgerData(env, 5); + auto noop = [](test::jtx::Env& env, + std::string partition, + std::string severity) { + Json::Value params{Json::objectValue}; + params[jss::severity] = severity; + params[jss::partition] = partition; + // env.client().invoke("log_level", params); + }; + + // Create journal for debugging + auto j = env.app().logs().journal("Catalogue_test"); + noop(env, "CatalogueTools", "trace"); + noop(env, "Catalogue_test", "trace"); + + JLOG(j.trace()) << "Test environment created, prepared ledger data"; + // Store some key state information before catalogue creation auto const sourceLedger = env.closed(); auto const bobKeylet = keylet::account(Account("bob").id()); @@ -274,6 +431,9 @@ class Catalogue_test : public beast::unit_test::suite Account("bob").id(), Currency(to_currency("EUR"))); + JLOG(j.trace()) << "Source ledger seq: " << sourceLedger->info().seq + << " hash: " << sourceLedger->info().hash; + // Get original state entries auto const bobAcct = sourceLedger->read(bobKeylet); auto const charlieAcct = sourceLedger->read(charlieKeylet); @@ -302,6 +462,10 @@ class Catalogue_test : public beast::unit_test::suite // First create a catalogue uint32_t minLedger = 3; uint32_t maxLedger = sourceLedger->info().seq; + + JLOG(j.trace()) << "Creating catalogue from ledger " << minLedger + << " to " << maxLedger; + { Json::Value params{Json::objectValue}; params[jss::min_ledger] = minLedger; @@ -311,27 +475,43 @@ class Catalogue_test : public beast::unit_test::suite auto const result = env.client().invoke("catalogue_create", params)[jss::result]; BEAST_EXPECT(result[jss::status] == jss::success); + + JLOG(j.trace()) << "Catalogue created: " << result.toStyledString(); } - // Create a new environment for loading with unique port + // Create a new environment for loading the catalogue + // We use a separate environment with incremented ports to avoid + // conflicts Note: The default RWDB backend works fine - the key insight + // is that pinned ledgers bypass the cache, so we need to poll for + // availability as the async publishAcqLedger jobs complete Env loadEnv{ *this, - test::jtx::envconfig(), + catalogueEnvconfig(), features, }; + noop(loadEnv, "CatalogueTools", "trace"); + noop(loadEnv, "Catalogue_test", "trace"); + + // Create journal for load environment + auto loadJ = loadEnv.app().logs().journal("Catalogue_test"); + // Now load the catalogue Json::Value params{Json::objectValue}; params[jss::input_file] = cataloguePath; + JLOG(loadJ.trace()) << "Loading catalogue from " << cataloguePath; + auto const result = loadEnv.client().invoke("catalogue_load", params)[jss::result]; + JLOG(loadJ.trace()) + << "Catalogue load result: " << result.toStyledString(); + BEAST_EXPECT(result[jss::status] == jss::success); BEAST_EXPECT(result[jss::ledger_min] == minLedger); BEAST_EXPECT(result[jss::ledger_max] == maxLedger); BEAST_EXPECT(result[jss::ledger_count] == (maxLedger - minLedger + 1)); - // Verify complete_ledgers reflects loaded ledgers auto const newCompleteLedgers = loadEnv.app().getLedgerMaster().getCompleteLedgers(); @@ -346,20 +526,36 @@ class Catalogue_test : public beast::unit_test::suite // Compare all ledgers from 3 to 16 inclusive for (std::uint32_t seq = 3; seq <= 16; ++seq) { - auto const sourceLedger = - env.app().getLedgerMaster().getLedgerByHash( - env.app().getLedgerMaster().getHashBySeq(seq)); + JLOG(j.trace()) << "Comparing ledger " << seq; - auto const loadedLedger = - loadEnv.app().getLedgerMaster().getLedgerByHash( - loadEnv.app().getLedgerMaster().getHashBySeq(seq)); + // Get the source ledger (doesn't need to be validated) + auto const sourceLedger = getLedgerWithRetry( + {.env = env, + .seq = seq, + .pollInterval = std::chrono::milliseconds(100), + .maxWaitTime = std::chrono::milliseconds(5000), + .requireValidated = false}); + + // Get the loaded ledger (must be validated) + auto const loadedLedger = getLedgerWithRetry( + {.env = loadEnv, + .seq = seq, + .pollInterval = std::chrono::milliseconds(500), + .maxWaitTime = std::chrono::milliseconds(30000), + .requireValidated = true}); if (!sourceLedger || !loadedLedger) { + JLOG(j.trace()) + << "Failed to get ledger " << seq + << " source: " << (sourceLedger ? "ok" : "missing") + << " loaded: " << (loadedLedger ? "ok" : "missing"); BEAST_EXPECT(false); // Test failure continue; } + JLOG(j.trace()) << "Got both ledgers for seq " << seq; + // Check basic ledger properties BEAST_EXPECT(sourceLedger->info().seq == loadedLedger->info().seq); BEAST_EXPECT( @@ -404,6 +600,10 @@ class Catalogue_test : public beast::unit_test::suite std::size_t sourceCount = std::ranges::distance(sourceLedger->sles); std::size_t loadedCount = std::ranges::distance(loadedLedger->sles); + JLOG(j.trace()) + << "Ledger " << seq << " SLE count - source: " << sourceCount + << " loaded: " << loadedCount; + BEAST_EXPECT(sourceCount == loadedCount); // Check existence of imported keylets @@ -411,6 +611,13 @@ class Catalogue_test : public beast::unit_test::suite { auto const key = sle->key(); bool exists = loadedLedger->exists(keylet::unchecked(key)); + + if (!exists) + { + JLOG(j.trace()) + << "Ledger " << seq << " missing key: " << key; + } + BEAST_EXPECT(exists); // If it exists, check the serialized form matches @@ -421,6 +628,13 @@ class Catalogue_test : public beast::unit_test::suite sle->add(s1); loadedSle->add(s2); bool serializedEqual = (s1.peekData() == s2.peekData()); + + if (!serializedEqual) + { + JLOG(j.trace()) + << "Ledger " << seq << " mismatch for key: " << key; + } + BEAST_EXPECT(serializedEqual); } } @@ -429,10 +643,20 @@ class Catalogue_test : public beast::unit_test::suite for (auto const& sle : loadedLedger->sles) { auto const key = sle->key(); - BEAST_EXPECT(sourceLedger->exists(keylet::unchecked(key))); + bool exists = sourceLedger->exists(keylet::unchecked(key)); + + if (!exists) + { + JLOG(j.trace()) + << "Ledger " << seq << " extra key in loaded: " << key; + } + + BEAST_EXPECT(exists); } } + JLOG(j.trace()) << "Ledger comparison complete"; + auto const loadedBobAcct = loadedLedger->read(bobKeylet); auto const loadedCharlieAcct = loadedLedger->read(charlieKeylet); auto const loadedEurTrust = loadedLedger->read(eurTrustKeylet); @@ -523,10 +747,11 @@ class Catalogue_test : public beast::unit_test::suite // Try to load catalogue in environment with different network ID Env env2{ *this, - envconfig([](std::unique_ptr cfg) { + [&]() { + auto cfg = catalogueEnvconfig(); cfg->NETWORK_ID = 456; return cfg; - }), + }(), features, }; @@ -553,7 +778,7 @@ class Catalogue_test : public beast::unit_test::suite // Create environment and test data Env env{ *this, - envconfig(), + catalogueEnvconfig(), features, nullptr, beast::severities::kDisabled, @@ -650,7 +875,7 @@ class Catalogue_test : public beast::unit_test::suite // Create environment and test data Env env{ *this, - envconfig(), + catalogueEnvconfig(), features, nullptr, beast::severities::kDisabled, @@ -729,7 +954,7 @@ class Catalogue_test : public beast::unit_test::suite using namespace test::jtx; // Create environment and test data - Env env{*this, envconfig(), features}; + Env env{*this, catalogueEnvconfig(), features}; prepareLedgerData(env, 5); boost::filesystem::path tempDir = @@ -818,7 +1043,7 @@ class Catalogue_test : public beast::unit_test::suite using namespace test::jtx; // Create environment - Env env{*this, envconfig(), features}; + Env env{*this, catalogueEnvconfig(), features}; boost::filesystem::path tempDir = boost::filesystem::temp_directory_path() / diff --git a/src/xrpld/app/ledger/Ledger.cpp b/src/xrpld/app/ledger/Ledger.cpp index 10d1c4436..145298dac 100644 --- a/src/xrpld/app/ledger/Ledger.cpp +++ b/src/xrpld/app/ledger/Ledger.cpp @@ -1044,7 +1044,8 @@ pendSaveValidated( Application& app, std::shared_ptr const& ledger, bool isSynchronous, - bool isCurrent) + bool isCurrent, + std::function callback) { if (!app.getHashRouter().setFlags(ledger->info().hash, SF_SAVED)) { @@ -1056,6 +1057,8 @@ pendSaveValidated( { // Either we don't need it to be finished // or it is finished + if (callback) + callback(true); return true; } } @@ -1069,6 +1072,8 @@ pendSaveValidated( JLOG(stream) << "Pend save with seq in pending saves " << ledger->info().seq; + if (callback) + callback(true); return true; } @@ -1077,15 +1082,20 @@ pendSaveValidated( app.getJobQueue().addJob( isCurrent ? jtPUBLEDGER : jtPUBOLDLEDGER, std::to_string(ledger->seq()), - [&app, ledger, isCurrent]() { - saveValidatedLedger(app, ledger, isCurrent); + [&app, ledger, isCurrent, callback]() { + bool success = saveValidatedLedger(app, ledger, isCurrent); + if (callback) + callback(success); })) { return true; } // The JobQueue won't do the Job. Do the save synchronously. - return saveValidatedLedger(app, ledger, isCurrent); + bool success = saveValidatedLedger(app, ledger, isCurrent); + if (callback) + callback(success); + return success; } void diff --git a/src/xrpld/app/ledger/Ledger.h b/src/xrpld/app/ledger/Ledger.h index 0476859c8..3c7c2c4e6 100644 --- a/src/xrpld/app/ledger/Ledger.h +++ b/src/xrpld/app/ledger/Ledger.h @@ -452,7 +452,8 @@ pendSaveValidated( Application& app, std::shared_ptr const& ledger, bool isSynchronous, - bool isCurrent); + bool isCurrent, + std::function callback = {}); std::shared_ptr loadLedgerHelper(LedgerInfo const& sinfo, Application& app, bool acquire); diff --git a/src/xrpld/app/ledger/LedgerMaster.h b/src/xrpld/app/ledger/LedgerMaster.h index e59b1b78f..c4ac1e510 100644 --- a/src/xrpld/app/ledger/LedgerMaster.h +++ b/src/xrpld/app/ledger/LedgerMaster.h @@ -139,6 +139,9 @@ public: RangeSet getPinnedLedgersRangeSet(); + void + setPinnedLedgersRangeSet(const RangeSet& range_set); + /** Apply held transactions to the open ledger This is normally called as we close the ledger. The open ledger remains open to handle new transactions @@ -205,6 +208,10 @@ public: void clearLedger(std::uint32_t seq); bool + isPinned(std::uint32_t seq); + void + unpinLedger(std::uint32_t seq); + bool isValidated(ReadView const& ledger); bool getValidatedRange(std::uint32_t& minVal, std::uint32_t& maxVal); @@ -359,7 +366,12 @@ private: std::recursive_mutex mCompleteLock; RangeSet mCompleteLedgers; + + // When both locks are needed, acquire them together with std::scoped_lock. + std::mutex mPinnedLock; RangeSet mPinnedLedgers; // Track pinned ledger ranges + bool mPinnedMergedToComplete{ + false}; // One-shot: pinned merged into mCompleteLedgers // Publish thread is running. bool mAdvanceThread{false}; diff --git a/src/xrpld/app/ledger/detail/LedgerMaster.cpp b/src/xrpld/app/ledger/detail/LedgerMaster.cpp index 2efc6dd9e..5b3275c04 100644 --- a/src/xrpld/app/ledger/detail/LedgerMaster.cpp +++ b/src/xrpld/app/ledger/detail/LedgerMaster.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -446,14 +447,25 @@ bool LedgerMaster::storeLedger(std::shared_ptr ledger, bool pin) { bool validated = ledger->info().validated; - // Returns true if we already had the ledger - if (!mLedgerHistory.insert(std::move(ledger), validated)) + // Returns true if we already had the ledger. + // + // When pin=true, we deliberately skip mLedgerHistory insertion. + // A single catalogue pack can contain tens of thousands of ledgers, + // each holding a full SHAMap of account/transaction nodes. Keeping + // them all in the in-memory history cache would consume hundreds of + // GB of RAM. Instead, pinned ledgers are persisted to the nodestore + // and SQLite, and are retrievable via RPC (which reads from SQLite), + // but are NOT kept in memory. + if (!pin && !mLedgerHistory.insert(std::move(ledger), validated)) return false; if (pin) { uint32_t seq = ledger->info().seq; - mPinnedLedgers.insert(range(seq, seq)); + { + std::lock_guard sl(mPinnedLock); + mPinnedLedgers.insert(range(seq, seq)); + } JLOG(m_journal.info()) << "Pinned ledger : " << seq; } return true; @@ -512,7 +524,7 @@ LedgerMaster::haveLedger(std::uint32_t seq) void LedgerMaster::clearLedger(std::uint32_t seq) { - std::lock_guard sl(mCompleteLock); + std::scoped_lock lock(mCompleteLock, mPinnedLock); // Don't clear pinned ledgers if (boost::icl::contains(mPinnedLedgers, seq)) @@ -525,6 +537,20 @@ LedgerMaster::clearLedger(std::uint32_t seq) mCompleteLedgers.erase(seq); } +bool +LedgerMaster::isPinned(std::uint32_t seq) +{ + std::lock_guard sl(mPinnedLock); + return boost::icl::contains(mPinnedLedgers, seq); +} + +void +LedgerMaster::unpinLedger(std::uint32_t seq) +{ + std::lock_guard sl(mPinnedLock); + mPinnedLedgers.erase(range(seq, seq)); +} + bool LedgerMaster::isValidated(ReadView const& ledger) { @@ -861,7 +887,19 @@ LedgerMaster::setFullLedger( pendSaveValidated(app_, ledger, isSynchronous, isCurrent); { - std::lock_guard ml(mCompleteLock); + std::scoped_lock lock(mCompleteLock, mPinnedLock); + // One-time merge of pinned ranges into mCompleteLedgers. + // For NORMAL/NETWORK startup, this fires on the first validated + // ledger (network quorum). For LOAD/standalone, the merge + // already happened in setPinnedLedgersRangeSet() and this + // is a no-op (mPinnedMergedToComplete is already true). + if (!mPinnedMergedToComplete && !mPinnedLedgers.empty()) + { + for (auto const& interval : mPinnedLedgers) + mCompleteLedgers.insert(interval); + mPinnedMergedToComplete = true; + } + mCompleteLedgers.insert(ledger->info().seq); } @@ -1286,53 +1324,134 @@ LedgerMaster::findNewLedgersToPublish( auto valLedger = mValidLedger.get(); std::uint32_t valSeq = valLedger->info().seq; + // Create a range of ledgers we need to publish + RangeSet toPublish; + toPublish.insert(range(pubSeq, valSeq)); + + // CRITICAL: Skip publishing pinned ledgers (except the most recent). + // + // When loading catalogues, we deliberately avoid storing pinned ledgers in + // the AcceptedLedger cache to prevent memory bloat with millions of + // ledgers. However, NetworkOPs::pubLedger() expects every published ledger + // to either: + // 1. Already exist in the AcceptedLedger cache, OR + // 2. Be able to create and cache a new AcceptedLedger + // + // For pinned ledgers, we skip the cache entirely during + // saveValidatedLedger. If we try to publish them, pubLedger creates a new + // AcceptedLedger, but canonicalize_replace_client might return a different + // instance, causing: "Assertion failed: alpAccepted->getLedger().get() == + // lpAccepted.get()" + // + // This primarily affects STANDALONE MODE where catalogue_load calls + // switchLCL when loading historical ledgers newer than the current closed + // ledger. In live network mode, historical ledgers are typically older than + // the current ledger, so switchLCL -> tryAdvance -> publish isn't + // triggered. + // + // Solution: Only publish the most recent validated ledger (which needs + // publishing) and skip all intermediate pinned ledgers (already on disk). + { + std::lock_guard sll(mPinnedLock); + RangeSet pinnedExceptLast = mPinnedLedgers; + // Remove the most recent from the pinned set so we always publish it + if (boost::icl::contains(pinnedExceptLast, valSeq)) + pinnedExceptLast.erase(range(valSeq, valSeq)); + // Subtract pinned ledgers from the set to publish + toPublish -= pinnedExceptLast; + } + scope_unlock sul{sl}; try { - for (std::uint32_t seq = pubSeq; seq <= valSeq; ++seq) + for (auto const& interval : toPublish) { - JLOG(m_journal.trace()) - << "Trying to fetch/publish valid ledger " << seq; + // Some sequences may be absent from toPublish because they are + // pinned ledgers already persisted to disk and intentionally not + // republished. The decision logic — "can pubSeq safely jump + // across this gap?" — lives in detail::canSkipPinnedGap so it + // is unit-testable without spinning up a full LedgerMaster. + // + // Core invariant (enforced by canSkipPinnedGap): never publish + // across a gap in non-pinned ledgers. Publication must remain + // contiguous for every non-pinned sequence from + // mPubLedgerSeq + 1 forward. If pubSeq is lagging because an + // earlier non-pinned ledger could not be fetched/published, we + // stop here rather than publish newer ledgers out of order. + // + // In production this branch is essentially dead code: pinned + // ranges are old historical catalogue data and pubSeq tracks + // the recent published tip, so toPublish won't have pinned- + // induced gaps. It only matters in test/standalone catalogue- + // load scenarios where pinned ranges can sit near pubSeq. + if (pubSeq < interval.first()) + { + bool canSkip; + { + std::lock_guard sll(mPinnedLock); + canSkip = detail::canSkipPinnedGap( + pubSeq, interval.first(), mPinnedLedgers); + } - std::shared_ptr ledger; - // This can throw - auto hash = hashOfSeq(*valLedger, seq, m_journal); - // VFALCO TODO Restructure this code so that zero is not - // used. - if (!hash) - hash = beast::zero; // kludge - if (seq == valSeq) - { - // We need to publish the ledger we just fully validated - ledger = valLedger; - } - else if (hash->isZero()) - { - JLOG(m_journal.fatal()) << "Ledger: " << valSeq - << " does not have hash for " << seq; - UNREACHABLE( - "ripple::LedgerMaster::findNewLedgersToPublish : ledger " - "not found"); - } - else - { - ledger = mLedgerHistory.getLedgerByHash(*hash); + if (!canSkip) + { + JLOG(m_journal.trace()) + << "Stopping publish at seq " << pubSeq + << " — skipped span [" << pubSeq << ", " + << interval.first() << ") contains non-pinned ledgers"; + break; + } + + pubSeq = interval.first(); } - if (!app_.config().LEDGER_REPLAY) + for (std::uint32_t seq = interval.first(); seq <= interval.last(); + ++seq) { - // Can we try to acquire the ledger we need? - if (!ledger && (++acqCount < ledger_fetch_size_)) - ledger = app_.getInboundLedgers().acquire( - *hash, seq, InboundLedger::Reason::GENERIC); - } + JLOG(m_journal.trace()) + << "Trying to fetch/publish valid ledger " << seq; - // Did we acquire the next ledger we need to publish? - if (ledger && (ledger->info().seq == pubSeq)) - { - ledger->setValidated(); - ret.push_back(ledger); - ++pubSeq; + std::shared_ptr ledger; + // This can throw + auto hash = hashOfSeq(*valLedger, seq, m_journal); + // VFALCO TODO Restructure this code so that zero is not + // used. + if (!hash) + hash = beast::zero; // kludge + if (seq == valSeq) + { + // We need to publish the ledger we just fully validated + ledger = valLedger; + } + else if (hash->isZero()) + { + JLOG(m_journal.fatal()) + << "Ledger: " << valSeq << " does not have hash for " + << seq; + UNREACHABLE( + "ripple::LedgerMaster::findNewLedgersToPublish : " + "ledger not found"); + } + else + { + ledger = mLedgerHistory.getLedgerByHash(*hash); + } + + if (!app_.config().LEDGER_REPLAY) + { + // Can we try to acquire the ledger we need? + if (!ledger && (++acqCount < ledger_fetch_size_)) + ledger = app_.getInboundLedgers().acquire( + *hash, seq, InboundLedger::Reason::GENERIC); + } + + // Did we acquire the next ledger we need to publish? + if (ledger && (ledger->info().seq == pubSeq)) + { + ledger->setValidated(); + ret.push_back(ledger); + ++pubSeq; + } } } @@ -1632,7 +1751,7 @@ LedgerMaster::getCompleteLedgers() std::string LedgerMaster::getPinnedLedgers() { - std::lock_guard sl(mCompleteLock); + std::lock_guard sl(mPinnedLock); return to_string(mPinnedLedgers); } @@ -1646,10 +1765,46 @@ LedgerMaster::getCompleteLedgersRangeSet() RangeSet LedgerMaster::getPinnedLedgersRangeSet() { - std::lock_guard sl(mCompleteLock); + std::lock_guard sl(mPinnedLock); return mPinnedLedgers; } +void +LedgerMaster::setPinnedLedgersRangeSet(const RangeSet& range_set) +{ + std::scoped_lock lock(mCompleteLock, mPinnedLock); + if (!mPinnedLedgers.empty()) + { + Throw( + "Expected mPinnedLedgers to be empty on startup"); + } + mPinnedLedgers.assign(range_set); + + // Normally, we defer merging pinned ranges into mCompleteLedgers until + // the first validated ledger (in setFullLedger), so that pinned history + // only becomes queryable after the node has network quorum. But if + // mCompleteLedgers is already non-empty (LOAD/standalone startup already + // called setFullLedger before we got here), the one-shot in + // setFullLedger won't fire again. Merge immediately in that case. + if (!mCompleteLedgers.empty()) + { + for (auto const& interval : mPinnedLedgers) + mCompleteLedgers.insert(interval); + mPinnedMergedToComplete = true; + JLOG(m_journal.info()) + << "Merged pinned ranges into complete ledgers immediately " + << "(startup already past setFullLedger): " << to_string(range_set); + } + else + { + // mCompleteLedgers is empty — defer merge to setFullLedger() + // when the first validated ledger arrives (network quorum). + JLOG(m_journal.info()) + << "Loaded pinned ranges, will merge on first validation: " + << to_string(range_set); + } +} + std::optional LedgerMaster::getCloseTimeBySeq(LedgerIndex ledgerIndex) { @@ -1813,15 +1968,18 @@ LedgerMaster::setLedgerRangePresent( std::uint32_t maxV, bool pin) { - std::lock_guard sl(mCompleteLock); - mCompleteLedgers.insert(range(minV, maxV)); - if (pin) { + std::scoped_lock lock(mCompleteLock, mPinnedLock); + mCompleteLedgers.insert(range(minV, maxV)); mPinnedLedgers.insert(range(minV, maxV)); JLOG(m_journal.info()) << "Pinned ledger range: " << minV << " - " << maxV; + return; } + + std::lock_guard cl(mCompleteLock); + mCompleteLedgers.insert(range(minV, maxV)); } void @@ -1841,7 +1999,7 @@ LedgerMaster::getCacheHitRate() void LedgerMaster::clearPriorLedgers(LedgerIndex seq) { - std::lock_guard sl(mCompleteLock); + std::scoped_lock lock(mCompleteLock, mPinnedLock); if (seq <= 0) return; diff --git a/src/xrpld/app/ledger/detail/PublishGap.h b/src/xrpld/app/ledger/detail/PublishGap.h new file mode 100644 index 000000000..60af2ec84 --- /dev/null +++ b/src/xrpld/app/ledger/detail/PublishGap.h @@ -0,0 +1,87 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2026 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_APP_LEDGER_DETAIL_PUBLISH_GAP_H_INCLUDED +#define RIPPLE_APP_LEDGER_DETAIL_PUBLISH_GAP_H_INCLUDED + +#include + +#include + +namespace ripple { +namespace detail { + +/** Decide whether pubSeq can safely advance across a gap in the + publication set, given the pinned-ledger ranges. + + Used by LedgerMaster::findNewLedgersToPublish: when the + publish-target set has been split into intervals (because pinned + ranges were subtracted from it), pubSeq may be behind the start of + the next interval. We may only "jump" pubSeq forward across that + gap if every skipped sequence is pinned — pinned ledgers are + persisted on disk and are intentionally never republished, so + skipping them does not violate the contiguous-publication + invariant. If any sequence in the gap is *not* pinned, then pubSeq + is lagging because an earlier non-pinned ledger could not be + fetched/published; we must halt rather than publish newer ledgers + out of order. + + Practical note: in production this guard is essentially never + exercised. Pinned ranges are old historical catalogue ranges and + pubSeq tracks the recent published tip — they don't overlap. The + guard matters only in test/standalone catalogue-load scenarios + where a catalogue can be loaded into an environment whose ledger + history places pinned seqs near the publish horizon. We keep the + guard rather than rely on the assumption holding because (a) the + cost is one RangeSet subtraction per gap and (b) silently + publishing across a non-pinned gap would corrupt the stream + contract. + + @param pubSeq The next sequence the publisher would like + to publish. + @param intervalStart The first sequence in the next non-pinned + interval to publish. + @param pinned The current pinned-ledger ranges. + + @return true pubSeq can safely jump to intervalStart + (pubSeq >= intervalStart, or every skipped seq is + pinned). + @return false the gap [pubSeq, intervalStart) contains at least + one non-pinned sequence; the publisher must stop. +*/ +inline bool +canSkipPinnedGap( + std::uint32_t pubSeq, + std::uint32_t intervalStart, + RangeSet const& pinned) +{ + // No gap. + if (pubSeq >= intervalStart) + return true; + + RangeSet skipped; + skipped.insert(range(pubSeq, intervalStart - 1)); + skipped -= pinned; + return skipped.empty(); +} + +} // namespace detail +} // namespace ripple + +#endif diff --git a/src/xrpld/app/misc/README.md b/src/xrpld/app/misc/README.md index 52e6e1493..769958d73 100644 --- a/src/xrpld/app/misc/README.md +++ b/src/xrpld/app/misc/README.md @@ -152,4 +152,4 @@ deletion routine. * [fetch_depth] will be silently set to equal the online_delete setting if online_delete is greater than fetch_depth. * In the [node_db] section, there is a performance tuning option, delete_batch, -which sets the maximum size in ledgers for each SQL DELETE query. +which sets the maximum number of SQL rows deleted per batch. diff --git a/src/xrpld/app/misc/SHAMapStore.h b/src/xrpld/app/misc/SHAMapStore.h index d8415713a..c78e653b2 100644 --- a/src/xrpld/app/misc/SHAMapStore.h +++ b/src/xrpld/app/misc/SHAMapStore.h @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -61,6 +62,9 @@ public: virtual LedgerIndex setCanDelete(LedgerIndex canDelete) = 0; + virtual void + setPinnedRanges(RangeSet const& ranges) = 0; + /** Whether advisory delete is enabled. */ virtual bool advisoryDelete() const = 0; diff --git a/src/xrpld/app/misc/SHAMapStoreImp.cpp b/src/xrpld/app/misc/SHAMapStoreImp.cpp index aff3569d3..31c0d80c6 100644 --- a/src/xrpld/app/misc/SHAMapStoreImp.cpp +++ b/src/xrpld/app/misc/SHAMapStoreImp.cpp @@ -21,15 +21,20 @@ #include #include +#include #include #include #include #include +#include #include #include #include #include +#include + +#include namespace ripple { void @@ -79,6 +84,20 @@ SHAMapStoreImp::SavedStateDB::setLastRotated(LedgerIndex seq) ripple::setLastRotated(sqlDb_, seq); } +std::string +SHAMapStoreImp::SavedStateDB::getPinnedRanges() +{ + std::lock_guard lock(mutex_); + return ripple::getPinnedRanges(sqlDb_); +} + +void +SHAMapStoreImp::SavedStateDB::setPinnedRanges(std::string const& ranges) +{ + std::lock_guard lock(mutex_); + ripple::setPinnedRanges(sqlDb_, ranges); +} + //------------------------------------------------------------------------------ SHAMapStoreImp::SHAMapStoreImp( @@ -117,6 +136,9 @@ SHAMapStoreImp::SHAMapStoreImp( get_if_exists(section, "online_delete", deleteInterval_); + // Always initialize state database for pinned ranges persistence + state_db_.init(config, dbName_); + if (deleteInterval_) { // Configuration that affects the behavior of online delete @@ -153,7 +175,6 @@ SHAMapStoreImp::SHAMapStoreImp( std::to_string(config.LEDGER_HISTORY) + ")"); } - state_db_.init(config, dbName_); if (!config.mem_backend()) dbPaths(); } @@ -183,6 +204,8 @@ SHAMapStoreImp::makeNodeStore(int readThreads) { SavedState state = state_db_.getState(); + // Create the rotation backends - needed for both DatabaseRotating and + // DatabasePinned auto writableBackend = makeBackendRotating(state.writableDb); auto archiveBackend = makeBackendRotating(state.archiveDb); if (!state.writableDb.size()) @@ -192,19 +215,58 @@ SHAMapStoreImp::makeNodeStore(int readThreads) state_db_.setState(state); } - // Create NodeStore with two backends to allow online deletion of - // data - auto dbr = std::make_unique( - app_, - scheduler_, - readThreads, - std::move(writableBackend), - std::move(archiveBackend), - nscfg, - app_.logs().journal(nodeStoreName_)); - fdRequired_ += dbr->fdRequired(); - dbRotating_ = dbr.get(); - db.reset(dynamic_cast(dbr.release())); + // Check if DatabasePinned should be created + if (nscfg.exists("pinned_type")) + { + // DatabasePinned uses the same rotation backends as + // DatabaseRotating but adds a persistent backend for pinned nodes + + // Create persistent backend for pinned data + Section pinnedConfig = nscfg; + pinnedConfig.set("type", *nscfg.get("pinned_type")); + if (auto pinnedPath = nscfg.get("pinned_path")) + pinnedConfig.set("path", *pinnedPath); + auto pinnedBackend = NodeStore::Manager::instance().make_Backend( + pinnedConfig, + megabytes(app_.config().getValueFor( + SizedItem::burstSize, std::nullopt)), + scheduler_, + app_.logs().journal(nodeStoreName_)); + pinnedBackend->open(); + + // Create DatabasePinned with rotation backends + persistent + + auto dbp = std::make_unique( + app_, + scheduler_, + readThreads, + std::move(writableBackend), + std::move(archiveBackend), + std::move(pinnedBackend), + nscfg, + app_.logs().journal(NodeStore::DatabasePinnedImp::JournalName)); + + fdRequired_ += dbp->fdRequired(); + dbRotating_ = + dbp.get(); // DatabasePinned inherits from DatabaseRotating + db.reset(dynamic_cast(dbp.release())); + } + else + { + // Create NodeStore with two backends to allow online deletion of + // data + auto dbr = std::make_unique( + app_, + scheduler_, + readThreads, + std::move(writableBackend), + std::move(archiveBackend), + nscfg, + app_.logs().journal(nodeStoreName_)); + fdRequired_ += dbr->fdRequired(); + dbRotating_ = dbr.get(); + db.reset(dynamic_cast(dbr.release())); + } } else { @@ -265,6 +327,58 @@ SHAMapStoreImp::copyNode(std::uint64_t& nodeCount, SHAMapTreeNode const& node) return true; } +void +SHAMapStoreImp::loadPinnedRanges() +{ + auto rangesStr = state_db_.getPinnedRanges(); + if (rangesStr.empty()) + return; + + auto const& nscfg = app_.config().section(ConfigSection::nodeDatabase()); + if (!nscfg.exists("pinned_type")) + { + JLOG(journal_.warn()) + << "Ignoring persisted pinned ranges because [node_db] " + "pinned_type is not configured: " + << rangesStr; + return; + } + + RangeSet persistedRanges; + if (!from_string(persistedRanges, rangesStr)) + { + Throw( + "Failed to parse persisted pinned ranges in state.db: " + + rangesStr); + } + + // state.db alone is not sufficient evidence that pinned history is + // available in the currently configured backend. Validate each interval + // boundary against the actual node store before advertising it as pinned. + auto const hasLedgerData = [this](std::uint32_t seq) { + auto const hash = app_.getRelationalDatabase().getHashByIndex(seq); + return hash.isNonZero() && + static_cast(app_.getNodeStore().fetchNodeObject(hash, seq)); + }; + + for (auto const& interval : persistedRanges) + { + if (!hasLedgerData(interval.lower()) || + !hasLedgerData(interval.upper())) + { + Throw( + "Persisted pinned interval " + + std::to_string(interval.lower()) + "-" + + std::to_string(interval.upper()) + + " is not present in the currently configured pinned store"); + } + } + + JLOG(journal_.info()) << "Loaded pinned ranges from database: " + << rangesStr; + app_.getLedgerMaster().setPinnedLedgersRangeSet(persistedRanges); +} + void SHAMapStoreImp::run() { @@ -486,17 +600,19 @@ SHAMapStoreImp::makeBackendRotating(std::string path) Section section{app_.config().section(ConfigSection::nodeDatabase())}; boost::filesystem::path newPath; - if (path.size()) + if (!path.empty()) { newPath = path; } else { + // Create new empty backend with unique path boost::filesystem::path p = get(section, "path"); p /= dbPrefix_; p += ".%%%%"; newPath = boost::filesystem::unique_path(p); } + section.set("path", newPath.string()); auto backend{NodeStore::Manager::instance().make_Backend( @@ -510,56 +626,43 @@ SHAMapStoreImp::makeBackendRotating(std::string path) } void -SHAMapStoreImp::clearSql( +SHAMapStoreImp::clearSqlRanges( LedgerIndex lastRotated, - std::string const& TableName, + RangeSet const& pinned, + std::string const& tableName, std::function()> const& getMinSeq, - std::function const& deleteBeforeSeq) + std::function const&)> const& deleteInRanges) { XRPL_ASSERT( deleteInterval_, "ripple::SHAMapStoreImp::clearSql : nonzero delete interval"); - LedgerIndex min = std::numeric_limits::max(); - - { - JLOG(journal_.trace()) - << "Begin: Look up lowest value of: " << TableName; - auto m = getMinSeq(); - JLOG(journal_.trace()) << "End: Look up lowest value of: " << TableName; - if (!m) - return; - min = *m; - } - - if (min > lastRotated || healthWait() == stopping) + auto m = getMinSeq(); + if (!m) return; - if (min == lastRotated) + + if (healthWait() == stopping) + return; + + // The pure interval-math is in detail::computeOnlineDeleteTargets so + // it can be unit-tested without spinning up SHAMapStoreImp. It + // returns the disjoint deletable intervals after subtracting pinned + // ranges from the base window [minSeq, lastRotated - 1]. + auto const target = + detail::computeOnlineDeleteTargets(*m, lastRotated, pinned); + if (target.empty()) { - // Micro-optimization mainly to clarify logs - JLOG(journal_.trace()) << "Nothing to delete from " << TableName; + JLOG(journal_.trace()) << "Nothing to delete from " << tableName + << " after considering pins."; return; } - JLOG(journal_.debug()) << "start deleting in: " << TableName << " from " - << min << " to " << lastRotated; - while (min < lastRotated) - { - min = std::min(lastRotated, min + deleteBatch_); - JLOG(journal_.trace()) - << "Begin: Delete up to " << deleteBatch_ - << " rows with LedgerSeq < " << min << " from: " << TableName; - deleteBeforeSeq(min); - JLOG(journal_.trace()) - << "End: Delete up to " << deleteBatch_ << " rows with LedgerSeq < " - << min << " from: " << TableName; - if (healthWait() == stopping) - return; - if (min < lastRotated) - std::this_thread::sleep_for(backOff_); - if (healthWait() == stopping) - return; - } - JLOG(journal_.debug()) << "finished deleting from: " << TableName; + JLOG(journal_.debug()) << "Pruning " << tableName + << ". Target ranges: " << to_string(target); + + // The deleteInRanges lambda will handle the actual database operations + deleteInRanges(target); + + JLOG(journal_.debug()) << "finished deleting from: " << tableName; } void @@ -581,6 +684,14 @@ SHAMapStoreImp::freshenCaches() void SHAMapStoreImp::clearPrior(LedgerIndex lastRotated) { + // Get pinned ranges to exclude from deletion + auto pinnedRanges = app_.getLedgerMaster().getPinnedLedgersRangeSet(); + if (!pinnedRanges.empty()) + { + JLOG(journal_.info()) + << "Online delete with pinned ranges: " << to_string(pinnedRanges); + } + // Do not allow ledgers to be acquired from the network // that are about to be deleted. minimumOnline_ = lastRotated + 1; @@ -600,36 +711,96 @@ SHAMapStoreImp::clearPrior(LedgerIndex lastRotated) if (app_.config().useTxTables()) { - clearSql( + clearSqlRanges( lastRotated, + pinnedRanges, "Transactions", [&db]() -> std::optional { return db->getTransactionsMinLedgerSeq(); }, - [&db](LedgerIndex min) -> void { - db->deleteTransactionsBeforeLedgerSeq(min); + [this, &db](RangeSet const& ranges) -> void { + for (auto const& interval : ranges) + { + // Simple delete loop with LIMIT + while (true) + { + if (healthWait() == stopping) + return; + + auto deleted = db->deleteTransactionsInRange( + interval.lower(), interval.upper(), deleteBatch_); + + if (deleted == 0) + break; + + JLOG(journal_.trace()) << "Deleted " << deleted + << " rows from Transactions"; + std::this_thread::sleep_for(backOff_); + } + } }); if (healthWait() == stopping) return; - clearSql( + clearSqlRanges( lastRotated, + pinnedRanges, "AccountTransactions", [&db]() -> std::optional { return db->getAccountTransactionsMinLedgerSeq(); }, - [&db](LedgerIndex min) -> void { - db->deleteAccountTransactionsBeforeLedgerSeq(min); + [this, &db](RangeSet const& ranges) -> void { + for (auto const& interval : ranges) + { + // Simple delete loop with LIMIT + while (true) + { + if (healthWait() == stopping) + return; + + auto deleted = db->deleteAccountTransactionsInRange( + interval.lower(), interval.upper(), deleteBatch_); + + if (deleted == 0) + break; + + JLOG(journal_.trace()) + << "Deleted " << deleted + << " rows from AccountTransactions"; + std::this_thread::sleep_for(backOff_); + } + } }); if (healthWait() == stopping) return; } - clearSql( + clearSqlRanges( lastRotated, + pinnedRanges, "Ledgers", [db]() -> std::optional { return db->getMinLedgerSeq(); }, - [db](LedgerIndex min) -> void { db->deleteBeforeLedgerSeq(min); }); + [this, db](RangeSet const& ranges) -> void { + for (auto const& interval : ranges) + { + // Simple delete loop with LIMIT + while (true) + { + if (healthWait() == stopping) + return; + + auto deleted = db->deleteLedgersInRange( + interval.lower(), interval.upper(), deleteBatch_); + + if (deleted == 0) + break; + + JLOG(journal_.trace()) + << "Deleted " << deleted << " rows from Ledgers"; + std::this_thread::sleep_for(backOff_); + } + } + }); if (healthWait() == stopping) return; } diff --git a/src/xrpld/app/misc/SHAMapStoreImp.h b/src/xrpld/app/misc/SHAMapStoreImp.h index 7d36f092b..1d4ac2bf1 100644 --- a/src/xrpld/app/misc/SHAMapStoreImp.h +++ b/src/xrpld/app/misc/SHAMapStoreImp.h @@ -24,8 +24,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -67,6 +69,10 @@ private: setState(SavedState const& state); void setLastRotated(LedgerIndex seq); + std::string + getPinnedRanges(); + void + setPinnedRanges(std::string const& ranges); }; Application& app_; @@ -132,6 +138,12 @@ public: : fetch_depth; } + bool + isOnlineDeleteEnabled() const + { + return deleteInterval_ > 0; + } + std::unique_ptr makeNodeStore(int readThreads) override; @@ -168,6 +180,14 @@ public: void onLedgerClosed(std::shared_ptr const& ledger) override; + void + setPinnedRanges(RangeSet const& ranges) override + { + // Save pinned ranges to database for persistence + std::string rangesStr = to_string(ranges); + state_db_.setPinnedRanges(rangesStr); + } + void rendezvous() const override; int @@ -210,11 +230,14 @@ private: * Call with mutex object unlocked. */ void - clearSql( + clearSqlRanges( LedgerIndex lastRotated, - std::string const& TableName, + RangeSet const& pinned, + std::string const& tableName, std::function()> const& getMinSeq, - std::function const& deleteBeforeSeq); + std::function const&)> const& + deleteInRanges); + void clearCaches(LedgerIndex validatedSeq); void @@ -233,12 +256,24 @@ private: [[nodiscard]] HealthResult healthWait(); + void + loadPinnedRanges(); + public: void start() override { + // Always load pinned ranges on startup + loadPinnedRanges(); + + // NOTE: Startup cleanup of unpinned ledgers was removed - it caused + // SQLite lock contention and confusing pauses. Database will shrink + // incrementally during rotation. Custom tools will provide pruning + // management for specialized needs. if (deleteInterval_) + { thread_ = std::thread(&SHAMapStoreImp::run, this); + } } void diff --git a/src/xrpld/app/misc/detail/OnlineDeleteRanges.h b/src/xrpld/app/misc/detail/OnlineDeleteRanges.h new file mode 100644 index 000000000..3935b0edf --- /dev/null +++ b/src/xrpld/app/misc/detail/OnlineDeleteRanges.h @@ -0,0 +1,77 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2026 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_APP_MISC_DETAIL_ONLINE_DELETE_RANGES_H_INCLUDED +#define RIPPLE_APP_MISC_DETAIL_ONLINE_DELETE_RANGES_H_INCLUDED + +#include + +#include + +namespace ripple { +namespace detail { + +/** Compute the actual SQL-deletion target windows for an online-delete + pass, given a base [minSeq, lastRotated - 1] window and the pinned + ranges that must be preserved. + + The base window is the half-open span the caller would normally + delete (everything older than `lastRotated`, but no older than the + minimum sequence currently stored in the table being cleared). + Pinned ranges (catalogue history that must survive rotation) are + subtracted from the base — the result may be empty (everything is + pinned), one interval (no overlap, or pinned only trims an edge), + or multiple disjoint intervals (pins cut a hole in the base). + + Pure function so the math is unit-testable without standing up + SHAMapStoreImp. The actual side-effecting deletions (driven by + a per-table lambda in clearSqlRanges) operate on whatever interval + set this helper returns. + + @param minSeq The smallest sequence currently stored in the + table being cleared. If >= lastRotated, the + base window is empty and we return {}. + @param lastRotated The exclusive upper bound — sequences from + this seq forward are kept. + @param pinned Pinned-ledger ranges to preserve. + + @return The set of intervals to actually delete. Empty if either + the base window is empty or pinned ranges fully cover it. +*/ +inline RangeSet +computeOnlineDeleteTargets( + std::uint32_t minSeq, + std::uint32_t lastRotated, + RangeSet const& pinned) +{ + // Empty base window: nothing newer than minSeq qualifies for + // deletion. + if (minSeq >= lastRotated) + return {}; + + RangeSet target; + target.insert(range(minSeq, lastRotated - 1)); + target -= pinned; + return target; +} + +} // namespace detail +} // namespace ripple + +#endif diff --git a/src/xrpld/app/rdb/State.h b/src/xrpld/app/rdb/State.h index e65e9d4d5..6590cdef0 100644 --- a/src/xrpld/app/rdb/State.h +++ b/src/xrpld/app/rdb/State.h @@ -92,6 +92,23 @@ setSavedState(soci::session& session, SavedState const& state); void setLastRotated(soci::session& session, LedgerIndex seq); +/** + * @brief getPinnedRanges Returns the serialized pinned ledger ranges. + * @param session Session with the database. + * @return Serialized range set string (e.g., + * "1000000-2000000,3000000-3500000"). + */ +std::string +getPinnedRanges(soci::session& session); + +/** + * @brief setPinnedRanges Updates the pinned ledger ranges. + * @param session Session with the database. + * @param ranges Serialized range set string to save. + */ +void +setPinnedRanges(soci::session& session, std::string const& ranges); + } // namespace ripple #endif diff --git a/src/xrpld/app/rdb/backend/RWDBDatabase.h b/src/xrpld/app/rdb/backend/RWDBDatabase.h index 67f9ab455..edbcbed92 100644 --- a/src/xrpld/app/rdb/backend/RWDBDatabase.h +++ b/src/xrpld/app/rdb/backend/RWDBDatabase.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -566,6 +567,120 @@ public: // No-op for in-memory database } + std::size_t + deleteLedgersInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit = std::nullopt) override + { + XRPL_ASSERT( + minSeq <= maxSeq, + "RWDBDatabase::deleteLedgersInRange : minSeq <= maxSeq"); + XRPL_ASSERT( + !rowLimit || *rowLimit > 0, + "RWDBDatabase::deleteLedgersInRange : rowLimit must be positive"); + std::unique_lock lock(mutex_); + auto it = ledgers_.lower_bound(minSeq); + auto end = ledgers_.upper_bound(maxSeq); + + std::size_t count = 0; + while (it != end) + { + if (rowLimit && count >= *rowLimit) + break; + + ledgerHashToSeq_.erase(it->second.info.hash); + it = ledgers_.erase(it); + ++count; + } + return count; + } + + std::size_t + deleteTransactionsInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit = std::nullopt) override + { + XRPL_ASSERT( + minSeq <= maxSeq, + "RWDBDatabase::deleteTransactionsInRange : minSeq <= maxSeq"); + XRPL_ASSERT( + !rowLimit || *rowLimit > 0, + "RWDBDatabase::deleteTransactionsInRange : rowLimit must be " + "positive"); + if (!useTxTables_) + return 0; + + std::unique_lock lock(mutex_); + auto it = ledgers_.lower_bound(minSeq); + auto end = ledgers_.upper_bound(maxSeq); + + // Erase from both the per-ledger map and the global index. + // Important: erase from per-ledger map as we go so that partial + // deletes (hitting limit) make forward progress on the next call. + std::size_t count = 0; + while (it != end) + { + auto txIt = it->second.transactions.begin(); + while (txIt != it->second.transactions.end()) + { + if (rowLimit && count >= *rowLimit) + return count; + + transactionMap_.erase(txIt->first); + txIt = it->second.transactions.erase(txIt); + ++count; + } + ++it; + } + return count; + } + + std::size_t + deleteAccountTransactionsInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit = std::nullopt) override + { + XRPL_ASSERT( + minSeq <= maxSeq, + "RWDBDatabase::deleteAccountTransactionsInRange : minSeq <= " + "maxSeq"); + XRPL_ASSERT( + !rowLimit || *rowLimit > 0, + "RWDBDatabase::deleteAccountTransactionsInRange : rowLimit must be " + "positive"); + if (!useTxTables_) + return 0; + + std::unique_lock lock(mutex_); + std::size_t count = 0; + + for (auto& [_, accountData] : accountTxMap_) + { + auto txIt = accountData.ledgerTxMap.lower_bound(minSeq); + auto txEnd = accountData.ledgerTxMap.upper_bound(maxSeq); + + while (txIt != txEnd) + { + std::size_t toDelete = txIt->second.size(); + if (rowLimit && count + toDelete > *rowLimit) + { + // Partial deletion from this ledger + toDelete = *rowLimit - count; + txIt->second.resize(txIt->second.size() - toDelete); + count += toDelete; + return count; + } + + count += toDelete; + txIt = accountData.ledgerTxMap.erase(txIt); + } + } + return count; + } + ~RWDBDatabase() { // Regular maps can use standard clear diff --git a/src/xrpld/app/rdb/backend/SQLiteDatabase.h b/src/xrpld/app/rdb/backend/SQLiteDatabase.h index 27c01c1b8..c8579afa1 100644 --- a/src/xrpld/app/rdb/backend/SQLiteDatabase.h +++ b/src/xrpld/app/rdb/backend/SQLiteDatabase.h @@ -21,6 +21,7 @@ #define RIPPLE_APP_RDB_BACKEND_SQLITEDATABASE_H_INCLUDED #include +#include namespace ripple { @@ -295,6 +296,47 @@ public: virtual uint32_t getKBUsedTransaction() = 0; + /** + * @brief deleteLedgersInRange Deletes ledgers within the specified range. + * @param minSeq Minimum ledger sequence (inclusive). + * @param maxSeq Maximum ledger sequence (inclusive). + * @param rowLimit Optional limit on number of rows to delete. + * @return Number of rows deleted. + */ + virtual std::size_t + deleteLedgersInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit = std::nullopt) = 0; + + /** + * @brief deleteTransactionsInRange Deletes transactions within the + * specified ledger sequence range. + * @param minSeq Minimum ledger sequence (inclusive). + * @param maxSeq Maximum ledger sequence (inclusive). + * @param rowLimit Optional limit on number of rows to delete. + * @return Number of rows deleted. + */ + virtual std::size_t + deleteTransactionsInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit = std::nullopt) = 0; + + /** + * @brief deleteAccountTransactionsInRange Deletes account transactions + * within the specified ledger sequence range. + * @param minSeq Minimum ledger sequence (inclusive). + * @param maxSeq Maximum ledger sequence (inclusive). + * @param rowLimit Optional limit on number of rows to delete. + * @return Number of rows deleted. + */ + virtual std::size_t + deleteAccountTransactionsInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit = std::nullopt) = 0; + /** * @brief Closes the ledger database */ diff --git a/src/xrpld/app/rdb/backend/detail/Node.cpp b/src/xrpld/app/rdb/backend/detail/Node.cpp index 15b2560b2..a8c3c03e9 100644 --- a/src/xrpld/app/rdb/backend/detail/Node.cpp +++ b/src/xrpld/app/rdb/backend/detail/Node.cpp @@ -157,6 +157,45 @@ deleteBeforeLedgerSeq( << ledgerSeq << ";"; } +std::size_t +deleteRange( + soci::session& session, + TableType type, + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit) +{ + XRPL_ASSERT( + minSeq <= maxSeq, "ripple::detail::deleteRange : minSeq <= maxSeq"); + XRPL_ASSERT( + !rowLimit || *rowLimit > 0, + "ripple::detail::deleteRange : rowLimit must be positive"); + std::string sql; + if (rowLimit) + { + // SQLite doesn't support DELETE...LIMIT unless compiled with + // SQLITE_ENABLE_UPDATE_DELETE_LIMIT. Use subquery workaround. + sql = "DELETE FROM " + to_string(type) + + " WHERE rowid IN (SELECT rowid FROM " + to_string(type) + + " WHERE LedgerSeq >= " + std::to_string(minSeq) + + " AND LedgerSeq <= " + std::to_string(maxSeq) + + " ORDER BY LedgerSeq ASC LIMIT " + std::to_string(*rowLimit) + ");"; + } + else + { + sql = "DELETE FROM " + to_string(type) + + " WHERE LedgerSeq >= " + std::to_string(minSeq) + + " AND LedgerSeq <= " + std::to_string(maxSeq) + ";"; + } + + session << sql; + + // Get the number of rows deleted + long changes = 0; + session << "SELECT changes();", soci::into(changes); + return static_cast(changes); +} + std::size_t getRows(soci::session& session, TableType type) { @@ -224,8 +263,11 @@ saveValidatedLedger( Serializer s(128); s.add32(HashPrefix::ledgerMaster); addRaw(ledger->info(), s); + // Use pinnedLEDGER type for pinned ledgers, hotLEDGER for others + auto ledgerType = + app.getLedgerMaster().isPinned(seq) ? pinnedLEDGER : hotLEDGER; app.getNodeStore().store( - hotLEDGER, std::move(s.modData()), ledger->info().hash, seq); + ledgerType, std::move(s.modData()), ledger->info().hash, seq); } std::shared_ptr aLedger; @@ -235,8 +277,13 @@ saveValidatedLedger( if (!aLedger) { aLedger = std::make_shared(ledger, app); - app.getAcceptedLedgerCache().canonicalize_replace_client( - ledger->info().hash, aLedger); + + // Only cache if the ledger is NOT in the pinned range + if (!app.getLedgerMaster().isPinned(ledger->info().seq)) + { + app.getAcceptedLedgerCache().canonicalize_replace_client( + ledger->info().hash, aLedger); + } } } catch (std::exception const&) diff --git a/src/xrpld/app/rdb/backend/detail/Node.h b/src/xrpld/app/rdb/backend/detail/Node.h index 59c484d20..b12e61389 100644 --- a/src/xrpld/app/rdb/backend/detail/Node.h +++ b/src/xrpld/app/rdb/backend/detail/Node.h @@ -102,6 +102,25 @@ deleteBeforeLedgerSeq( TableType type, LedgerIndex ledgerSeq); +/** + * @brief deleteRange Deletes entries in given table + * for the ledgers within the specified range (inclusive) + * with an optional limit on the number of rows to delete. + * @param session Session with database. + * @param type Table ID from which entries will be deleted. + * @param minSeq Minimum ledger sequence (inclusive). + * @param maxSeq Maximum ledger sequence (inclusive). + * @param rowLimit Optional limit on number of rows to delete. + * @return Number of rows actually deleted. + */ +std::size_t +deleteRange( + soci::session& session, + TableType type, + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit); + /** * @brief getRows Returns number of rows in given table. * @param session Session with database. diff --git a/src/xrpld/app/rdb/backend/detail/SQLiteDatabase.cpp b/src/xrpld/app/rdb/backend/detail/SQLiteDatabase.cpp index 0dd0d9926..3e61b26cd 100644 --- a/src/xrpld/app/rdb/backend/detail/SQLiteDatabase.cpp +++ b/src/xrpld/app/rdb/backend/detail/SQLiteDatabase.cpp @@ -83,6 +83,24 @@ public: void deleteAccountTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) override; + std::size_t + deleteLedgersInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit = std::nullopt) override; + + std::size_t + deleteTransactionsInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit = std::nullopt) override; + + std::size_t + deleteAccountTransactionsInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit = std::nullopt) override; + std::size_t getTransactionCount() override; @@ -949,6 +967,48 @@ SQLiteDatabaseImp::closeLedgerDB() lgrdb_.reset(); } +std::size_t +SQLiteDatabaseImp::deleteLedgersInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit) +{ + if (!existsLedger()) + return 0; + + auto db = checkoutLedger(); + return detail::deleteRange( + *db, detail::TableType::Ledgers, minSeq, maxSeq, rowLimit); +} + +std::size_t +SQLiteDatabaseImp::deleteTransactionsInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit) +{ + if (!existsTransaction()) + return 0; + + auto db = checkoutTransaction(); + return detail::deleteRange( + *db, detail::TableType::Transactions, minSeq, maxSeq, rowLimit); +} + +std::size_t +SQLiteDatabaseImp::deleteAccountTransactionsInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit) +{ + if (!existsTransaction()) + return 0; + + auto db = checkoutTransaction(); + return detail::deleteRange( + *db, detail::TableType::AccountTransactions, minSeq, maxSeq, rowLimit); +} + void SQLiteDatabaseImp::closeTransactionDB() { diff --git a/src/xrpld/app/rdb/detail/State.cpp b/src/xrpld/app/rdb/detail/State.cpp index c3de86068..edad02cc6 100644 --- a/src/xrpld/app/rdb/detail/State.cpp +++ b/src/xrpld/app/rdb/detail/State.cpp @@ -43,6 +43,11 @@ initStateDB( " CanDeleteSeq INTEGER" ");"; + session << "CREATE TABLE IF NOT EXISTS PinnedLedgers (" + " Key INTEGER PRIMARY KEY," + " RangeSet TEXT" + ");"; + std::int64_t count = 0; { // SOCI requires boost::optional (not std::optional) as the parameter. @@ -75,6 +80,17 @@ initStateDB( { session << "INSERT INTO CanDelete VALUES (1, 0);"; } + + // Initialize PinnedLedgers table with empty range if not exists + { + boost::optional countO; + session << "SELECT COUNT(Key) FROM PinnedLedgers WHERE Key = 1;", + soci::into(countO); + if (!countO || *countO == 0) + { + session << "INSERT INTO PinnedLedgers VALUES (1, '');"; + } + } } LedgerIndex @@ -127,4 +143,23 @@ setLastRotated(soci::session& session, LedgerIndex seq) soci::use(seq); } +std::string +getPinnedRanges(soci::session& session) +{ + boost::optional rangeStr; + session << "SELECT RangeSet FROM PinnedLedgers WHERE Key = 1;", + soci::into(rangeStr); + + if (rangeStr) + return *rangeStr; + return ""; +} + +void +setPinnedRanges(soci::session& session, std::string const& ranges) +{ + session << "UPDATE PinnedLedgers SET RangeSet = :ranges WHERE Key = 1;", + soci::use(ranges); +} + } // namespace ripple diff --git a/src/xrpld/core/detail/Config.cpp b/src/xrpld/core/detail/Config.cpp index 34ae49c35..46f15a664 100644 --- a/src/xrpld/core/detail/Config.cpp +++ b/src/xrpld/core/detail/Config.cpp @@ -1077,6 +1077,57 @@ Config::loadFromString(std::string const& fileContents) } } } + + // DatabasePinned validation (applies to all modes including standalone) + auto db_section = section(ConfigSection::nodeDatabase()); + if (db_section.exists("pinned_type")) + { + // Ensure base type is specified + if (!db_section.exists("type")) + { + Throw( + "type must be specified when using pinned_type"); + } + + // Ensure pinned_path is specified for filesystem-backed types. + // In-memory backends (rwdb) don't need a path. + auto pinnedTypeForPath = get(db_section, "pinned_type", ""); + boost::algorithm::to_lower(pinnedTypeForPath); + bool const needsPath = + pinnedTypeForPath != "rwdb" && pinnedTypeForPath != "memory"; + if (needsPath && !db_section.exists("pinned_path")) + { + Throw( + "pinned_path is required when pinned_type is set"); + } + + // Ensure online_delete is set (pinning without deletion is + // redundant) + if (auto delete_interval = get(db_section, "online_delete", 0); + delete_interval == 0) + { + Throw( + "pinned_type requires online_delete (pinning without " + "deletion is redundant)"); + } + + // In non-standalone mode, require a durable backend. + // Standalone mode is unrestricted (tests use rwdb, etc). + if (!RUN_STANDALONE) + { + auto pinnedType = get(db_section, "pinned_type", ""); + boost::algorithm::to_lower(pinnedType); + if (pinnedType != "nudb" && pinnedType != "rocksdb") + { + Throw( + "pinned_type must be a durable backend (NuDB or " + "RocksDB), got '" + + pinnedType + + "'. Non-durable backends lose pinned data on " + "restart."); + } + } + } } boost::filesystem::path diff --git a/src/xrpld/nodestore/NodeObject.h b/src/xrpld/nodestore/NodeObject.h index f5a1fdb07..c88e99d9a 100644 --- a/src/xrpld/nodestore/NodeObject.h +++ b/src/xrpld/nodestore/NodeObject.h @@ -34,9 +34,43 @@ enum NodeObjectType : std::uint32_t { hotLEDGER = 1, hotACCOUNT_NODE = 3, hotTRANSACTION_NODE = 4, - hotDUMMY = 512 // an invalid or missing object + hotDUMMY = 512, // an invalid or missing object + + // Uncached variants - these bypass the cache when stored. + // These are routing-only values that MUST be reduced to their hot + // equivalents before serialization (on-disk format is a single byte). + pinnedACCOUNT_NODE = 1003, + pinnedTRANSACTION_NODE = 1004, + pinnedLEDGER = 1005 }; +/** Returns true if the type is a pinned (routing-only) variant. */ +inline bool +isPinnedType(NodeObjectType type) +{ + return type == pinnedACCOUNT_NODE || type == pinnedTRANSACTION_NODE || + type == pinnedLEDGER; +} + +/** Map pinned types back to their serializable hot equivalents. + Returns the type unchanged if it is not a pinned variant. +*/ +inline NodeObjectType +toHotType(NodeObjectType type) +{ + switch (type) + { + case pinnedACCOUNT_NODE: + return hotACCOUNT_NODE; + case pinnedTRANSACTION_NODE: + return hotTRANSACTION_NODE; + case pinnedLEDGER: + return hotLEDGER; + default: + return type; + } +} + /** A simple object that the Ledger uses to store entries. NodeObjects are comprised of a type, a hash, and a blob. They can be uniquely identified by the hash, which is a half-SHA512 of diff --git a/src/xrpld/nodestore/detail/DatabaseNodeImp.cpp b/src/xrpld/nodestore/detail/DatabaseNodeImp.cpp index 85e5d3c0d..0264fc17c 100644 --- a/src/xrpld/nodestore/detail/DatabaseNodeImp.cpp +++ b/src/xrpld/nodestore/detail/DatabaseNodeImp.cpp @@ -32,9 +32,16 @@ DatabaseNodeImp::store( { storeStats(1, data.size()); + // Pinned types bypass the cache and get reduced to hot equivalents + bool skipCache = isPinnedType(type); + if (skipCache) + type = toHotType(type); + auto obj = NodeObject::createObject(type, std::move(data), hash); backend_->store(obj); - if (cache_) + + // Only add to cache if it's not an uncached type + if (cache_ && !skipCache) { // After the store, replace a negative cache entry if there is one cache_->canonicalize( diff --git a/src/xrpld/nodestore/detail/DatabaseNodeImp.h b/src/xrpld/nodestore/detail/DatabaseNodeImp.h index 881d8a67c..49074110d 100644 --- a/src/xrpld/nodestore/detail/DatabaseNodeImp.h +++ b/src/xrpld/nodestore/detail/DatabaseNodeImp.h @@ -26,7 +26,6 @@ namespace ripple { namespace NodeStore { - class DatabaseNodeImp : public Database { public: diff --git a/src/xrpld/nodestore/detail/DatabasePinnedImp.cpp b/src/xrpld/nodestore/detail/DatabasePinnedImp.cpp new file mode 100644 index 000000000..040554b1b --- /dev/null +++ b/src/xrpld/nodestore/detail/DatabasePinnedImp.cpp @@ -0,0 +1,272 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2025 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include +#include +#include + +namespace ripple { +namespace NodeStore { + +DatabasePinnedImp::DatabasePinnedImp( + Application& app, + Scheduler& scheduler, + int readThreads, + std::shared_ptr writableBackend, + std::shared_ptr archiveBackend, + std::shared_ptr persistent, + Section const& config, + beast::Journal j) + : DatabaseRotating(scheduler, readThreads, config, j) + , app_(app) + , rotating_( + app, + scheduler, + readThreads, + std::move(writableBackend), + std::move(archiveBackend), + config, + j) + , persistent_(std::move(persistent)) +{ + // Update fdRequired to include all backends + fdRequired_ = rotating_.fdRequired(); + if (persistent_) + fdRequired_ += persistent_->fdRequired(); +} + +void +DatabasePinnedImp::store( + NodeObjectType type, + Blob&& data, + uint256 const& hash, + std::uint32_t ledgerSeq) +{ + // Route based on type + if (isPinnedType(type)) + { + // Reduce to serializable hot type before storage + type = toHotType(type); + + // Pinned types go to persistent storage + auto count = ++pinnedStoreCount_; + if (count % 1000 == 0) + { + JLOG(j_.trace()) + << "Pinned stores: " << count << " (type=" << type << ")"; + } + + auto nObj = NodeObject::createObject(type, std::move(data), hash); + persistent_->store(nObj); + storeStats(1, nObj->getData().size()); + } + else + { + // Hot types go through rotating storage + auto count = ++hotStoreCount_; + if (count % 10000 == 0) + { + JLOG(j_.trace()) + << "Hot stores: " << count << " (type=" << type << ")"; + } + rotating_.store(type, std::move(data), hash, ledgerSeq); + } +} + +std::shared_ptr +DatabasePinnedImp::fetchNodeObject( + uint256 const& hash, + std::uint32_t ledgerSeq, + FetchReport& fetchReport, + bool duplicate) +{ + // Optimization: Use ledgerSeq to predict backend order + // Most callers (13/18) provide reliable ledgerSeq: SHAMap node fetches, + // ledger headers, tx metadata, ledger copying operations. For these cases, + // checking the likely backend first reduces unnecessary lookups. + // When ledgerSeq=0 (unknown), falls back to default order. + // IMPORTANT: Always checks both backends for correctness. + if (ledgerSeq != 0) + { + refreshPinnedRangesCache(); + + if (likelyPinned(ledgerSeq)) + { + // Try persistent backend first (pinned data) + if (auto obj = tryPersistent(hash, fetchReport)) + return obj; + + // Fallback: try rotating backends + return rotating_.fetchNodeObject( + hash, ledgerSeq, fetchReport, duplicate); + } + else + { + // Not pinned: fall through to default path below + } + } + + // Default path: try rotating backends first (hot data) + if (auto obj = + rotating_.fetchNodeObject(hash, ledgerSeq, fetchReport, duplicate)) + return obj; + + // Fallback: try persistent backend (pinned data) + return tryPersistent(hash, fetchReport); +} + +void +DatabasePinnedImp::rotate( + std::unique_ptr&& newBackend, + std::function const& f) +{ + // Delegate to the inner rotating database. Pinned data lives in + // persistent_ and is never touched by rotation — only hot data + // in rotating_ participates in the rotate/archive/delete cycle. + rotating_.rotate(std::move(newBackend), f); +} + +std::string +DatabasePinnedImp::getName() const +{ + return "Pinned:" + rotating_.getName() + "+" + persistent_->getName(); +} + +std::int32_t +DatabasePinnedImp::getWriteLoad() const +{ + return rotating_.getWriteLoad() + persistent_->getWriteLoad(); +} + +void +DatabasePinnedImp::importDatabase(Database& source) +{ + // Import is not supported with DatabasePinned. The dual-backend + // routing relies on knowing which ledger ranges are pinned, which + // is only possible with catalogue packs where the ranges are known + // quantities. A generic import would need to scan for ledger + // headers and walk SHAMap trees to determine routing — not + // impossible, but not currently implemented. + Throw( + "--import is not supported when [node_db] pinned_type is configured. " + "DatabasePinned uses separate rotating and persistent backends, and a " + "generic node import cannot determine pinned ledger ranges or update " + "state.db pinned range metadata. Use catalogue_load for pinned " + "history import, or run --import with a non-pinned node store."); +} + +bool +DatabasePinnedImp::isSameDB(std::uint32_t s1, std::uint32_t s2) +{ + // Used by async read threads to determine if a fetched object can + // be reused for requests at different sequences. Since rotating and + // persistent are both part of the same logical database (same hash + // space), objects are always reusable regardless of which backend + // they came from. + return true; +} + +void +DatabasePinnedImp::sync() +{ + rotating_.sync(); + persistent_->sync(); +} + +bool +DatabasePinnedImp::storeLedger(std::shared_ptr const& srcLedger) +{ + // Store to persistent since ledger headers should be pinned + return Database::storeLedger(*srcLedger, persistent_); +} + +void +DatabasePinnedImp::sweep() +{ + // Delegate to rotating - persistent has no cache + rotating_.sweep(); +} + +void +DatabasePinnedImp::for_each(std::function)> f) +{ + // Visit both rotating and persistent backends + rotating_.for_each(f); + persistent_->for_each(f); +} + +void +DatabasePinnedImp::refreshPinnedRangesCache() const +{ + using namespace std::chrono; + + auto now = steady_clock::now(); + auto lastRefreshTime = + steady_clock::time_point(steady_clock::duration(lastRefresh_.load())); + + // Only refresh if cache is stale (>1 second old) + if (now - lastRefreshTime < seconds(1)) + return; + + // Create new RangeSet snapshot from LedgerMaster + auto newRanges = std::make_shared>( + app_.getLedgerMaster().getPinnedLedgersRangeSet()); + + // Atomic swap - all concurrent readers see old or new, never torn state + std::atomic_store(&cachedPinnedRanges_, newRanges); + lastRefresh_.store(now.time_since_epoch().count()); +} + +bool +DatabasePinnedImp::likelyPinned(std::uint32_t ledgerSeq) const +{ + auto ranges = std::atomic_load(&cachedPinnedRanges_); + return ranges && boost::icl::contains(*ranges, ledgerSeq); +} + +std::shared_ptr +DatabasePinnedImp::tryPersistent(uint256 const& hash, FetchReport& fetchReport) +{ + std::shared_ptr nodeObject; + Status status; + try + { + status = persistent_->fetch(hash.data(), &nodeObject); + } + catch (std::exception const& e) + { + JLOG(j_.fatal()) << "Exception fetching from persistent: " << e.what(); + Rethrow(); + } + + if (status == ok && nodeObject) + { + fetchReport.wasFound = true; + // Note: We do NOT copy pinned data to rotating storage even if + // duplicate=true. Pinned data stays in persistent storage. + } + return nodeObject; +} + +} // namespace NodeStore +} // namespace ripple diff --git a/src/xrpld/nodestore/detail/DatabasePinnedImp.h b/src/xrpld/nodestore/detail/DatabasePinnedImp.h new file mode 100644 index 000000000..107b20843 --- /dev/null +++ b/src/xrpld/nodestore/detail/DatabasePinnedImp.h @@ -0,0 +1,154 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2025 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_DATABASEPINNEDIMP_H_INCLUDED +#define RIPPLE_NODESTORE_DATABASEPINNEDIMP_H_INCLUDED + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ripple { +namespace NodeStore { + +/** + * DatabasePinned routes nodes to either rotating memory storage (via + * DatabaseRotatingImp) or persistent storage (NuDB) based on NodeObjectType + * values. + * + * Uses composition: DatabaseRotatingImp handles all rotation logic for hot + * nodes, while this class adds a persistent layer for pinned nodes. + * + * Pinned types (pinnedACCOUNT_NODE, pinnedTRANSACTION_NODE, pinnedLEDGER) go to + * persistent storage and stay forever. Hot types go through the normal + * rotation. + */ +class DatabasePinnedImp : public DatabaseRotating +{ +private: + Application& app_; // For accessing LedgerMaster's pinned ranges + DatabaseRotatingImp rotating_; // Handles rotation for hot nodes + std::shared_ptr persistent_; // NuDB for pinned nodes + + // Debug counters for store routing + std::atomic pinnedStoreCount_{0}; + std::atomic hotStoreCount_{0}; + + // Lock-free cached pinned ranges for backend selection hint + // Use std::atomic_load/store free functions for thread-safe access + mutable std::shared_ptr> cachedPinnedRanges_; + mutable std::atomic + lastRefresh_{0}; + +public: + static constexpr auto JournalName = "DatabasePinned"; + + DatabasePinnedImp( + Application& app, + Scheduler& scheduler, + int readThreads, + std::shared_ptr writableBackend, + std::shared_ptr archiveBackend, + std::shared_ptr persistent, + Section const& config, + beast::Journal j); + + ~DatabasePinnedImp() override + { + stop(); + } + + // Total count of store() calls routed to the persistent backend + // (pinned types). Exposed for tests and diagnostics. + uint64_t + pinnedStoreCount() const + { + return pinnedStoreCount_.load(std::memory_order_relaxed); + } + + // Total count of store() calls routed to the rotating backend + // (hot types). Exposed for tests and diagnostics. + uint64_t + hotStoreCount() const + { + return hotStoreCount_.load(std::memory_order_relaxed); + } + + // DatabaseRotating interface - delegates to rotating_ + void + rotate( + std::unique_ptr&& newBackend, + std::function const& f) override; + + // Database interface implementation + std::string + getName() const override; + std::int32_t + getWriteLoad() const override; + void + importDatabase(Database& source) override; + bool + isSameDB(std::uint32_t s1, std::uint32_t s2) override; + void + store( + NodeObjectType type, + Blob&& data, + uint256 const& hash, + std::uint32_t ledgerSeq) override; + void + sync() override; + bool + storeLedger(std::shared_ptr const& srcLedger) override; + void + sweep() override; + +private: + std::shared_ptr + fetchNodeObject( + uint256 const& hash, + std::uint32_t ledgerSeq, + FetchReport& fetchReport, + bool duplicate) override; + + // Helper methods for backend selection optimization + void + refreshPinnedRangesCache() const; + + bool + likelyPinned(std::uint32_t ledgerSeq) const; + + std::shared_ptr + tryPersistent(uint256 const& hash, FetchReport& fetchReport); + + void + for_each(std::function)> f) override; +}; + +} // namespace NodeStore +} // namespace ripple + +#endif \ No newline at end of file diff --git a/src/xrpld/nodestore/detail/DatabaseRotatingImp.cpp b/src/xrpld/nodestore/detail/DatabaseRotatingImp.cpp index 1499976d0..e194e29a9 100644 --- a/src/xrpld/nodestore/detail/DatabaseRotatingImp.cpp +++ b/src/xrpld/nodestore/detail/DatabaseRotatingImp.cpp @@ -61,57 +61,7 @@ DatabaseRotatingImp::rotate( { std::lock_guard lock(mutex_); - // Before rotating, ensure all pinned ledgers are in the writable - // backend - JLOG(j_.info()) - << "Ensuring pinned ledgers are preserved before backend rotation"; - - // Use a lambda to handle the preservation of pinned ledgers - auto ensurePinnedLedgersInWritable = [this]() { - // Get list of pinned ledgers - auto pinnedLedgers = - app_.getLedgerMaster().getPinnedLedgersRangeSet(); - - for (auto const& range : pinnedLedgers) - { - for (auto seq = range.lower(); seq <= range.upper(); ++seq) - { - uint256 hash = app_.getLedgerMaster().getHashBySeq(seq); - if (hash.isZero()) - continue; - - // Try to load the ledger - auto ledger = app_.getLedgerMaster().getLedgerByHash(hash); - if (ledger && ledger->isImmutable()) - { - // If we have the ledger, store it in the writable - // backend - JLOG(j_.debug()) << "Ensuring pinned ledger " << seq - << " is in writable backend"; - // TQ: TODO: check this - Database::storeLedger(*ledger, writableBackend_); - } - else - { - // If we don't have the ledger in memory, try to fetch - // its objects directly - JLOG(j_.debug()) - << "Attempting to copy pinned ledger " << seq - << " header to writable backend"; - std::shared_ptr headerObj; - Status status = - archiveBackend_->fetch(hash.data(), &headerObj); - if (status == ok && headerObj) - writableBackend_->store(headerObj); - } - } - } - }; - - // Execute the lambda - ensurePinnedLedgersInWritable(); - - // Now it's safe to mark the archive backend for deletion + // Mark the archive backend for deletion archiveBackend_->setDeletePath(); oldArchiveBackend = std::move(archiveBackend_); diff --git a/src/xrpld/nodestore/detail/DatabaseRotatingImp.h b/src/xrpld/nodestore/detail/DatabaseRotatingImp.h index 9052de07e..ba8b6754a 100644 --- a/src/xrpld/nodestore/detail/DatabaseRotatingImp.h +++ b/src/xrpld/nodestore/detail/DatabaseRotatingImp.h @@ -29,6 +29,9 @@ namespace NodeStore { class DatabaseRotatingImp : public DatabaseRotating { + friend class DatabasePinnedImp; // Allow DatabasePinnedImp to access + // private members + public: DatabaseRotatingImp() = delete; DatabaseRotatingImp(DatabaseRotatingImp const&) = delete; diff --git a/src/xrpld/rpc/detail/CatalogueStream.cpp b/src/xrpld/rpc/detail/CatalogueStream.cpp new file mode 100644 index 000000000..45cd1018a --- /dev/null +++ b/src/xrpld/rpc/detail/CatalogueStream.cpp @@ -0,0 +1,363 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012-2014 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include + +#include + +#include +#include +#include +#include +#include + +namespace ripple { +namespace RPC { + +namespace { + +std::size_t +serializeSHAMapToStream( + SHAMap const& shaMap, + CatalogueOutputStream& stream, + SHAMapNodeType nodeType, + std::optional> baseSHAMap = + std::nullopt) +{ + using StreamState = + std::pair; + + static std::mutex streamStateMutex; + static std::unordered_map streamBytesWritten; + + constexpr std::uint64_t flushThreshold = 256 * 1024 * 1024; + std::uint64_t localBytesWritten = 0; + + auto tryFlush = [](auto& s) { + if constexpr (requires(decltype(s) str) { str.flush(); }) + { + s.flush(); + } + }; + + { + std::lock_guard lock(streamStateMutex); + auto const now = std::chrono::steady_clock::now(); + + if (auto const it = + streamBytesWritten.find(static_cast(&stream)); + it != streamBytesWritten.end()) + { + localBytesWritten = it->second.first; + } + + for (auto it = streamBytesWritten.begin(); + it != streamBytesWritten.end();) + { + if (now - it->second.second > std::chrono::minutes(5)) + it = streamBytesWritten.erase(it); + else + ++it; + } + } + + auto serializeLeaf = [&stream, &localBytesWritten, &tryFlush]( + SHAMapItem const& item, + SHAMapNodeType type) -> bool { + stream.write(reinterpret_cast(&type), 1); + localBytesWritten += 1; + + auto const key = item.key(); + stream.write(reinterpret_cast(key.data()), 32); + localBytesWritten += 32; + + auto const data = item.slice(); + std::uint32_t size = data.size(); + stream.write(reinterpret_cast(&size), 4); + localBytesWritten += 4; + + stream.write(reinterpret_cast(data.data()), size); + localBytesWritten += size; + + if (localBytesWritten >= flushThreshold) + { + tryFlush(stream); + localBytesWritten = 0; + } + + return !stream.fail(); + }; + + auto serializeRemovedLeaf = + [&stream, &localBytesWritten, &tryFlush](uint256 const& key) -> bool { + auto t = SHAMapNodeType::tnREMOVE; + stream.write(reinterpret_cast(&t), 1); + localBytesWritten += 1; + + stream.write(reinterpret_cast(key.data()), 32); + localBytesWritten += 32; + + if (localBytesWritten >= flushThreshold) + { + tryFlush(stream); + localBytesWritten = 0; + } + + return !stream.fail(); + }; + + std::size_t nodeCount = 0; + + if (baseSHAMap) + { + auto const& baseMap = baseSHAMap->get(); + + if (shaMap.getHash() != baseMap.getHash()) + { + SHAMap::Delta differences; + + if (shaMap.compare( + baseMap, differences, std::numeric_limits::max())) + { + for (auto const& [key, deltaItem] : differences) + { + auto const& newItem = deltaItem.first; + auto const& oldItem = deltaItem.second; + + if (!oldItem && newItem) + { + if (serializeLeaf(*newItem, nodeType)) + ++nodeCount; + } + else if (oldItem && !newItem) + { + if (serializeRemovedLeaf(key)) + ++nodeCount; + } + else if ( + oldItem && newItem && + oldItem->slice() != newItem->slice()) + { + if (serializeLeaf(*newItem, nodeType)) + ++nodeCount; + } + } + + auto t = SHAMapNodeType::tnTERMINAL; + stream.write(reinterpret_cast(&t), 1); + localBytesWritten += 1; + + if (localBytesWritten >= flushThreshold) + { + tryFlush(stream); + localBytesWritten = 0; + } + + std::lock_guard lock(streamStateMutex); + streamBytesWritten[static_cast(&stream)] = { + localBytesWritten, std::chrono::steady_clock::now()}; + + return nodeCount; + } + } + else + { + return 0; + } + } + + for (auto const& item : shaMap) + { + if (serializeLeaf(item, nodeType)) + ++nodeCount; + } + + auto t = SHAMapNodeType::tnTERMINAL; + stream.write(reinterpret_cast(&t), 1); + localBytesWritten += 1; + + if (localBytesWritten >= flushThreshold) + { + tryFlush(stream); + localBytesWritten = 0; + } + + { + std::lock_guard lock(streamStateMutex); + streamBytesWritten[static_cast(&stream)] = { + localBytesWritten, std::chrono::steady_clock::now()}; + } + + return nodeCount; +} + +bool +deserializeSHAMapFromStream( + SHAMap& shaMap, + CatalogueInputStream& stream, + SHAMapNodeType nodeType, + NodeObjectType flushType, + bool allowRemoval, + beast::Journal const& j) +{ + try + { + auto deserializeLeaf = [&shaMap, &stream, &j, nodeType, allowRemoval]( + SHAMapNodeType& parsedType) -> bool { + stream.read(reinterpret_cast(&parsedType), 1); + + if (parsedType == SHAMapNodeType::tnTERMINAL) + return false; + + uint256 key; + std::uint32_t size{0}; + + stream.read(reinterpret_cast(key.data()), 32); + + if (stream.fail()) + { + JLOG(j.error()) + << "Deserialization: stream stopped unexpectedly " + << "while trying to read key of next entry"; + return false; + } + + if (parsedType == SHAMapNodeType::tnREMOVE) + { + if (!allowRemoval) + { + JLOG(j.error()) + << "Deserialization: unexpected removal in this map " + "type"; + return false; + } + + if (!shaMap.hasItem(key)) + { + JLOG(j.error()) + << "Deserialization: removal of key " << to_string(key) + << " but key is already absent."; + return false; + } + + shaMap.delItem(key); + return true; + } + + stream.read(reinterpret_cast(&size), 4); + + if (stream.fail()) + { + JLOG(j.error()) + << "Deserialization: stream stopped unexpectedly while " + << "trying to read size of data for key " << to_string(key); + return false; + } + + if (size > 1024 * 1024 * 1024) + { + JLOG(j.error()) + << "Deserialization: size of " << to_string(key) + << " is suspiciously large (" << size << " bytes), " + << "bailing."; + return false; + } + + std::vector data(size); + stream.read(reinterpret_cast(data.data()), size); + + if (stream.fail()) + { + JLOG(j.error()) + << "Deserialization: Unexpected EOF while reading data " + << "for " << to_string(key); + return false; + } + + auto item = make_shamapitem(key, makeSlice(data)); + if (shaMap.hasItem(key)) + return shaMap.updateGiveItem(nodeType, std::move(item)); + + return shaMap.addGiveItem(nodeType, std::move(item)); + }; + + auto lastParsed = SHAMapNodeType::tnREMOVE; + while (!stream.eof() && deserializeLeaf(lastParsed)) + ; + + if (lastParsed != SHAMapNodeType::tnTERMINAL) + { + JLOG(j.error()) + << "Deserialization: Unexpected EOF, terminal node not found."; + return false; + } + + shaMap.flushDirty(flushType); + return true; + } + catch (std::exception const& e) + { + JLOG(j.error()) << "Exception during deserialization: " << e.what(); + return false; + } +} + +} // namespace + +std::size_t +serializeStateMapToStream( + SHAMap const& stateMap, + CatalogueOutputStream& stream, + std::optional> prevStateMap) +{ + return serializeSHAMapToStream( + stateMap, stream, SHAMapNodeType::tnACCOUNT_STATE, prevStateMap); +} + +std::size_t +serializeTxMapToStream(SHAMap const& txMap, CatalogueOutputStream& stream) +{ + return serializeSHAMapToStream( + txMap, stream, SHAMapNodeType::tnTRANSACTION_MD); +} + +bool +deserializeStateMapFromStream( + SHAMap& stateMap, + CatalogueInputStream& stream, + NodeObjectType flushType, + beast::Journal const& j) +{ + return deserializeSHAMapFromStream( + stateMap, stream, SHAMapNodeType::tnACCOUNT_STATE, flushType, true, j); +} + +bool +deserializeTxMapFromStream( + SHAMap& txMap, + CatalogueInputStream& stream, + NodeObjectType flushType, + beast::Journal const& j) +{ + return deserializeSHAMapFromStream( + txMap, stream, SHAMapNodeType::tnTRANSACTION_MD, flushType, false, j); +} + +} // namespace RPC +} // namespace ripple diff --git a/src/xrpld/rpc/detail/CatalogueStream.h b/src/xrpld/rpc/detail/CatalogueStream.h new file mode 100644 index 000000000..7b19f6485 --- /dev/null +++ b/src/xrpld/rpc/detail/CatalogueStream.h @@ -0,0 +1,65 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012-2014 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_RPC_CATALOGUESTREAM_H_INCLUDED +#define RIPPLE_RPC_CATALOGUESTREAM_H_INCLUDED + +#include +#include +#include + +#include + +#include +#include + +namespace ripple { +namespace RPC { + +using CatalogueInputStream = boost::iostreams::filtering_istream; +using CatalogueOutputStream = boost::iostreams::filtering_ostream; + +std::size_t +serializeStateMapToStream( + SHAMap const& stateMap, + CatalogueOutputStream& stream, + std::optional> prevStateMap = + std::nullopt); + +std::size_t +serializeTxMapToStream(SHAMap const& txMap, CatalogueOutputStream& stream); + +bool +deserializeStateMapFromStream( + SHAMap& stateMap, + CatalogueInputStream& stream, + NodeObjectType flushType, + beast::Journal const& j); + +bool +deserializeTxMapFromStream( + SHAMap& txMap, + CatalogueInputStream& stream, + NodeObjectType flushType, + beast::Journal const& j); + +} // namespace RPC +} // namespace ripple + +#endif diff --git a/src/xrpld/rpc/handlers/Catalogue.cpp b/src/xrpld/rpc/handlers/Catalogue.cpp index 57a59550d..2b4d34b72 100644 --- a/src/xrpld/rpc/handlers/Catalogue.cpp +++ b/src/xrpld/rpc/handlers/Catalogue.cpp @@ -21,10 +21,13 @@ #include #include #include +#include +#include #include #include #include #include +#include #include #include #include @@ -678,10 +681,10 @@ doCatalogueCreate(RPC::JsonContext& context) return false; } - size_t stateNodesWritten = - ledger->stateMap().serializeToStream(*compStream, prevStateMap); + size_t stateNodesWritten = RPC::serializeStateMapToStream( + ledger->stateMap(), *compStream, prevStateMap); size_t txNodesWritten = - ledger->txMap().serializeToStream(*compStream); + RPC::serializeTxMapToStream(ledger->txMap(), *compStream); predictor.addLedger(info.seq, byteCounter.getBytesWritten()); @@ -910,6 +913,21 @@ doCatalogueLoad(RPC::JsonContext& context) } } opCleanup; + // Reject if DatabasePinned is not configured. Without it, loaded + // data lands in rotating storage and will be rotated away. + { + auto const& nscfg = + context.app.config().section(ConfigSection::nodeDatabase()); + if (!nscfg.exists("pinned_type")) + { + return rpcError( + rpcINVALID_PARAMS, + "catalogue_load requires [node_db] pinned_type to be " + "configured. Without it, loaded data will be lost on " + "the next database rotation."); + } + } + if (!context.params.isMember(jss::input_file)) return rpcError(rpcINVALID_PARAMS, "expected input_file"); @@ -971,6 +989,10 @@ doCatalogueLoad(RPC::JsonContext& context) uint8_t version = getCatalogueVersion(header.version); uint8_t compressionLevel = getCompressionLevel(header.version); + if (header.min_ledger > header.max_ledger) + return rpcError( + rpcINVALID_PARAMS, "catalogue min_ledger must be <= max_ledger"); + // Initialize status tracking { std::unique_lock writeLock(catalogueStatusMutex); @@ -1191,7 +1213,11 @@ doCatalogueLoad(RPC::JsonContext& context) ledger->setLedgerInfo(info); // Deserialize the complete state map from leaf nodes - if (!ledger->stateMap().deserializeFromStream(*decompStream)) + if (!RPC::deserializeStateMapFromStream( + ledger->stateMap(), + *decompStream, + pinnedACCOUNT_NODE, + context.j)) { JLOG(context.j.error()) << "Failed to deserialize base ledger state"; @@ -1217,7 +1243,11 @@ doCatalogueLoad(RPC::JsonContext& context) *snapshot); // Apply delta (only leaf-node changes) - if (!ledger->stateMap().deserializeFromStream(*decompStream)) + if (!RPC::deserializeStateMapFromStream( + ledger->stateMap(), + *decompStream, + pinnedACCOUNT_NODE, + context.j)) { JLOG(context.j.error()) << "Failed to apply delta to ledger " << info.seq; @@ -1226,17 +1256,17 @@ doCatalogueLoad(RPC::JsonContext& context) } // pull in the tx map - if (!ledger->txMap().deserializeFromStream(*decompStream)) + if (!RPC::deserializeTxMapFromStream( + ledger->txMap(), + *decompStream, + pinnedTRANSACTION_NODE, + context.j)) { JLOG(context.j.error()) << "Failed to apply delta to ledger " << info.seq; return rpcError(rpcINTERNAL, "Failed to apply ledger delta"); } - // Finalize the ledger - ledger->stateMap().flushDirty(hotACCOUNT_NODE); - ledger->txMap().flushDirty(hotTRANSACTION_NODE); - ledger->setAccepted( info.closeTime, info.closeTimeResolution, @@ -1260,12 +1290,84 @@ doCatalogueLoad(RPC::JsonContext& context) rpcINTERNAL, "Catalogue file contains a corrupted ledger."); } - // Save in database - pendSaveValidated(context.app, ledger, false, false); - - // Store in ledger master + // IMPORTANT: Mark as pinned BEFORE saving to database. + // This ensures isPinned() returns true when saveValidatedLedger + // checks, which: (a) routes to persistent backend via pinnedLEDGER + // type, and (b) skips AcceptedLedgerCache to avoid memory bloat. context.app.getLedgerMaster().storeLedger(ledger, true); + // Scope guard: un-pin if we exit without a successful save. + // Dismissed on success below. + bool saveDone = false; + auto unpinGuard = [&]() { + if (!saveDone) + context.app.getLedgerMaster().unpinLedger(ledger->info().seq); + }; + // Use a simple RAII wrapper to guarantee the guard runs + struct OnExit + { + std::function fn; + ~OnExit() + { + fn(); + } + } unpinOnExit{unpinGuard}; + + // Save in database - wait for completion to avoid memory bloat. + // Uses pendSaveValidated to respect job queue tuning on live + // servers (jtPUBOLDLEDGER), runs synchronously in standalone. + { + auto savePromise = std::make_shared>(); + auto saveFuture = savePromise->get_future(); + + bool queued = pendSaveValidated( + context.app, + ledger, + context.app.config().standalone(), + false, + [savePromise](bool success) { + savePromise->set_value(success); + }); + + if (!queued) + return rpcError(rpcINTERNAL, "Failed to save ledger"); + + // Wait for the async save to complete. Note: if the save + // job throws, the JobQueue has no exception handling — the + // process will std::terminate before we ever see a + // broken_promise here. The catch is defensive in case the + // job queue gains exception handling in the future. + bool saved = false; + try + { + saved = saveFuture.get(); + } + catch (std::future_error const& e) + { + JLOG(context.j.error()) + << "Save job for ledger " << ledger->info().seq + << " failed with exception (promise broken): " << e.what(); + return rpcError( + rpcINTERNAL, + "Save job crashed for ledger " + + std::to_string(ledger->info().seq)); + } + + if (!saved) + { + JLOG(context.j.error()) << "Failed to save ledger " + << ledger->info().seq << " to SQLite"; + return rpcError( + rpcINTERNAL, + "Failed to save ledger " + + std::to_string(ledger->info().seq) + + " to SQLite database"); + } + + // Save succeeded — dismiss the guard + saveDone = true; + } + if (info.seq == header.max_ledger && context.app.getLedgerMaster().getClosedLedger()->info().seq < info.seq) @@ -1277,6 +1379,20 @@ doCatalogueLoad(RPC::JsonContext& context) context.app.getLedgerMaster().setLedgerRangePresent( header.min_ledger, info.seq, true); + // Persist pinned ranges to state.db after every ledger. + // This is a single row UPDATE, so it's cheap. + // + // DURABILITY NOTE: There is a small crash window between the + // ledger save above and this setPinnedRanges call. If the + // process crashes in that window, the ledger data is safely + // in the persistent backend (always opened based on config, + // independent of state.db), and fetchNodeObject will still + // find it via the tryPersistent fallback. However, state.db + // won't record the pinned range, so mCompleteLedgers won't + // include those seqs until catalogue_load is re-run. + context.app.getSHAMapStore().setPinnedRanges( + context.app.getLedgerMaster().getPinnedLedgersRangeSet()); + // Store the ledger prevLedger = ledger; ledgersLoaded++; diff --git a/src/xrpld/shamap/SHAMap.h b/src/xrpld/shamap/SHAMap.h index 2174181bf..e2c9bf4c0 100644 --- a/src/xrpld/shamap/SHAMap.h +++ b/src/xrpld/shamap/SHAMap.h @@ -367,35 +367,6 @@ public: void invariants() const; -public: - /** - * Serialize a SHAMap to a stream, optionally as a delta from another map - * Only leaf nodes are serialized since inner nodes can be reconstructed. - * - * @param stream The output stream to write to - * @param writtenNodes Set to track written node hashes to avoid duplicates - * @param baseSHAMap Optional base map to compute delta against - * @return Number of nodes written - */ - template - std::size_t - serializeToStream( - StreamType& stream, - std::optional> baseSHAMap = - std::nullopt) const; - - /** - * Deserialize a SHAMap from a stream - * Reconstructs the full tree from leaf nodes. - * - * @param stream The input stream to read from - * @param baseSHAMap Optional base map to apply deltas to - * @return True if deserialization succeeded - */ - template - bool - deserializeFromStream(StreamType& stream); - private: using SharedPtrNodeStack = std::stack, SHAMapNodeID>>; diff --git a/src/xrpld/shamap/detail/SHAMap.cpp b/src/xrpld/shamap/detail/SHAMap.cpp index a39c0d5e7..d45c606e6 100644 --- a/src/xrpld/shamap/detail/SHAMap.cpp +++ b/src/xrpld/shamap/detail/SHAMap.cpp @@ -1301,395 +1301,4 @@ SHAMap::invariants() const node->invariants(true); } -template -std::size_t -SHAMap::serializeToStream( - StreamType& stream, - std::optional> baseSHAMap) const -{ - // Static map to track bytes written to streams - static std::mutex streamMapMutex; - static std::unordered_map< - void*, - std::pair> - streamBytesWritten; - - // Flush threshold: 256 MiB - constexpr uint64_t flushThreshold = 256 * 1024 * 1024; - - // Local byte counter for this stream - uint64_t localBytesWritten = 0; - - // Single lambda that uses compile-time check for flush method existence - auto tryFlush = [](auto& s) { - if constexpr (requires(decltype(s) str) { str.flush(); }) - { - s.flush(); - } - // No-op if flush doesn't exist - compiler will optimize this branch out - }; - - // Get the current bytes written from the global map (with lock) - { - std::lock_guard lock(streamMapMutex); - auto it = streamBytesWritten.find(static_cast(&stream)); - if (it != streamBytesWritten.end()) - { - localBytesWritten = it->second.first; - } - - // Random cleanup of old entries (while we have the lock) - if (!streamBytesWritten.empty()) - { - auto now = std::chrono::steady_clock::now(); - size_t randomIndex = std::rand() % streamBytesWritten.size(); - auto cleanupIt = std::next(streamBytesWritten.begin(), randomIndex); - - // If entry is older than 5 minutes, remove it - if (now - cleanupIt->second.second > std::chrono::minutes(5)) - { - streamBytesWritten.erase(cleanupIt); - } - } - } - - std::unordered_set> writtenNodes; - - if (!root_) - return 0; - - std::size_t nodeCount = 0; - - auto serializeLeaf = [&stream, &localBytesWritten, &tryFlush]( - SHAMapLeafNode const& node) -> bool { - // write the node type - auto t = node.getType(); - stream.write(reinterpret_cast(&t), 1); - localBytesWritten += 1; - - // write the key - auto const key = node.peekItem()->key(); - stream.write(reinterpret_cast(key.data()), 32); - localBytesWritten += 32; - - // write the data size - auto data = node.peekItem()->slice(); - uint32_t size = data.size(); - stream.write(reinterpret_cast(&size), 4); - localBytesWritten += 4; - - // write the data - stream.write(reinterpret_cast(data.data()), size); - localBytesWritten += size; - - // Check if we should flush without locking - if (localBytesWritten >= flushThreshold) - { - tryFlush(stream); - localBytesWritten = 0; - } - - return !stream.fail(); - }; - - auto serializeRemovedLeaf = - [&stream, &localBytesWritten, &tryFlush](uint256 const& key) -> bool { - // to indicate a node is removed it is written with a removal type - auto t = SHAMapNodeType::tnREMOVE; - stream.write(reinterpret_cast(&t), 1); - localBytesWritten += 1; - - // write the key - stream.write(reinterpret_cast(key.data()), 32); - localBytesWritten += 32; - - // Check if we should flush without locking - if (localBytesWritten >= flushThreshold) - { - tryFlush(stream); - localBytesWritten = 0; - } - - return !stream.fail(); - }; - - // If we're creating a delta, first compute the differences - if (baseSHAMap && baseSHAMap->get().root_) - { - const SHAMap& baseMap = baseSHAMap->get(); - - // Only compute delta if the maps are different - if (getHash() != baseMap.getHash()) - { - Delta differences; - - if (compare(baseMap, differences, std::numeric_limits::max())) - { - // Process each difference - for (auto const& [key, deltaItem] : differences) - { - auto const& newItem = deltaItem.first; - auto const& oldItem = deltaItem.second; - - if (!oldItem && newItem) - { - // Added item - SHAMapLeafNode* leaf = findKey(key); - if (leaf && serializeLeaf(*leaf)) - ++nodeCount; - } - else if (oldItem && !newItem) - { - // Removed item - if (serializeRemovedLeaf(key)) - ++nodeCount; - } - else if ( - oldItem && newItem && - oldItem->slice() != newItem->slice()) - { - // Modified item - SHAMapLeafNode* leaf = findKey(key); - if (leaf && serializeLeaf(*leaf)) - ++nodeCount; - } - } - - // write a terminal symbol to indicate the map stream has ended - auto t = SHAMapNodeType::tnTERMINAL; - stream.write(reinterpret_cast(&t), 1); - localBytesWritten += 1; - - // Check if we should flush without locking - if (localBytesWritten >= flushThreshold) - { - tryFlush(stream); - localBytesWritten = 0; - } - - // Update the global counter at the end (with lock) - { - std::lock_guard lock(streamMapMutex); - auto& streamData = - streamBytesWritten[static_cast(&stream)]; - streamData.first = localBytesWritten; - streamData.second = std::chrono::steady_clock::now(); - } - - return nodeCount; - } - } - else - { - // Maps are identical, nothing to write - return 0; - } - } - - // Otherwise walk the entire tree and serialize all leaf nodes - std::function walkTree = - [&](SHAMapTreeNode const& node, SHAMapNodeID const& nodeID) { - if (node.isLeaf()) - { - auto const& leaf = static_cast(node); - auto const& hash = leaf.getHash(); - - // Avoid duplicates - if (writtenNodes.insert(hash).second) - { - if (serializeLeaf(leaf)) - ++nodeCount; - } - return; - } - - // It's an inner node, process its children - auto const& inner = static_cast(node); - for (int i = 0; i < branchFactor; ++i) - { - if (!inner.isEmptyBranch(i)) - { - auto const& childHash = inner.getChildHash(i); - - // Skip already written nodes - if (writtenNodes.find(childHash) != writtenNodes.end()) - continue; - - auto childNode = - descendThrow(const_cast(&inner), i); - if (childNode) - { - SHAMapNodeID childID = nodeID.getChildNodeID(i); - walkTree(*childNode, childID); - } - } - } - }; - - // Start walking from root - walkTree(*root_, SHAMapNodeID()); - - // write a terminal symbol to indicate the map stream has ended - auto t = SHAMapNodeType::tnTERMINAL; - stream.write(reinterpret_cast(&t), 1); - localBytesWritten += 1; - - // Check if we should flush one last time without locking - if (localBytesWritten >= flushThreshold) - { - tryFlush(stream); - localBytesWritten = 0; - } - - // Update the global counter at the end (with lock) - { - std::lock_guard lock(streamMapMutex); - auto& streamData = streamBytesWritten[static_cast(&stream)]; - streamData.first = localBytesWritten; - streamData.second = std::chrono::steady_clock::now(); - } - - return nodeCount; -} - -template -bool -SHAMap::deserializeFromStream(StreamType& stream) -{ - try - { - JLOG(journal_.info()) << "Deserialization: Starting to deserialize " - "from stream"; - - if (state_ != SHAMapState::Modifying && state_ != SHAMapState::Synching) - return false; - - if (!root_) - root_ = std::make_shared(cowid_); - - // Define a lambda to deserialize a leaf node - auto deserializeLeaf = - [this, &stream](SHAMapNodeType& nodeType /* out */) -> bool { - stream.read(reinterpret_cast(&nodeType), 1); - - if (nodeType == SHAMapNodeType::tnTERMINAL) - { - // end of map - return false; - } - - uint256 key; - uint32_t size{0}; - - stream.read(reinterpret_cast(key.data()), 32); - - if (stream.fail()) - { - JLOG(journal_.error()) - << "Deserialization: stream stopped unexpectedly " - << "while trying to read key of next entry"; - return false; - } - - if (nodeType == SHAMapNodeType::tnREMOVE) - { - // deletion - if (!hasItem(key)) - { - JLOG(journal_.error()) - << "Deserialization: removal of key " << to_string(key) - << " but key is already absent."; - return false; - } - delItem(key); - return true; - } - - stream.read(reinterpret_cast(&size), 4); - - if (stream.fail()) - { - JLOG(journal_.error()) - << "Deserialization: stream stopped unexpectedly" - << " while trying to read size of data for key " - << to_string(key); - return false; - } - - if (size > 1024 * 1024 * 1024) - { - JLOG(journal_.error()) - << "Deserialization: size of " << to_string(key) - << " is suspiciously large (" << size - << " bytes), bailing."; - return false; - } - - std::vector data; - data.resize(size); - - stream.read(reinterpret_cast(data.data()), size); - if (stream.fail()) - { - JLOG(journal_.error()) - << "Deserialization: Unexpected EOF while reading data for " - << to_string(key); - return false; - } - - auto item = make_shamapitem(key, makeSlice(data)); - if (hasItem(key)) - return updateGiveItem(nodeType, std::move(item)); - - return addGiveItem(nodeType, std::move(item)); - }; - - SHAMapNodeType lastParsed; - while (!stream.eof() && deserializeLeaf(lastParsed)) - ; - - if (lastParsed != SHAMapNodeType::tnTERMINAL) - { - JLOG(journal_.error()) - << "Deserialization: Unexpected EOF, terminal node not found."; - return false; - } - - // Flush any dirty nodes and update hashes - flushDirty(hotUNKNOWN); - - return true; - } - catch (std::exception const& e) - { - JLOG(journal_.error()) - << "Exception during deserialization: " << e.what(); - return false; - } -} - -// explicit instantiation of templates for rpc::Catalogue - -using FilteringInputStream = boost::iostreams::filtering_stream< - boost::iostreams::input, - char, - std::char_traits, - std::allocator, - boost::iostreams::public_>; - -template bool -SHAMap::deserializeFromStream(FilteringInputStream&); - -using FilteringOutputStream = boost::iostreams::filtering_stream< - boost::iostreams::output, - char, - std::char_traits, - std::allocator, - boost::iostreams::public_>; - -template std::size_t -SHAMap::serializeToStream( - FilteringOutputStream&, - std::optional> baseSHAMap) const; - } // namespace ripple