diff --git a/.codecov.yml b/.codecov.yml index 191144aae..6df378619 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -1,6 +1,37 @@ +codecov: + require_ci_to_pass: true + +comment: + behavior: default + layout: reach,diff,flags,tree,reach + show_carryforward_flags: false + coverage: + range: "60..80" + precision: 1 + round: nearest status: project: default: target: 60% threshold: 2% + patch: + default: + target: auto + threshold: 2% + changes: false + +github_checks: + annotations: true + +parsers: + cobertura: + partials_as_hits: true + handle_missing_conditions : true + +slack_app: false + +ignore: + - "src/test/" + - "include/xrpl/beast/test/" + - "include/xrpl/beast/unit_test/" diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index de2d90d36..18ee663b1 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,8 +1,25 @@ # This feature requires Git >= 2.24 # To use it by default in git blame: # git config blame.ignoreRevsFile .git-blame-ignore-revs +# Format first-party source according to .clang-format 50760c693510894ca368e90369b0cc2dabfd07f3 -e2384885f5f630c8f0ffe4bf21a169b433a16858 -241b9ddde9e11beb7480600fd5ed90e1ef109b21 -760f16f56835663d9286bd29294d074de26a7ba6 -0eebe6a5f4246fced516d52b83ec4e7f47373edd +# Reintroduce Clang-Format & Levelization +da1d20d6d5d862716125d60899b80fab5302954a +# Consolidate external libraries +da1d20d6d5d862716125d60899b80fab5302954a +# Rename .hpp to .h +0345a2645d0f5ad900f4fbbcaff96040d3a887fc +# Format formerly .hpp files +5a227dc719016e10045e17c9396ad401118044f1 +# Rewrite includes +e61880699997398f5a746e6c4034edc7632661f5 +# Move CMake directory (#4997) +e47b1c1b3b97c3f6d11858ee02f463596e29e7f0 +# Rearrange sources (#4997) +bfafa2bb39e562901736d656806bd700c3699a2f +# Rewrite includes (#4997) +e61880699997398f5a746e6c4034edc7632661f5 +# Recompute loops (#4997) +d25b5dcd568bb96c18e347d55fac10fe901a1bfb +# Reformat code with clang-format-18 +02749feea88ce61c1f7eeb2d61a57d8ecf07ab11 diff --git a/Builds/levelization/results/ordering.txt b/Builds/levelization/results/ordering.txt index b10a62584..86d97ce96 100644 --- a/Builds/levelization/results/ordering.txt +++ b/Builds/levelization/results/ordering.txt @@ -84,6 +84,7 @@ test.nodestore > xrpl.basics test.nodestore > xrpld.core test.nodestore > xrpld.nodestore test.nodestore > xrpld.unity +test.nodestore > xrpl.protocol test.overlay > test.jtx test.overlay > test.toplevel test.overlay > test.unit_test @@ -118,6 +119,7 @@ test.rpc > xrpld.core test.rpc > xrpld.net test.rpc > xrpld.overlay test.rpc > xrpld.rpc +test.rpc > xrpld.shamap test.rpc > xrpl.hook test.rpc > xrpl.json test.rpc > xrpl.protocol diff --git a/build-full.sh b/build-full.sh index 29b953ac3..c0e79b93b 100644 --- a/build-full.sh +++ b/build-full.sh @@ -12,17 +12,16 @@ echo "-- GITHUB_REPOSITORY: $1" echo "-- GITHUB_SHA: $2" echo "-- GITHUB_RUN_NUMBER: $4" -umask 0000; +umask 0000 #### -cd /io; -mkdir -p src/certs; -curl --silent -k https://raw.githubusercontent.com/RichardAH/rippled-release-builder/main/ca-bundle/certbundle.h -o src/certs/certbundle.h; -if [ "`grep certbundle.h src/xrpld/net/detail/RegisterSSLCerts.cpp | wc -l`" -eq "0" ] -then - cp src/xrpld/net/detail/RegisterSSLCerts.cpp src/xrpld/net/detail/RegisterSSLCerts.cpp.old - perl -i -pe "s/^{/{ +cd /io +mkdir -p src/certs +curl --silent -k https://raw.githubusercontent.com/RichardAH/rippled-release-builder/main/ca-bundle/certbundle.h -o src/certs/certbundle.h +if [ "$(grep certbundle.h src/xrpld/net/detail/RegisterSSLCerts.cpp | wc -l)" -eq "0" ]; then + cp src/xrpld/net/detail/RegisterSSLCerts.cpp src/xrpld/net/detail/RegisterSSLCerts.cpp.old + perl -i -pe "s/^{/{ #ifdef EMBEDDED_CA_BUNDLE BIO *cbio = BIO_new_mem_buf(ca_bundle.data(), ca_bundle.size()); X509_STORE *cts = SSL_CTX_get_cert_store(ctx.native_handle()); @@ -68,15 +67,14 @@ fi source /opt/rh/gcc-toolset-11/enable export PATH=/usr/local/bin:$PATH export CC='/usr/lib64/ccache/gcc' && -export CXX='/usr/lib64/ccache/g++' && -echo "-- Build Rippled --" && -pwd && + export CXX='/usr/lib64/ccache/g++' && + echo "-- Build Rippled --" && + pwd && + echo "MOVING TO [ build-core.sh ]" -echo "MOVING TO [ build-core.sh ]"; - -printenv > .env.temp; -cat .env.temp | grep '=' | sed s/\\\(^[^=]\\+=\\\)/\\1\\\"/g|sed s/\$/\\\"/g > .env; -rm .env.temp; +printenv >.env.temp +cat .env.temp | grep '=' | sed s/\\\(^[^=]\\+=\\\)/\\1\\\"/g | sed s/\$/\\\"/g >.env +rm .env.temp echo "Persisting ENV:" cat .env diff --git a/cmake/CodeCoverage.cmake b/cmake/CodeCoverage.cmake index d6323c5cc..31a2f0f4c 100644 --- a/cmake/CodeCoverage.cmake +++ b/cmake/CodeCoverage.cmake @@ -95,6 +95,9 @@ # - replace both functions setup_target_for_coverage_gcovr_* with a single setup_target_for_coverage_gcovr # - add support for all gcovr output formats # +# 2024-04-03, Bronek Kozicki +# - add support for output formats: jacoco, clover, lcov +# # USAGE: # # 1. Copy this file into your cmake modules path. @@ -256,10 +259,10 @@ endif() # BASE_DIRECTORY "../" # Base directory for report # # (defaults to PROJECT_SOURCE_DIR) # FORMAT "cobertura" # Output format, one of: -# # xml cobertura sonarqube json-summary -# # json-details coveralls csv txt -# # html-single html-nested html-details -# # (xml is an alias to cobertura; +# # xml cobertura sonarqube jacoco clover +# # json-summary json-details coveralls csv +# # txt html-single html-nested html-details +# # lcov (xml is an alias to cobertura; # # if no format is set, defaults to xml) # EXCLUDE "src/dir1/*" "src/dir2/*" # Patterns to exclude (can be relative # # to BASE_DIRECTORY, with CMake 3.4+) @@ -308,6 +311,8 @@ function(setup_target_for_coverage_gcovr) set(GCOVR_OUTPUT_FILE ${Coverage_NAME}.txt) elseif(Coverage_FORMAT STREQUAL "csv") set(GCOVR_OUTPUT_FILE ${Coverage_NAME}.csv) + elseif(Coverage_FORMAT STREQUAL "lcov") + set(GCOVR_OUTPUT_FILE ${Coverage_NAME}.lcov) else() set(GCOVR_OUTPUT_FILE ${Coverage_NAME}.xml) endif() @@ -320,6 +325,14 @@ function(setup_target_for_coverage_gcovr) set(Coverage_FORMAT cobertura) # overwrite xml elseif(Coverage_FORMAT STREQUAL "sonarqube") list(APPEND GCOVR_ADDITIONAL_ARGS --sonarqube "${GCOVR_OUTPUT_FILE}" ) + elseif(Coverage_FORMAT STREQUAL "jacoco") + list(APPEND GCOVR_ADDITIONAL_ARGS --jacoco "${GCOVR_OUTPUT_FILE}" ) + list(APPEND GCOVR_ADDITIONAL_ARGS --jacoco-pretty ) + elseif(Coverage_FORMAT STREQUAL "clover") + list(APPEND GCOVR_ADDITIONAL_ARGS --clover "${GCOVR_OUTPUT_FILE}" ) + list(APPEND GCOVR_ADDITIONAL_ARGS --clover-pretty ) + elseif(Coverage_FORMAT STREQUAL "lcov") + list(APPEND GCOVR_ADDITIONAL_ARGS --lcov "${GCOVR_OUTPUT_FILE}" ) elseif(Coverage_FORMAT STREQUAL "json-summary") list(APPEND GCOVR_ADDITIONAL_ARGS --json-summary "${GCOVR_OUTPUT_FILE}" ) list(APPEND GCOVR_ADDITIONAL_ARGS --json-summary-pretty) diff --git a/hook/extern.h b/hook/extern.h index d9c14d7a6..3399bc6b5 100644 --- a/hook/extern.h +++ b/hook/extern.h @@ -2,6 +2,9 @@ // Generated using generate_extern.sh #include #ifndef HOOK_EXTERN +#ifdef __cplusplus +extern "C" { +#endif extern int32_t __attribute__((noduplicate)) _g(uint32_t guard_id, uint32_t maxiter); @@ -336,5 +339,8 @@ prepare( uint32_t read_ptr, uint32_t read_len); +#ifdef __cplusplus +} +#endif #define HOOK_EXTERN #endif // HOOK_EXTERN diff --git a/hook/generate_error.sh b/hook/generate_error.sh index 5aa232f2b..b1e85092b 100755 --- a/hook/generate_error.sh +++ b/hook/generate_error.sh @@ -9,7 +9,7 @@ ENUM_FILE="$SCRIPT_DIR/../include/xrpl/hook/Enum.h" echo '// For documentation please see: https://xrpl-hooks.readme.io/reference/' echo '// Generated using generate_error.sh' echo '#ifndef HOOK_ERROR_CODES' -sed -n '/enum hook_return_code/,/};/p' "$ENUM_FILE" | +sed -n '/enum class hook_return_code/,/};/p' "$ENUM_FILE" | awk ' function ltrim(s) { sub(/^[[:space:]]+/, "", s); return s } function rtrim(s) { sub(/[[:space:]]+$/, "", s); return s } @@ -31,7 +31,7 @@ sed -n '/enum hook_return_code/,/};/p' "$ENUM_FILE" | { line = $0 - if (line ~ /enum[[:space:]]+hook_return_code/) + if (line ~ /enum[[:space:]]+class[[:space:]]+hook_return_code/) next if (line ~ /^[[:space:]]*\{/) next diff --git a/hook/generate_extern.sh b/hook/generate_extern.sh index f16647304..468b9aab6 100755 --- a/hook/generate_extern.sh +++ b/hook/generate_extern.sh @@ -11,6 +11,9 @@ APPLY_HOOK="$SCRIPT_DIR/../include/xrpl/hook/hook_api.macro" echo '// Generated using generate_extern.sh' echo '#include ' echo '#ifndef HOOK_EXTERN' + echo '#ifdef __cplusplus' + echo 'extern "C" {' + echo '#endif' echo awk ' function trim(s) { @@ -46,6 +49,9 @@ APPLY_HOOK="$SCRIPT_DIR/../include/xrpl/hook/hook_api.macro" } ' "$APPLY_HOOK" + echo '#ifdef __cplusplus' + echo '}' + echo '#endif' echo '#define HOOK_EXTERN' echo '#endif // HOOK_EXTERN' } | ( diff --git a/hook/sfcodes.h b/hook/sfcodes.h index 94827a7a3..3dc2a4435 100644 --- a/hook/sfcodes.h +++ b/hook/sfcodes.h @@ -189,6 +189,7 @@ #define sfSignatureReward ((6U << 16U) + 29U) #define sfMinAccountCreateAmount ((6U << 16U) + 30U) #define sfLPTokenBalance ((6U << 16U) + 31U) +#define sfTrustLineRewardAccumulator ((6U << 16U) + 99U) #define sfPublicKey ((7U << 16U) + 1U) #define sfMessageKey ((7U << 16U) + 2U) #define sfSigningPubKey ((7U << 16U) + 3U) @@ -220,6 +221,7 @@ #define sfProvider ((7U << 16U) + 30U) #define sfMPTokenMetadata ((7U << 16U) + 31U) #define sfCredentialType ((7U << 16U) + 32U) +#define sfHookName ((7U << 16U) + 97U) #define sfRemarkValue ((7U << 16U) + 98U) #define sfRemarkName ((7U << 16U) + 99U) #define sfAccount ((8U << 16U) + 1U) @@ -255,6 +257,7 @@ #define sfIssuingChainIssue ((24U << 16U) + 2U) #define sfAsset ((24U << 16U) + 3U) #define sfAsset2 ((24U << 16U) + 4U) +#define sfClaimCurrency ((24U << 16U) + 5U) #define sfXChainBridge ((25U << 16U) + 1U) #define sfTransactionMetaData ((14U << 16U) + 2U) #define sfCreatedNode ((14U << 16U) + 3U) @@ -292,6 +295,8 @@ #define sfActiveValidator ((14U << 16U) + 95U) #define sfGenesisMint ((14U << 16U) + 96U) #define sfRemark ((14U << 16U) + 97U) +#define sfHighReward ((14U << 16U) + 98U) +#define sfLowReward ((14U << 16U) + 99U) #define sfSigners ((15U << 16U) + 3U) #define sfSignerEntries ((15U << 16U) + 4U) #define sfTemplate ((15U << 16U) + 5U) diff --git a/include/xrpl/hook/Enum.h b/include/xrpl/hook/Enum.h index 29f7ed16b..9dbce0f33 100644 --- a/include/xrpl/hook/Enum.h +++ b/include/xrpl/hook/Enum.h @@ -16,6 +16,7 @@ #define featureHooksUpdate1 "1" #define featureHooksUpdate2 "1" #define fix20250131 "1" +#define fixGuardDepth32 "1" namespace hook_api { struct Rules { @@ -319,7 +320,7 @@ namespace compare_mode { enum compare_mode : uint32_t { EQUAL = 1, LESS = 2, GREATER = 4 }; } -enum hook_return_code : int64_t { +enum class hook_return_code : int64_t { SUCCESS = 0, // return codes > 0 are reserved for hook apis to return "success" OUT_OF_BOUNDS = @@ -386,7 +387,7 @@ enum hook_return_code : int64_t { TOO_MANY_NAMESPACES = -45 }; -enum ExitType : uint8_t { +enum class ExitType : uint8_t { UNSET = 0, WASM_ERROR = 1, ROLLBACK = 2, @@ -443,6 +444,7 @@ getImportWhitelist(Rules const& rules) enum GuardRulesVersion : uint64_t { GuardRuleFix20250131 = 0x00000001, + GuardRuleDepth32 = 0x00000002, }; inline uint64_t @@ -451,6 +453,8 @@ getGuardRulesVersion(Rules const& rules) uint64_t version = 0; if (rules.enabled(fix20250131)) version |= GuardRuleFix20250131; + if (rules.enabled(fixGuardDepth32)) + version |= GuardRuleDepth32; return version; } diff --git a/include/xrpl/hook/Guard.h b/include/xrpl/hook/Guard.h index 051d82128..53d9f010f 100644 --- a/include/xrpl/hook/Guard.h +++ b/include/xrpl/hook/Guard.h @@ -204,9 +204,13 @@ struct WasmBlkInf } // compute worst case execution time inline uint64_t -compute_wce(const WasmBlkInf* blk, int level, bool* recursion_limit_reached) +compute_wce( + const WasmBlkInf* blk, + int level, + int max_level, + bool* recursion_limit_reached) { - if (level > 16) + if (level > max_level) { *recursion_limit_reached = true; return 0; @@ -233,8 +237,8 @@ compute_wce(const WasmBlkInf* blk, int level, bool* recursion_limit_reached) if (blk->children.size() > 0) for (auto const& child : blk->children) - worst_case_execution += - compute_wce(child, level + 1, recursion_limit_reached); + worst_case_execution += compute_wce( + child, level + 1, max_level, recursion_limit_reached); if (parent == 0 || parent->iteration_bound == @@ -788,12 +792,17 @@ check_guard( } bool recursion_limit_reached = false; - uint64_t wce = compute_wce(&(*root), 0, &recursion_limit_reached); + int max_level = 16; + if (rulesVersion & hook_api::GuardRuleDepth32) + max_level = 32; + uint64_t wce = + compute_wce(&(*root), 0, max_level, &recursion_limit_reached); if (recursion_limit_reached) { GUARDLOG(hook::log::NESTING_LIMIT) << "GuardCheck " - << "Maximum allowable depth of blocks reached (16 levels). Flatten " + << "Maximum allowable depth of blocks reached (" << max_level + << " levels). Flatten " "your loops and conditions!.\n"; return {}; } diff --git a/include/xrpl/hook/Macro.h b/include/xrpl/hook/Macro.h index 99d1f42f7..45e6a8db9 100644 --- a/include/xrpl/hook/Macro.h +++ b/include/xrpl/hook/Macro.h @@ -89,58 +89,69 @@ #define WASM_VAL_TYPE(T, b) CAT2(TYP_, T) -#define DECLARE_HOOK_FUNCTION(R, F, ...) \ - R F(hook::HookContext& hookCtx, \ - WasmEdge_CallingFrameContext const& frameCtx __VA_OPT__( \ - COMMA __VA_ARGS__)); \ - extern WasmEdge_Result WasmFunction##F( \ - void* data_ptr, \ - const WasmEdge_CallingFrameContext* frameCtx, \ - const WasmEdge_Value* in, \ - WasmEdge_Value* out); \ - extern WasmEdge_ValType WasmFunctionParams##F[]; \ - extern WasmEdge_ValType WasmFunctionResult##F[]; \ - extern WasmEdge_FunctionTypeContext* WasmFunctionType##F; \ +#define UNSIGNED_TYPE(T) std::make_unsigned_t + +#define DECLARE_HOOK_FUNCTION(R, F, ...) \ + std::variant F( \ + hook::HookContext& hookCtx, \ + WasmEdge_CallingFrameContext const& frameCtx __VA_OPT__( \ + COMMA __VA_ARGS__)); \ + extern WasmEdge_Result WasmFunction##F( \ + void* data_ptr, \ + const WasmEdge_CallingFrameContext* frameCtx, \ + const WasmEdge_Value* in, \ + WasmEdge_Value* out); \ + extern WasmEdge_ValType WasmFunctionParams##F[]; \ + extern WasmEdge_ValType WasmFunctionResult##F[]; \ + extern WasmEdge_FunctionTypeContext* WasmFunctionType##F; \ extern WasmEdge_String WasmFunctionName##F; -#define DEFINE_HOOK_FUNCTION(R, F, ...) \ - WasmEdge_Result hook_api::WasmFunction##F( \ - void* data_ptr, \ - const WasmEdge_CallingFrameContext* frameCtx, \ - const WasmEdge_Value* in, \ - WasmEdge_Value* out) \ - { \ - __VA_OPT__(int _stack = 0;) \ - __VA_OPT__(FOR_VARS(VAR_ASSIGN, 2, __VA_ARGS__);) \ - hook::HookContext* hookCtx = \ - reinterpret_cast(data_ptr); \ - R return_code = hook_api::F( \ - *hookCtx, \ - *const_cast(frameCtx) \ - __VA_OPT__(COMMA STRIP_TYPES(__VA_ARGS__))); \ - if (return_code == RC_ROLLBACK || return_code == RC_ACCEPT) \ - return WasmEdge_Result_Terminate; \ - out[0] = RET_ASSIGN(R, return_code); \ - return WasmEdge_Result_Success; \ - }; \ - WasmEdge_ValType hook_api::WasmFunctionParams##F[] = { \ - __VA_OPT__(FOR_VARS(WASM_VAL_TYPE, 0, __VA_ARGS__))}; \ - WasmEdge_ValType hook_api::WasmFunctionResult##F[1] = { \ - WASM_VAL_TYPE(R, dummy)}; \ - WasmEdge_FunctionTypeContext* hook_api::WasmFunctionType##F = \ - WasmEdge_FunctionTypeCreate( \ - WasmFunctionParams##F, \ - VA_NARGS(NULL __VA_OPT__(, __VA_ARGS__)), \ - WasmFunctionResult##F, \ - 1); \ - WasmEdge_String hook_api::WasmFunctionName##F = \ - WasmEdge_StringCreateByCString(#F); \ - R hook_api::F( \ - hook::HookContext& hookCtx, \ - WasmEdge_CallingFrameContext const& frameCtx __VA_OPT__( \ +#define DEFINE_HOOK_FUNCTION(R, F, ...) \ + WasmEdge_Result hook_api::WasmFunction##F( \ + void* data_ptr, \ + const WasmEdge_CallingFrameContext* frameCtx, \ + const WasmEdge_Value* in, \ + WasmEdge_Value* out) \ + { \ + __VA_OPT__(int _stack = 0;) \ + __VA_OPT__(FOR_VARS(VAR_ASSIGN, 2, __VA_ARGS__);) \ + hook::HookContext* hookCtx = \ + reinterpret_cast(data_ptr); \ + auto const& return_code = hook_api::F( \ + *hookCtx, \ + *const_cast(frameCtx) \ + __VA_OPT__(COMMA STRIP_TYPES(__VA_ARGS__))); \ + if (std::holds_alternative(return_code) && \ + (std::get(return_code) == \ + RC_ROLLBACK || \ + std::get(return_code) == RC_ACCEPT)) \ + return WasmEdge_Result_Terminate; \ + out[0] = RET_ASSIGN( \ + R, \ + std::holds_alternative(return_code) \ + ? std::get(return_code) \ + : R(std::get(return_code))); \ + return WasmEdge_Result_Success; \ + }; \ + WasmEdge_ValType hook_api::WasmFunctionParams##F[] = { \ + __VA_OPT__(FOR_VARS(WASM_VAL_TYPE, 0, __VA_ARGS__))}; \ + WasmEdge_ValType hook_api::WasmFunctionResult##F[1] = { \ + WASM_VAL_TYPE(R, dummy)}; \ + WasmEdge_FunctionTypeContext* hook_api::WasmFunctionType##F = \ + WasmEdge_FunctionTypeCreate( \ + WasmFunctionParams##F, \ + VA_NARGS(NULL __VA_OPT__(, __VA_ARGS__)), \ + WasmFunctionResult##F, \ + 1); \ + WasmEdge_String hook_api::WasmFunctionName##F = \ + WasmEdge_StringCreateByCString(#F); \ + std::variant hook_api::F( \ + hook::HookContext& hookCtx, \ + WasmEdge_CallingFrameContext const& frameCtx __VA_OPT__( \ COMMA __VA_ARGS__)) #define HOOK_SETUP() \ + using enum hook_api::hook_return_code; \ try \ { \ [[maybe_unused]] ApplyContext& applyCtx = hookCtx.applyCtx; \ @@ -203,7 +214,7 @@ host_memory_ptr, \ guest_memory_length) \ { \ - int64_t bytes_written = 0; \ + uint64_t bytes_written = 0; \ WRITE_WASM_MEMORY( \ bytes_written, \ guest_dst_ptr, \ @@ -272,7 +283,7 @@ data_ptr < (data_ptr_in)) \ return INTERNAL_ERROR; \ if (data_len == 0) \ - return 0; \ + return 0ULL; \ if ((write_ptr_in) == 0) \ return data_as_int64(data_ptr, data_len); \ if (data_len > (write_len_in)) \ diff --git a/include/xrpl/protocol/Feature.h b/include/xrpl/protocol/Feature.h index d5dbd3969..617ae4709 100644 --- a/include/xrpl/protocol/Feature.h +++ b/include/xrpl/protocol/Feature.h @@ -33,35 +33,39 @@ * * Steps required to add new features to the code: * - * 1) In this file, increment `numFeatures` and add a uint256 declaration - * for the feature at the bottom - * 2) Add a uint256 definition for the feature to the corresponding source - * file (Feature.cpp). Use `registerFeature` to create the feature with - * the feature's name, `Supported::no`, and `VoteBehavior::DefaultNo`. This - * should be the only place the feature's name appears in code as a string. - * 3) Use the uint256 as the parameter to `view.rules.enabled()` to - * control flow into new code that this feature limits. - * 4) If the feature development is COMPLETE, and the feature is ready to be - * SUPPORTED, change the `registerFeature` parameter to Supported::yes. - * 5) When the feature is ready to be ENABLED, change the `registerFeature` - * parameter to `VoteBehavior::DefaultYes`. - * In general, any newly supported amendments (`Supported::yes`) should have - * a `VoteBehavior::DefaultNo` for at least one full release cycle. High - * priority bug fixes can be an exception to this rule of thumb. + * 1) Add the appropriate XRPL_FEATURE or XRPL_FIX macro definition for the + * feature to features.macro with the feature's name, `Supported::no`, and + * `VoteBehavior::DefaultNo`. + * + * 2) Use the generated variable name as the parameter to `view.rules.enabled()` + * to control flow into new code that this feature limits. (featureName or + * fixName) + * + * 3) If the feature development is COMPLETE, and the feature is ready to be + * SUPPORTED, change the macro parameter in features.macro to Supported::yes. + * + * 4) In general, any newly supported amendments (`Supported::yes`) should have + * a `VoteBehavior::DefaultNo` indefinitely so that external governance can + * make the decision on when to activate it. High priority bug fixes can be + * an exception to this rule. In such cases, ensure the fix has been + * clearly communicated to the community using appropriate channels, + * then change the macro parameter in features.macro to + * `VoteBehavior::DefaultYes`. The communication process is beyond + * the scope of these instructions. + * * * When a feature has been enabled for several years, the conditional code * may be removed, and the feature "retired". To retire a feature: - * 1) Remove the uint256 declaration from this file. - * 2) MOVE the uint256 definition in Feature.cpp to the "retired features" - * section at the end of the file. - * 3) CHANGE the name of the variable to start with "retired". - * 4) CHANGE the parameters of the `registerFeature` call to `Supported::yes` - * and `VoteBehavior::DefaultNo`. + * + * 1) MOVE the macro definition in features.macro to the "retired features" + * section at the end of the file, and change the macro to XRPL_RETIRE. + * * The feature must remain registered and supported indefinitely because it - * still exists in the ledger, but there is no need to vote for it because - * there's nothing to vote for. If it is removed completely from the code, any - * instances running that code will get amendment blocked. Removing the - * feature from the ledger is beyond the scope of these instructions. + * may exist in the Amendments object on ledger. There is no need to vote + * for it because there's nothing to vote for. If the feature definition is + * removed completely from the code, any instances running that code will get + * amendment blocked. Removing the feature from the ledger is beyond the scope + * of these instructions. * */ @@ -76,11 +80,32 @@ allAmendments(); namespace detail { +#pragma push_macro("XRPL_FEATURE") +#undef XRPL_FEATURE +#pragma push_macro("XRPL_FIX") +#undef XRPL_FIX +#pragma push_macro("XRPL_RETIRE") +#undef XRPL_RETIRE + +#define XRPL_FEATURE(name, supported, vote) +1 +#define XRPL_FIX(name, supported, vote) +1 +#define XRPL_RETIRE(name) +1 + // This value SHOULD be equal to the number of amendments registered in // Feature.cpp. Because it's only used to reserve storage, and determine how // large to make the FeatureBitset, it MAY be larger. It MUST NOT be less than // the actual number of amendments. A LogicError on startup will verify this. -static constexpr std::size_t numFeatures = 115; +static constexpr std::size_t numFeatures = + (0 + +#include + ); + +#undef XRPL_RETIRE +#pragma pop_macro("XRPL_RETIRE") +#undef XRPL_FIX +#pragma pop_macro("XRPL_FIX") +#undef XRPL_FEATURE +#pragma pop_macro("XRPL_FEATURE") /** Amendments that this server supports and the default voting behavior. Whether they are enabled depends on the Rules defined in the validated @@ -320,12 +345,17 @@ foreachFeature(FeatureBitset bs, F&& f) #undef XRPL_FEATURE #pragma push_macro("XRPL_FIX") #undef XRPL_FIX +#pragma push_macro("XRPL_RETIRE") +#undef XRPL_RETIRE #define XRPL_FEATURE(name, supported, vote) extern uint256 const feature##name; #define XRPL_FIX(name, supported, vote) extern uint256 const fix##name; +#define XRPL_RETIRE(name) #include +#undef XRPL_RETIRE +#pragma pop_macro("XRPL_RETIRE") #undef XRPL_FIX #pragma pop_macro("XRPL_FIX") #undef XRPL_FEATURE diff --git a/include/xrpl/protocol/TER.h b/include/xrpl/protocol/TER.h index c1c61166a..459c267ae 100644 --- a/include/xrpl/protocol/TER.h +++ b/include/xrpl/protocol/TER.h @@ -137,6 +137,7 @@ enum TEMcodes : TERUnderlyingType { temXCHAIN_BRIDGE_NONDOOR_OWNER, temXCHAIN_BRIDGE_BAD_MIN_ACCOUNT_CREATE_AMOUNT, temXCHAIN_BRIDGE_BAD_REWARD_AMOUNT, + temXCHAIN_TOO_MANY_ATTESTATIONS, // RESERVED - not used temHOOK_DATA_TOO_LARGE, temEMPTY_DID, diff --git a/include/xrpl/protocol/detail/features.macro b/include/xrpl/protocol/detail/features.macro index 4791025b3..43f3548e0 100644 --- a/include/xrpl/protocol/detail/features.macro +++ b/include/xrpl/protocol/detail/features.macro @@ -23,6 +23,9 @@ #if !defined(XRPL_FIX) #error "undefined macro: XRPL_FIX" #endif +#if !defined(XRPL_RETIRE) +#error "undefined macro: XRPL_RETIRE" +#endif // clang-format off @@ -31,6 +34,10 @@ // If you add an amendment here, then do not forget to increment `numFeatures` // in include/xrpl/protocol/Feature.h. +XRPL_FIX (HookMap, Supported::yes, VoteBehavior::DefaultYes) +XRPL_FIX (GuardDepth32, Supported::yes, VoteBehavior::DefaultNo) +XRPL_FEATURE(NamedHooks, Supported::yes, VoteBehavior::DefaultNo) +XRPL_FEATURE(IOURewardClaim, Supported::yes, VoteBehavior::DefaultNo) XRPL_FIX (IOULockedBalanceInvariant, Supported::yes, VoteBehavior::DefaultNo) XRPL_FIX (ImportIssuer, Supported::yes, VoteBehavior::DefaultYes) XRPL_FEATURE(HookAPISerializedType240, Supported::yes, VoteBehavior::DefaultNo) @@ -54,18 +61,18 @@ XRPL_FIX (FillOrKill, Supported::yes, VoteBehavior::DefaultYe XRPL_FEATURE(DID, Supported::no, VoteBehavior::DefaultNo) XRPL_FIX (DisallowIncomingV1, Supported::yes, VoteBehavior::DefaultYes) XRPL_FEATURE(XChainBridge, Supported::no, VoteBehavior::DefaultNo) -XRPL_FEATURE(AMM, Supported::yes, VoteBehavior::DefaultNo) +XRPL_FEATURE(AMM, Supported::no, VoteBehavior::DefaultNo) XRPL_FIX (ReducedOffersV1, Supported::yes, VoteBehavior::DefaultYes) -XRPL_FEATURE(HooksUpdate2, Supported::yes, VoteBehavior::DefaultNo); -XRPL_FEATURE(HookOnV2, Supported::yes, VoteBehavior::DefaultNo); -XRPL_FIX (HookAPI20251128, Supported::yes, VoteBehavior::DefaultYes); -XRPL_FIX (CronStacking, Supported::yes, VoteBehavior::DefaultYes); -XRPL_FEATURE(ExtendedHookState, Supported::yes, VoteBehavior::DefaultNo); -XRPL_FIX (InvalidTxFlags, Supported::yes, VoteBehavior::DefaultYes); -XRPL_FEATURE(Cron, Supported::yes, VoteBehavior::DefaultNo); -XRPL_FEATURE(IOUIssuerWeakTSH, Supported::yes, VoteBehavior::DefaultNo); -XRPL_FEATURE(DeepFreeze, Supported::yes, VoteBehavior::DefaultNo); -XRPL_FIX (ProvisionalDoubleThreading, Supported::yes, VoteBehavior::DefaultYes); +XRPL_FEATURE(HooksUpdate2, Supported::yes, VoteBehavior::DefaultNo) +XRPL_FEATURE(HookOnV2, Supported::yes, VoteBehavior::DefaultNo) +XRPL_FIX (HookAPI20251128, Supported::yes, VoteBehavior::DefaultYes) +XRPL_FIX (CronStacking, Supported::yes, VoteBehavior::DefaultYes) +XRPL_FEATURE(ExtendedHookState, Supported::yes, VoteBehavior::DefaultNo) +XRPL_FIX (InvalidTxFlags, Supported::yes, VoteBehavior::DefaultYes) +XRPL_FEATURE(Cron, Supported::yes, VoteBehavior::DefaultNo) +XRPL_FEATURE(IOUIssuerWeakTSH, Supported::yes, VoteBehavior::DefaultNo) +XRPL_FEATURE(DeepFreeze, Supported::yes, VoteBehavior::DefaultNo) +XRPL_FIX (ProvisionalDoubleThreading, Supported::yes, VoteBehavior::DefaultYes) XRPL_FEATURE(Clawback, Supported::yes, VoteBehavior::DefaultNo) XRPL_FIX (RewardClaimFlags, Supported::yes, VoteBehavior::DefaultYes) XRPL_FEATURE(HookCanEmit, Supported::yes, VoteBehavior::DefaultNo) @@ -149,4 +156,24 @@ XRPL_FIX (NFTokenDirV1, Supported::yes, VoteBehavior::Obsolete) XRPL_FEATURE(NonFungibleTokensV1, Supported::yes, VoteBehavior::Obsolete) XRPL_FEATURE(CryptoConditionsSuite, Supported::yes, VoteBehavior::Obsolete) +// The following amendments have been active for at least two years. Their +// pre-amendment code has been removed and the identifiers are deprecated. +// All known amendments and amendments that may appear in a validated +// ledger must be registered either here or above with the "active" amendments +XRPL_RETIRE(MultiSign) +XRPL_RETIRE(TrustSetAuth) +XRPL_RETIRE(FeeEscalation) +XRPL_RETIRE(PayChan) +XRPL_RETIRE(CryptoConditions) +XRPL_RETIRE(TickSize) +XRPL_RETIRE(fix1368) +XRPL_RETIRE(Escrow) +XRPL_RETIRE(fix1373) +XRPL_RETIRE(EnforceInvariants) +XRPL_RETIRE(SortedDirectories) +XRPL_RETIRE(fix1201) +XRPL_RETIRE(fix1512) +XRPL_RETIRE(fix1523) +XRPL_RETIRE(fix1528) + // clang-format on diff --git a/include/xrpl/protocol/detail/ledger_entries.macro b/include/xrpl/protocol/detail/ledger_entries.macro index 8c9266061..10075ff75 100644 --- a/include/xrpl/protocol/detail/ledger_entries.macro +++ b/include/xrpl/protocol/detail/ledger_entries.macro @@ -93,7 +93,7 @@ LEDGER_ENTRY(ltCHECK, 0x0043, Check, check, ({ */ LEDGER_ENTRY(ltHOOK_DEFINITION, 'D', HookDefinition, hook_definition, ({ {sfHookHash, soeREQUIRED}, - {sfHookOn, soeREQUIRED}, + {sfHookOn, soeOPTIONAL}, {sfHookOnIncoming, soeOPTIONAL}, {sfHookOnOutgoing, soeOPTIONAL}, {sfHookCanEmit, soeOPTIONAL}, @@ -396,6 +396,8 @@ LEDGER_ENTRY(ltRIPPLE_STATE, 0x0072, RippleState, state, ({ {sfHighQualityOut, soeOPTIONAL}, {sfLockedBalance, soeOPTIONAL}, {sfLockCount, soeOPTIONAL}, + {sfHighReward, soeOPTIONAL}, + {sfLowReward, soeOPTIONAL}, })) /** The ledger object which lists the network's fee settings. diff --git a/include/xrpl/protocol/detail/sfields.macro b/include/xrpl/protocol/detail/sfields.macro index 0bfcdbae4..dcc3138e9 100644 --- a/include/xrpl/protocol/detail/sfields.macro +++ b/include/xrpl/protocol/detail/sfields.macro @@ -257,6 +257,7 @@ TYPED_SFIELD(sfPrice, AMOUNT, 28) TYPED_SFIELD(sfSignatureReward, AMOUNT, 29) TYPED_SFIELD(sfMinAccountCreateAmount, AMOUNT, 30) TYPED_SFIELD(sfLPTokenBalance, AMOUNT, 31) +TYPED_SFIELD(sfTrustLineRewardAccumulator,AMOUNT, 99) // variable length (common) TYPED_SFIELD(sfPublicKey, VL, 1) @@ -292,6 +293,7 @@ TYPED_SFIELD(sfAssetClass, VL, 29) TYPED_SFIELD(sfProvider, VL, 30) TYPED_SFIELD(sfMPTokenMetadata, VL, 31) TYPED_SFIELD(sfCredentialType, VL, 32) +TYPED_SFIELD(sfHookName, VL, 97) TYPED_SFIELD(sfRemarkValue, VL, 98) TYPED_SFIELD(sfRemarkName, VL, 99) @@ -340,6 +342,7 @@ TYPED_SFIELD(sfLockingChainIssue, ISSUE, 1) TYPED_SFIELD(sfIssuingChainIssue, ISSUE, 2) TYPED_SFIELD(sfAsset, ISSUE, 3) TYPED_SFIELD(sfAsset2, ISSUE, 4) +TYPED_SFIELD(sfClaimCurrency, ISSUE, 5) // bridge TYPED_SFIELD(sfXChainBridge, XCHAIN_BRIDGE, 1) @@ -386,6 +389,8 @@ UNTYPED_SFIELD(sfImportVLKey, OBJECT, 94) UNTYPED_SFIELD(sfActiveValidator, OBJECT, 95) UNTYPED_SFIELD(sfGenesisMint, OBJECT, 96) UNTYPED_SFIELD(sfRemark, OBJECT, 97) +UNTYPED_SFIELD(sfHighReward, OBJECT, 98) +UNTYPED_SFIELD(sfLowReward, OBJECT, 99) // array of objects (common) // ARRAY/1 is reserved for end of array diff --git a/include/xrpl/protocol/detail/transactions.macro b/include/xrpl/protocol/detail/transactions.macro index baed75230..ac0643b9d 100644 --- a/include/xrpl/protocol/detail/transactions.macro +++ b/include/xrpl/protocol/detail/transactions.macro @@ -550,6 +550,7 @@ TRANSACTION(ttIMPORT, 97, Import, ({ * from a specified hook */ TRANSACTION(ttCLAIM_REWARD, 98, ClaimReward, ({ {sfIssuer, soeOPTIONAL}, + {sfClaimCurrency, soeOPTIONAL}, })) /** This transaction invokes a hook, providing arbitrary data. Essentially as a 0 drop payment. **/ diff --git a/include/xrpl/protocol/jss.h b/include/xrpl/protocol/jss.h index 964457536..4355544bc 100644 --- a/include/xrpl/protocol/jss.h +++ b/include/xrpl/protocol/jss.h @@ -76,6 +76,7 @@ JSS(Holder); // field. JSS(HookApiVersion); // field JSS(HookCanEmit); // field JSS(HookHash); // field +JSS(HookName); // field JSS(HookNamespace); // field JSS(HookOn); // field JSS(HookOnIncoming); // field diff --git a/release-builder.sh b/release-builder.sh index ac9c46981..712da3790 100755 --- a/release-builder.sh +++ b/release-builder.sh @@ -11,11 +11,11 @@ echo "START BUILDING (HOST)" echo "Cleaning previously built binary" rm -f release-build/xahaud -BUILD_CORES=$(echo "scale=0 ; `nproc` / 1.337" | bc) +BUILD_CORES=$(echo "scale=0 ; $(nproc) / 1.337" | bc) if [[ "$GITHUB_REPOSITORY" == "" ]]; then #Default - BUILD_CORES=${BUILD_CORES:-8} + BUILD_CORES=${BUILD_CORES:-8} fi # Ensure still works outside of GH Actions by setting these to /dev/null @@ -31,21 +31,19 @@ echo "-- GITHUB_SHA: $GITHUB_SHA" echo "-- GITHUB_RUN_NUMBER: $GITHUB_RUN_NUMBER" echo "-- CONTAINER_NAME: $CONTAINER_NAME" -which docker 2> /dev/null 2> /dev/null -if [ "$?" -eq "1" ] -then +which docker 2>/dev/null 2>/dev/null +if [ "$?" -eq "1" ]; then echo 'Docker not found. Install it first.' exit 1 fi -stat .git 2> /dev/null 2> /dev/null -if [ "$?" -eq "1" ] -then +stat .git 2>/dev/null 2>/dev/null +if [ "$?" -eq "1" ]; then echo 'Run this inside the source directory. (.git dir not found).' exit 1 fi -STATIC_CONTAINER=$(docker ps -a | grep $CONTAINER_NAME |wc -l) +STATIC_CONTAINER=$(docker ps -a | grep $CONTAINER_NAME | wc -l) CACHE_VOLUME_NAME="xahau-release-builder-cache" @@ -57,13 +55,14 @@ if false; then docker stop $CONTAINER_NAME else echo "No static container, build on temp container" - rm -rf release-build; - mkdir -p release-build; + rm -rf release-build + mkdir -p release-build docker volume create $CACHE_VOLUME_NAME # Create inline Dockerfile with environment setup for build-full.sh - DOCKERFILE_CONTENT=$(cat <<'DOCKERFILE_EOF' + DOCKERFILE_CONTENT=$( + cat <<'DOCKERFILE_EOF' FROM ghcr.io/phusion/holy-build-box:4.0.1-amd64 ARG BUILD_CORES=8 @@ -218,7 +217,7 @@ RUN /hbb_exe/activate-exec bash -c "ccache -M 100G && \ ln -s ../../bin/ccache /usr/lib64/ccache/c++" DOCKERFILE_EOF -) + ) # Build custom Docker image IMAGE_NAME="xahaud-builder:latest" @@ -228,14 +227,14 @@ DOCKERFILE_EOF if [[ "$GITHUB_REPOSITORY" == "" ]]; then # Non GH, local building echo "Non-GH runner, local building, temp container" - docker run -i --user 0:$(id -g) --rm -v /data/builds:/data/builds -v `pwd`:/io -v "$CACHE_VOLUME_NAME":/cache --network host "$IMAGE_NAME" /hbb_exe/activate-exec bash -c "source /opt/rh/gcc-toolset-11/enable && bash -x /io/build-full.sh '$GITHUB_REPOSITORY' '$GITHUB_SHA' '$BUILD_CORES' '$GITHUB_RUN_NUMBER'" + docker run -i --user 0:$(id -g) --rm -v /data/builds:/data/builds -v $(pwd):/io -v "$CACHE_VOLUME_NAME":/cache --network host "$IMAGE_NAME" /hbb_exe/activate-exec bash -c "source /opt/rh/gcc-toolset-11/enable && bash -x /io/build-full.sh '$GITHUB_REPOSITORY' '$GITHUB_SHA' '$BUILD_CORES' '$GITHUB_RUN_NUMBER'" else # GH Action, runner echo "GH Action, runner, clean & re-create create persistent container" docker rm -f $CONTAINER_NAME - echo "echo 'Stopping container: $CONTAINER_NAME'" >> "$JOB_CLEANUP_SCRIPT" - echo "docker stop --time=15 \"$CONTAINER_NAME\" || echo 'Failed to stop container or container not running'" >> "$JOB_CLEANUP_SCRIPT" - docker run -di --user 0:$(id -g) --name $CONTAINER_NAME -v /data/builds:/data/builds -v `pwd`:/io -v "$CACHE_VOLUME_NAME":/cache --network host "$IMAGE_NAME" /hbb_exe/activate-exec bash + echo "echo 'Stopping container: $CONTAINER_NAME'" >>"$JOB_CLEANUP_SCRIPT" + echo "docker stop --time=15 \"$CONTAINER_NAME\" || echo 'Failed to stop container or container not running'" >>"$JOB_CLEANUP_SCRIPT" + docker run -di --user 0:$(id -g) --name $CONTAINER_NAME -v /data/builds:/data/builds -v $(pwd):/io -v "$CACHE_VOLUME_NAME":/cache --network host "$IMAGE_NAME" /hbb_exe/activate-exec bash docker exec -i $CONTAINER_NAME /hbb_exe/activate-exec bash -c "source /opt/rh/gcc-toolset-11/enable && bash -x /io/build-full.sh '$GITHUB_REPOSITORY' '$GITHUB_SHA' '$BUILD_CORES' '$GITHUB_RUN_NUMBER'" docker stop $CONTAINER_NAME fi diff --git a/src/libxrpl/protocol/AMMCore.cpp b/src/libxrpl/protocol/AMMCore.cpp index 3bebfc465..1ea83afc2 100644 --- a/src/libxrpl/protocol/AMMCore.cpp +++ b/src/libxrpl/protocol/AMMCore.cpp @@ -66,7 +66,7 @@ invalidAMMAsset( Issue const& issue, std::optional> const& pair) { - if (badCurrency() == issue.currency) + if (isBadCurrency(issue.currency)) return temBAD_CURRENCY; if (isXRP(issue) && issue.account.isNonZero()) return temBAD_ISSUER; diff --git a/src/libxrpl/protocol/Feature.cpp b/src/libxrpl/protocol/Feature.cpp index 05164489e..52e1d7217 100644 --- a/src/libxrpl/protocol/Feature.cpp +++ b/src/libxrpl/protocol/Feature.cpp @@ -250,12 +250,9 @@ FeatureCollections::registerFeature( Feature const* i = getByName(name); if (!i) { - // If this check fails, and you just added a feature, increase the - // numFeatures value in Feature.h check( features.size() < detail::numFeatures, - "More features defined than allocated. Adjust numFeatures in " - "Feature.h."); + "More features defined than allocated."); auto const f = sha512Half(Slice(name.data(), name.size())); @@ -424,45 +421,26 @@ featureToName(uint256 const& f) #undef XRPL_FEATURE #pragma push_macro("XRPL_FIX") #undef XRPL_FIX +#pragma push_macro("XRPL_RETIRE") +#undef XRPL_RETIRE #define XRPL_FEATURE(name, supported, vote) \ uint256 const feature##name = registerFeature(#name, supported, vote); #define XRPL_FIX(name, supported, vote) \ uint256 const fix##name = registerFeature("fix" #name, supported, vote); +#define XRPL_RETIRE(name) \ + [[deprecated("The referenced amendment has been retired"), maybe_unused]] \ + uint256 const retired##name = retireFeature(#name); #include +#undef XRPL_RETIRE +#pragma pop_macro("XRPL_RETIRE") #undef XRPL_FIX #pragma pop_macro("XRPL_FIX") #undef XRPL_FEATURE #pragma pop_macro("XRPL_FEATURE") -// clang-format off - -// The following amendments have been active for at least two years. Their -// pre-amendment code has been removed and the identifiers are deprecated. -// All known amendments and amendments that may appear in a validated -// ledger must be registered either here or above with the "active" amendments -[[deprecated("The referenced amendment has been retired"), maybe_unused]] -uint256 const - retiredMultiSign = retireFeature("MultiSign"), - retiredTrustSetAuth = retireFeature("TrustSetAuth"), - retiredFeeEscalation = retireFeature("FeeEscalation"), - retiredPayChan = retireFeature("PayChan"), - retiredCryptoConditions = retireFeature("CryptoConditions"), - retiredTickSize = retireFeature("TickSize"), - retiredFix1368 = retireFeature("fix1368"), - retiredEscrow = retireFeature("Escrow"), - retiredFix1373 = retireFeature("fix1373"), - retiredEnforceInvariants = retireFeature("EnforceInvariants"), - retiredSortedDirectories = retireFeature("SortedDirectories"), - retiredFix1201 = retireFeature("fix1201"), - retiredFix1512 = retireFeature("fix1512"), - retiredFix1523 = retireFeature("fix1523"), - retiredFix1528 = retireFeature("fix1528"); - -// clang-format on - // All of the features should now be registered, since variables in a cpp file // are initialized from top to bottom. // diff --git a/src/libxrpl/protocol/InnerObjectFormats.cpp b/src/libxrpl/protocol/InnerObjectFormats.cpp index 55ef5c609..c1853427c 100644 --- a/src/libxrpl/protocol/InnerObjectFormats.cpp +++ b/src/libxrpl/protocol/InnerObjectFormats.cpp @@ -99,6 +99,7 @@ InnerObjectFormats::InnerObjectFormats() {sfHookOnOutgoing, soeOPTIONAL}, {sfHookCanEmit, soeOPTIONAL}, {sfHookApiVersion, soeOPTIONAL}, + {sfHookName, soeOPTIONAL}, {sfFlags, soeOPTIONAL}}); add(sfHookGrant.jsonName, @@ -253,6 +254,24 @@ InnerObjectFormats::InnerObjectFormats() {sfIssuer, soeREQUIRED}, {sfCredentialType, soeREQUIRED}, }); + + add(sfHighReward.jsonName, + sfHighReward.getCode(), + { + {sfRewardLgrFirst, soeREQUIRED}, + {sfRewardLgrLast, soeREQUIRED}, + {sfRewardTime, soeREQUIRED}, + {sfTrustLineRewardAccumulator, soeREQUIRED}, + }); + + add(sfLowReward.jsonName, + sfLowReward.getCode(), + { + {sfRewardLgrFirst, soeREQUIRED}, + {sfRewardLgrLast, soeREQUIRED}, + {sfRewardTime, soeREQUIRED}, + {sfTrustLineRewardAccumulator, soeREQUIRED}, + }); } InnerObjectFormats const& diff --git a/src/libxrpl/protocol/Issue.cpp b/src/libxrpl/protocol/Issue.cpp index 179cb1eb1..a42baf8ad 100644 --- a/src/libxrpl/protocol/Issue.cpp +++ b/src/libxrpl/protocol/Issue.cpp @@ -111,7 +111,7 @@ issueFromJson(Json::Value const& v) } auto const currency = to_currency(curStr.asString()); - if (currency == badCurrency() || currency == noCurrency()) + if (isBadCurrency(currency) || currency == noCurrency()) { Throw("issueFromJson currency must be a valid currency"); } diff --git a/src/libxrpl/protocol/STCurrency.cpp b/src/libxrpl/protocol/STCurrency.cpp index 56ac19da1..55726fef5 100644 --- a/src/libxrpl/protocol/STCurrency.cpp +++ b/src/libxrpl/protocol/STCurrency.cpp @@ -103,7 +103,7 @@ currencyFromJson(SField const& name, Json::Value const& v) } auto const currency = to_currency(v.asString()); - if (currency == badCurrency() || currency == noCurrency()) + if (isBadCurrency(currency) || currency == noCurrency()) { Throw( "currencyFromJson currency must be a valid currency"); diff --git a/src/libxrpl/protocol/TxFormats.cpp b/src/libxrpl/protocol/TxFormats.cpp index 777e5e645..55de0ab4d 100644 --- a/src/libxrpl/protocol/TxFormats.cpp +++ b/src/libxrpl/protocol/TxFormats.cpp @@ -48,6 +48,7 @@ TxFormats::TxFormats() {sfFirstLedgerSequence, soeOPTIONAL}, {sfNetworkID, soeOPTIONAL}, {sfHookParameters, soeOPTIONAL}, + {sfHookName, soeOPTIONAL}, }; #pragma push_macro("UNWRAP") diff --git a/src/test/app/AMMClawback_test.cpp b/src/test/app/AMMClawback_test.cpp index d2ad29003..e990cc4cb 100644 --- a/src/test/app/AMMClawback_test.cpp +++ b/src/test/app/AMMClawback_test.cpp @@ -32,9 +32,12 @@ class AMMClawback_test : public beast::unit_test::suite testcase("test invalid request"); using namespace jtx; + auto const features = + supported_amendments() | featureAMM | featureAMMClawback; + // Test if holder does not exist. { - Env env(*this); + Env env(*this, features); Account gw{"gateway"}; Account alice{"alice"}; env.fund(XRP(100000), gw, alice); @@ -60,7 +63,7 @@ class AMMClawback_test : public beast::unit_test::suite // Test if asset pair provided does not exist. This should // return terNO_AMM error. { - Env env(*this); + Env env(*this, features); Account gw{"gateway"}; Account alice{"alice"}; env.fund(XRP(100000), gw, alice); @@ -93,7 +96,7 @@ class AMMClawback_test : public beast::unit_test::suite // Test if the issuer field and holder field is the same. This should // return temMALFORMED error. { - Env env(*this); + Env env(*this, features); Account gw{"gateway"}; Account alice{"alice"}; env.fund(XRP(10000), gw, alice); @@ -123,7 +126,7 @@ class AMMClawback_test : public beast::unit_test::suite // Test if the Asset field matches the Account field. { - Env env(*this); + Env env(*this, features); Account gw{"gateway"}; Account alice{"alice"}; env.fund(XRP(10000), gw, alice); @@ -155,7 +158,7 @@ class AMMClawback_test : public beast::unit_test::suite // Test if the Amount field matches the Asset field. { - Env env(*this); + Env env(*this, features); Account gw{"gateway"}; Account alice{"alice"}; env.fund(XRP(10000), gw, alice); @@ -188,7 +191,7 @@ class AMMClawback_test : public beast::unit_test::suite // Test if the Amount is invalid, which is less than zero. { - Env env(*this); + Env env(*this, features); Account gw{"gateway"}; Account alice{"alice"}; env.fund(XRP(10000), gw, alice); @@ -229,7 +232,7 @@ class AMMClawback_test : public beast::unit_test::suite // Test if the issuer did not set asfAllowTrustLineClawback, AMMClawback // transaction is prohibited. { - Env env(*this); + Env env(*this, features); Account gw{"gateway"}; Account alice{"alice"}; env.fund(XRP(10000), gw, alice); @@ -254,7 +257,7 @@ class AMMClawback_test : public beast::unit_test::suite // Test invalid flag. { - Env env(*this); + Env env(*this, features); Account gw{"gateway"}; Account alice{"alice"}; env.fund(XRP(10000), gw, alice); @@ -282,7 +285,7 @@ class AMMClawback_test : public beast::unit_test::suite // Test if tfClawTwoAssets is set when the two assets in the AMM pool // are not issued by the same issuer. { - Env env(*this); + Env env(*this, features); Account gw{"gateway"}; Account alice{"alice"}; env.fund(XRP(10000), gw, alice); @@ -313,7 +316,7 @@ class AMMClawback_test : public beast::unit_test::suite // Test clawing back XRP is being prohibited. { - Env env(*this); + Env env(*this, features); Account gw{"gateway"}; Account alice{"alice"}; env.fund(XRP(1000000), gw, alice); @@ -2085,7 +2088,8 @@ class AMMClawback_test : public beast::unit_test::suite void run() override { - FeatureBitset const all{jtx::supported_amendments()}; + FeatureBitset const all{ + jtx::supported_amendments() | featureAMM | featureAMMClawback}; testInvalidRequest(); testFeatureDisabled(all - featureAMMClawback); diff --git a/src/test/app/AMMExtended_test.cpp b/src/test/app/AMMExtended_test.cpp index 16e66d803..5475b4a71 100644 --- a/src/test/app/AMMExtended_test.cpp +++ b/src/test/app/AMMExtended_test.cpp @@ -1403,7 +1403,8 @@ private: testOffers() { using namespace jtx; - FeatureBitset const all{supported_amendments()}; + FeatureBitset const all{ + supported_amendments() | featureAMM | featureAMMClawback}; testRmFundedOffer(all); testEnforceNoRipple(all); testFillModes(all); @@ -2802,8 +2803,11 @@ private: testcase("limitQuality"); using namespace jtx; + auto const features = + supported_amendments() | featureAMM | featureAMMClawback; + { - Env env(*this); + Env env(*this, features); fund(env, gw, {alice, bob, carol}, XRP(10'000), {USD(2'000)}); @@ -2829,11 +2833,13 @@ private: using namespace jtx; + auto const features = + supported_amendments() | featureAMM | featureAMMClawback; + for (auto const withFix : {true, false}) { - auto const feats = withFix - ? supported_amendments() - : supported_amendments() - FeatureBitset{fix1781}; + auto const feats = + withFix ? features : features - FeatureBitset{fix1781}; // Payment path starting with XRP Env env(*this, feats); @@ -2861,7 +2867,7 @@ private: } { // Payment path ending with XRP - Env env(*this); + Env env(*this, features); // Note, if alice doesn't have default ripple, then pay fails // with tecPATH_DRY. fund( @@ -2885,7 +2891,7 @@ private: // Payment where loop is formed in the middle of the path, not // on an endpoint auto const JPY = gw["JPY"]; - Env env(*this); + Env env(*this, features); // Note, if alice doesn't have default ripple, then pay fails // with tecPATH_DRY. fund( @@ -3113,7 +3119,10 @@ private: using namespace jtx; - Env env(*this); + auto const features = + supported_amendments() | featureAMM | featureAMMClawback; + + Env env(*this, features); fund(env, gw, {alice, bob, carol}, XRP(10'000)); env.trust(USD(1'000), alice, bob, carol); @@ -3828,7 +3837,8 @@ private: testFlow() { using namespace jtx; - FeatureBitset const all{supported_amendments()}; + FeatureBitset const all{ + supported_amendments() | featureAMM | featureAMMClawback}; FeatureBitset const ownerPaysFee{featureOwnerPaysFee}; testFalseDry(all); @@ -3844,7 +3854,8 @@ private: testCrossingLimits() { using namespace jtx; - FeatureBitset const all{supported_amendments()}; + FeatureBitset const all{ + supported_amendments() | featureAMM | featureAMMClawback}; testStepLimit(all); } @@ -3852,14 +3863,16 @@ private: testDeliverMin() { using namespace jtx; - FeatureBitset const all{supported_amendments()}; + FeatureBitset const all{ + supported_amendments() | featureAMM | featureAMMClawback}; test_convert_all_of_an_asset(all); } void testDepositAuth() { - auto const supported{jtx::supported_amendments()}; + auto const supported{ + jtx::supported_amendments() | featureAMM | featureAMMClawback}; testPayment(supported - featureDepositPreauth); testPayment(supported); testPayIOU(); @@ -3869,7 +3882,8 @@ private: testFreeze() { using namespace test::jtx; - auto const sa = supported_amendments(); + auto const sa = + supported_amendments() | featureAMM | featureAMMClawback; testRippleState(sa); testGlobalFreeze(sa); testOffersWhenFrozen(sa); @@ -3879,7 +3893,8 @@ private: testMultisign() { using namespace jtx; - auto const all = supported_amendments(); + auto const all = + supported_amendments() | featureAMM | featureAMMClawback; testTxMultisign( all - featureMultiSignReserve - featureExpandedSignerList); @@ -3891,7 +3906,8 @@ private: testPayStrand() { using namespace jtx; - auto const all = supported_amendments(); + auto const all = + supported_amendments() | featureAMM | featureAMMClawback; testToStrand(all); testRIPD1373(all); diff --git a/src/test/app/AMM_test.cpp b/src/test/app/AMM_test.cpp index f5f21fb1f..5e4bc39e3 100644 --- a/src/test/app/AMM_test.cpp +++ b/src/test/app/AMM_test.cpp @@ -56,6 +56,9 @@ private: using namespace jtx; + auto const features = + supported_amendments() | featureAMM | featureAMMClawback; + // XRP to IOU testAMM([&](AMM& ammAlice, Env&) { BEAST_EXPECT(ammAlice.expectBalances( @@ -72,7 +75,7 @@ private: // IOU to IOU + transfer fee { - Env env{*this}; + Env env{*this, features}; fund(env, gw, {alice}, {USD(20'000), BTC(0.5)}, Fund::All); env(rate(gw, 1.25)); env.close(); @@ -86,7 +89,7 @@ private: // Require authorization is set, account is authorized { - Env env{*this}; + Env env{*this, features}; env.fund(XRP(30'000), gw, alice); env.close(); env(fset(gw, asfRequireAuth)); @@ -100,7 +103,7 @@ private: // Cleared global freeze { - Env env{*this}; + Env env{*this, features}; env.fund(XRP(30'000), gw, alice); env.close(); env.trust(USD(30'000), alice); @@ -141,9 +144,12 @@ private: using namespace jtx; + auto const features = + supported_amendments() | featureAMM | featureAMMClawback; + // Can't have both XRP tokens { - Env env{*this}; + Env env{*this, features}; fund(env, gw, {alice}, {USD(30'000)}, Fund::All); AMM ammAlice( env, alice, XRP(10'000), XRP(10'000), ter(temBAD_AMM_TOKENS)); @@ -152,7 +158,7 @@ private: // Can't have both tokens the same IOU { - Env env{*this}; + Env env{*this, features}; fund(env, gw, {alice}, {USD(30'000)}, Fund::All); AMM ammAlice( env, alice, USD(10'000), USD(10'000), ter(temBAD_AMM_TOKENS)); @@ -161,7 +167,7 @@ private: // Can't have zero or negative amounts { - Env env{*this}; + Env env{*this, features}; fund(env, gw, {alice}, {USD(30'000)}, Fund::All); AMM ammAlice(env, alice, XRP(0), USD(10'000), ter(temBAD_AMOUNT)); BEAST_EXPECT(!ammAlice.ammExists()); @@ -177,7 +183,7 @@ private: // Bad currency { - Env env{*this}; + Env env{*this, features}; fund(env, gw, {alice}, {USD(30'000)}, Fund::All); AMM ammAlice( env, alice, XRP(10'000), BAD(10'000), ter(temBAD_CURRENCY)); @@ -186,7 +192,7 @@ private: // Insufficient IOU balance { - Env env{*this}; + Env env{*this, features}; fund(env, gw, {alice}, {USD(30'000)}, Fund::All); AMM ammAlice( env, alice, XRP(10'000), USD(40'000), ter(tecUNFUNDED_AMM)); @@ -195,7 +201,7 @@ private: // Insufficient XRP balance { - Env env{*this}; + Env env{*this, features}; fund(env, gw, {alice}, {USD(30'000)}, Fund::All); AMM ammAlice( env, alice, XRP(40'000), USD(10'000), ter(tecUNFUNDED_AMM)); @@ -204,7 +210,7 @@ private: // Invalid trading fee { - Env env{*this}; + Env env{*this, features}; fund(env, gw, {alice}, {USD(30'000)}, Fund::All); AMM ammAlice( env, @@ -229,7 +235,7 @@ private: // Invalid flags { - Env env{*this}; + Env env{*this, features}; fund(env, gw, {alice}, {USD(30'000)}, Fund::All); AMM ammAlice( env, @@ -248,7 +254,7 @@ private: // Invalid Account { - Env env{*this}; + Env env{*this, features}; Account bad("bad"); env.memoize(bad); AMM ammAlice( @@ -268,7 +274,7 @@ private: // Require authorization is set { - Env env{*this}; + Env env{*this, features}; env.fund(XRP(30'000), gw, alice); env.close(); env(fset(gw, asfRequireAuth)); @@ -281,7 +287,7 @@ private: // Globally frozen { - Env env{*this}; + Env env{*this, features}; env.fund(XRP(30'000), gw, alice); env.close(); env(fset(gw, asfGlobalFreeze)); @@ -294,7 +300,7 @@ private: // Individually frozen { - Env env{*this}; + Env env{*this, features}; env.fund(XRP(30'000), gw, alice); env.close(); env(trust(gw, alice["USD"](30'000))); @@ -307,7 +313,7 @@ private: // Insufficient reserve, XRP/IOU { - Env env(*this); + Env env(*this, features); auto const starting_xrp = XRP(1'000) + reserve(env, 3) + env.current()->fees().base * 4; env.fund(starting_xrp, gw); @@ -324,7 +330,7 @@ private: // Insufficient reserve, IOU/IOU { - Env env(*this); + Env env(*this, features); auto const starting_xrp = reserve(env, 4) + env.current()->fees().base * 5; env.fund(starting_xrp, gw); @@ -343,7 +349,7 @@ private: // Insufficient fee { - Env env(*this); + Env env(*this, features); fund(env, gw, {alice}, XRP(2'000), {USD(2'000), EUR(2'000)}); AMM ammAlice( env, @@ -394,7 +400,7 @@ private: // Issuer has DefaultRipple disabled { - Env env(*this); + Env env(*this, features); env.fund(XRP(30'000), gw); env(fclear(gw, asfDefaultRipple)); AMM ammGw(env, gw, XRP(10'000), USD(10'000), ter(terNO_RIPPLE)); @@ -1119,7 +1125,7 @@ private: // Insufficient reserve, XRP/IOU { - Env env(*this); + Env env(*this, features); auto const starting_xrp = reserve(env, 4) + env.current()->fees().base * 4; env.fund(XRP(10'000), gw); @@ -1154,7 +1160,7 @@ private: // Insufficient reserve, IOU/IOU { - Env env(*this); + Env env(*this, features); auto const starting_xrp = reserve(env, 4) + env.current()->fees().base * 4; env.fund(XRP(10'000), gw); @@ -1356,7 +1362,8 @@ private: testcase("Deposit"); using namespace jtx; - auto const all = supported_amendments(); + auto const all = + supported_amendments() | featureAMM | featureAMMClawback; // Equal deposit: 1000000 tokens, 10% of the current pool testAMM([&](AMM& ammAlice, Env& env) { @@ -1516,7 +1523,7 @@ private: // IOU to IOU + transfer fee { - Env env{*this}; + Env env{*this, all}; fund(env, gw, {alice}, {USD(20'000), BTC(0.5)}, Fund::All); env(rate(gw, 1.25)); env.close(); @@ -1657,7 +1664,8 @@ private: testcase("Invalid Withdraw"); using namespace jtx; - auto const all = supported_amendments(); + auto const all = + supported_amendments() | featureAMM | featureAMMClawback; testAMM( [&](AMM& ammAlice, Env& env) { @@ -1680,7 +1688,7 @@ private: {{XRP(99), USD(99)}}); { - Env env{*this}; + Env env{*this, all}; env.fund(XRP(30'000), gw, alice, bob); env.close(); env(fset(gw, asfRequireAuth)); @@ -2228,7 +2236,8 @@ private: testcase("Withdraw"); using namespace jtx; - auto const all = supported_amendments(); + auto const all = + supported_amendments() | featureAMM | featureAMMClawback; // Equal withdrawal by Carol: 1000000 of tokens, 10% of the current // pool @@ -2442,7 +2451,7 @@ private: // IOU to IOU + transfer fee { - Env env{*this}; + Env env{*this, all}; fund(env, gw, {alice}, {USD(20'000), BTC(0.5)}, Fund::All); env(rate(gw, 1.25)); env.close(); @@ -2605,7 +2614,8 @@ private: { testcase("Fee Vote"); using namespace jtx; - auto const all = supported_amendments(); + auto const all = + supported_amendments() | featureAMM | featureAMMClawback; // One vote sets fee to 1%. testAMM([&](AMM& ammAlice, Env& env) { @@ -2723,9 +2733,12 @@ private: using namespace jtx; using namespace std::chrono; + auto const features = + supported_amendments() | featureAMM | featureAMMClawback; + // burn all the LPTokens through a AMMBid transaction { - Env env(*this); + Env env(*this, features); fund(env, gw, {alice}, XRP(2'000), {USD(2'000)}); AMM amm(env, gw, XRP(1'000), USD(1'000), false, 1'000); @@ -2744,7 +2757,7 @@ private: // burn all the LPTokens through a AMMBid transaction { - Env env(*this); + Env env(*this, features); fund(env, gw, {alice}, XRP(2'000), {USD(2'000)}); AMM amm(env, gw, XRP(1'000), USD(1'000), false, 1'000); @@ -2922,7 +2935,7 @@ private: // Bid all tokens, still own the slot { - Env env(*this); + Env env(*this, features); fund(env, gw, {alice, bob}, XRP(1'000), {USD(1'000)}); AMM amm(env, gw, XRP(10), USD(1'000)); auto const lpIssue = amm.lptIssue(); @@ -3361,12 +3374,15 @@ private: using namespace std::chrono; using namespace std::literals::chrono_literals; + auto const features = + supported_amendments() | featureAMM | featureAMMClawback; + // Can't pay into AMM account. // Can't pay out since there is no keys for (auto const& acct : {gw, alice}) { { - Env env(*this); + Env env(*this, features); fund(env, gw, {alice, carol}, XRP(1'000), {USD(100)}); // XRP balance is below reserve AMM ammAlice(env, acct, XRP(10), USD(10)); @@ -3381,7 +3397,7 @@ private: ter(tecNO_PERMISSION)); } { - Env env(*this); + Env env(*this, features); fund(env, gw, {alice, carol}, XRP(10'000'000), {USD(10'000)}); // XRP balance is above reserve AMM ammAlice(env, acct, XRP(1'000'000), USD(100)); @@ -4417,7 +4433,8 @@ private: { testcase("Amendment"); using namespace jtx; - FeatureBitset const all{supported_amendments()}; + FeatureBitset const all{ + supported_amendments() | featureAMM | featureAMMClawback}; FeatureBitset const noAMM{all - featureAMM}; FeatureBitset const noNumber{all - fixUniversalNumber}; FeatureBitset const noAMMAndNumber{ @@ -4467,6 +4484,9 @@ private: testcase("Rippling"); using namespace jtx; + auto const features = + supported_amendments() | featureAMM | featureAMMClawback; + // Rippling via AMM fails because AMM trust line has 0 limit. // Set up two issuers, A and B. Have each issue a token called TST. // Have another account C hold TST from both issuers, @@ -4478,7 +4498,7 @@ private: // to shift at a 1:1 rate with no fee applied has it not been // for 0 limit. { - Env env(*this); + Env env(*this, features); auto const A = Account("A"); auto const B = Account("B"); auto const TSTA = A["TST"]; @@ -5075,7 +5095,8 @@ private: testcase("Auto Delete"); using namespace jtx; - FeatureBitset const all{supported_amendments()}; + FeatureBitset const all{ + supported_amendments() | featureAMM | featureAMMClawback}; { Env env( @@ -5187,7 +5208,9 @@ private: { testcase("Clawback"); using namespace jtx; - Env env(*this); + auto const features = + supported_amendments() | featureAMM | featureAMMClawback; + Env env(*this, features); env.fund(XRP(2'000), gw); env.fund(XRP(2'000), alice); AMM amm(env, gw, XRP(1'000), USD(1'000)); @@ -5586,7 +5609,8 @@ private: { testcase("Fix Default Inner Object"); using namespace jtx; - FeatureBitset const all{supported_amendments()}; + FeatureBitset const all{ + supported_amendments() | featureAMM | featureAMMClawback}; auto test = [&](FeatureBitset features, TER const& err1, @@ -6305,7 +6329,7 @@ private: {{xrpPool, iouPool}}, 889, std::nullopt, - {jtx::supported_amendments()}); + {jtx::supported_amendments() | featureAMM | featureAMMClawback}); } void @@ -7121,7 +7145,8 @@ private: void run() override { - FeatureBitset const all{jtx::supported_amendments()}; + FeatureBitset const all{ + jtx::supported_amendments() | featureAMM | featureAMMClawback}; testInvalidInstance(); testInstanceCreate(); testInvalidDeposit(all); diff --git a/src/test/app/ClaimReward_test.cpp b/src/test/app/ClaimReward_test.cpp index d33e68aa2..becb8f039 100644 --- a/src/test/app/ClaimReward_test.cpp +++ b/src/test/app/ClaimReward_test.cpp @@ -27,6 +27,14 @@ namespace ripple { namespace test { struct ClaimReward_test : public beast::unit_test::suite { +private: + // helper + void static overrideFlag(Json::Value& jv) + { + jv[jss::Flags] = hsfOVERRIDE; + } + +public: bool expectRewards( jtx::Env const& env, @@ -60,6 +68,52 @@ struct ClaimReward_test : public beast::unit_test::suite return true; } + bool + expectRewardsIOU( + jtx::Env const& env, + jtx::Account const& acct, + jtx::IOU const& iou, + std::uint32_t ledgerFirst, + std::uint32_t ledgerLast, + STAmount accumulator, + std::uint32_t time) + { + auto const sle = env.le(keylet::line(acct, iou.account, iou.currency)); + BEAST_EXPECT(!!sle); + auto const& sfRewardField = + std::minmax(acct.id(), iou.account.id()).first == acct.id() + ? sfLowReward + : sfHighReward; + + if (!sle->isFieldPresent(sfRewardField)) + return false; + + auto const& reward = + static_cast(sle->peekAtField(sfRewardField)); + + if (!reward.isFieldPresent(sfRewardLgrFirst) || + reward.getFieldU32(sfRewardLgrFirst) != ledgerFirst) + { + return false; + } + if (!reward.isFieldPresent(sfRewardLgrLast) || + reward.getFieldU32(sfRewardLgrLast) != ledgerLast) + { + return false; + } + if (!reward.isFieldPresent(sfTrustLineRewardAccumulator) || + reward.getFieldAmount(sfTrustLineRewardAccumulator) != accumulator) + { + return false; + } + if (!reward.isFieldPresent(sfRewardTime) || + reward.getFieldU32(sfRewardTime) != time) + { + return false; + } + return true; + } + bool expectNoRewards(jtx::Env const& env, jtx::Account const& acct) { @@ -83,6 +137,24 @@ struct ClaimReward_test : public beast::unit_test::suite return true; } + bool + expectNoRewardsIOU( + jtx::Env const& env, + jtx::Account const& acct, + jtx::IOU const& iou) + { + auto const sle = env.le(keylet::line(acct, iou.account, iou.currency)); + BEAST_EXPECT(!!sle); + auto const& sfRewardField = + std::minmax(acct.id(), iou.account.id()).first == acct.id() + ? sfLowReward + : sfHighReward; + + if (sle->isFieldPresent(sfRewardField)) + return false; + return true; + } + void testEnabled(FeatureBitset features) { @@ -92,7 +164,7 @@ struct ClaimReward_test : public beast::unit_test::suite // setup env auto const alice = Account("alice"); - auto const issuer = Account("issuer"); + auto const issuer = Account::master; for (bool const withClaimReward : {false, true}) { @@ -102,7 +174,11 @@ struct ClaimReward_test : public beast::unit_test::suite withClaimReward ? features : features - featureBalanceRewards; Env env{*this, amend}; - env.fund(XRP(1000), alice, issuer); + env.fund(XRP(1000), alice); + env.close(); + + env(hook(issuer, {{hso(jtx::genesis::AcceptHook)}}, 0), + fee(XRP(1))); env.close(); auto const txResult = @@ -119,7 +195,10 @@ struct ClaimReward_test : public beast::unit_test::suite .count(); // CLAIM - env(reward::claim(alice), reward::issuer(issuer), txResult); + env(reward::claim(alice), + reward::issuer(issuer), + fee(XRP(1)), + txResult); env.close(); if (withClaimReward) @@ -195,14 +274,19 @@ struct ClaimReward_test : public beast::unit_test::suite Env env{*this, amend}; auto const alice = Account("alice"); - auto const issuer = Account("issuer"); + auto const issuer = Account::master; - env.fund(XRP(1000), alice, issuer); + env.fund(XRP(1000), alice); + env.close(); + + env(hook(issuer, {{hso(jtx::genesis::AcceptHook)}}, 0), + fee(XRP(1))); env.close(); auto tx = reward::claim(alice); env(tx, reward::issuer(issuer), + fee(XRP(1)), txflags(tfFullyCanonicalSig), withFixFlags ? ter(tesSUCCESS) : ter(temINVALID_FLAG)); env.close(); @@ -221,6 +305,105 @@ struct ClaimReward_test : public beast::unit_test::suite env(reward::claim(alice), reward::issuer(alice), ter(temMALFORMED)); env.close(); } + + // featureIOURewardClaim + + // temDISABLED + // featureIOURewardClaim amendment is disabled + { + test::jtx::Env env{ + *this, + network::makeNetworkConfig(21337), + features - featureIOURewardClaim}; + + auto const alice = Account("alice"); + auto const bob = Account("bob"); + auto const gw = Account("gw"); + env.fund(XRP(1000), alice, bob, gw); + env.close(); + + jtx::IOU const USD = gw["USD"]; + + env(reward::claim(alice), + reward::issuer(bob), + reward::claimCurrency(USD), + ter(temDISABLED)); + env.close(); + } + + // temMALFORMED + // ClaimCurrency.account cannot be the source account. + { + test::jtx::Env env{*this, network::makeNetworkConfig(21337)}; + + auto const alice = Account("alice"); + auto const bob = Account("bob"); + env.fund(XRP(1000), alice); + env.close(); + + jtx::IOU const USD = alice["USD"]; + + env(reward::claim(alice), + reward::issuer(bob), + reward::claimCurrency(USD), + ter(temMALFORMED)); + env.close(); + } + + // temMALFORMED + // Issuer cannot be Genesis account if ClaimCurrency is set. + { + test::jtx::Env env{*this, network::makeNetworkConfig(21337)}; + + auto const alice = Account("alice"); + auto const gw = Account("gw"); + env.fund(XRP(1000), alice, gw); + env.close(); + + jtx::IOU const USD = gw["USD"]; + + env(reward::claim(alice), + reward::issuer(Account::master), + reward::claimCurrency(USD), + ter(temBAD_ISSUER)); + } + + // MPT + { + // tested in testMPTInvalidInTx() at MPToken_test.cpp + } + + // XAH RewardClaim: Issuer must be the Genesis account if + // featureXahauGenesis and featureIOURewardClaim are enabled. + for (bool const withIOURewardClaim : {false, true}) + { + auto const amend = withIOURewardClaim + ? features + : features - featureIOURewardClaim; + + auto const alice = Account("alice"); + auto const badIssuer = Account("gw"); + auto const issuer = Account::master; + auto const USD = badIssuer["USD"]; + + Env env{*this, amend}; + env.fund(XRP(1000), alice, badIssuer); + env.close(); + + env(hook(issuer, {{hso(jtx::genesis::AcceptHook)}}, 0), + fee(XRP(1))); + env.close(); + + env(reward::claim(alice), + reward::issuer(badIssuer), + fee(XRP(1)), + withIOURewardClaim ? ter(temBAD_ISSUER) : ter(tesSUCCESS)); + + env(reward::claim(alice), + reward::issuer(issuer), + fee(XRP(1)), + ter(tesSUCCESS)); + } } void @@ -244,6 +427,7 @@ struct ClaimReward_test : public beast::unit_test::suite auto const alice = Account("alice"); auto const issuer = Account("issuer"); env.memoize(alice); + auto USD = issuer["USD"]; env.fund(XRP(1000), issuer); env.close(); @@ -251,7 +435,10 @@ struct ClaimReward_test : public beast::unit_test::suite auto tx = reward::claim(alice); tx[jss::Sequence] = 0; tx[jss::Fee] = 10; - env(tx, reward::issuer(issuer), ter(terNO_ACCOUNT)); + env(tx, + reward::issuer(issuer), + reward::claimCurrency(USD), + ter(terNO_ACCOUNT)); env.close(); } @@ -261,9 +448,9 @@ struct ClaimReward_test : public beast::unit_test::suite test::jtx::Env env{*this, network::makeNetworkConfig(21337)}; auto const alice = Account("alice"); - auto const issuer = Account("issuer"); + auto const issuer = Account::master; - env.fund(XRP(1000), alice, issuer); + env.fund(XRP(1000), alice); env.close(); env(reward::claim(alice), @@ -294,18 +481,26 @@ struct ClaimReward_test : public beast::unit_test::suite auto const issuer = Account("issuer"); env.memoize(issuer); + auto USD = issuer["USD"]; + env.fund(XRP(1000), alice); env.close(); auto tx = reward::claim(alice); - env(tx, reward::issuer(issuer), ter(tecNO_ISSUER)); + env(tx, + reward::issuer(issuer), + reward::claimCurrency(USD), + ter(tecNO_ISSUER)); env.close(); } // tecNO_PERMISSION // issuer is an AMM account { - test::jtx::Env env{*this, network::makeNetworkConfig(21337)}; + test::jtx::Env env{ + *this, + network::makeNetworkConfig(21337), + features | featureAMM}; auto const alice = Account("alice"); auto const issuer = Account("issuer"); @@ -320,9 +515,162 @@ struct ClaimReward_test : public beast::unit_test::suite env(reward::claim(alice), reward::issuer(amm.ammAccount()), + reward::claimCurrency(USD), ter(tecNO_PERMISSION)); env.close(); } + + // tecNO_TARGET + // no claim reward hook + { + Env env{*this}; + + auto const alice = Account("alice"); + auto const issuer = Account::master; + + env.fund(XRP(1000), alice); + env.close(); + + // Doesn't have hook + { + env(reward::claim(alice), + reward::issuer(issuer), + ter(tecNO_TARGET)); + env.close(); + } + // Invalid HookOn + { + auto hookObj = hso(jtx::genesis::AcceptHook, overrideFlag); + hookObj[jss::HookOn] = to_string(UINT256_BIT[ttCLAIM_REWARD]); + env(hook(issuer, {{hookObj}}, 0), fee(XRP(1))); + env.close(); + + env(reward::claim(alice), + reward::issuer(issuer), + ter(tecNO_TARGET)); + env.close(); + } + // Invalid IncomingHookOn + { + auto hookObj = hso(jtx::genesis::AcceptHook, overrideFlag); + hookObj.removeMember(jss::HookOn); + hookObj[jss::HookOnIncoming] = + to_string(UINT256_BIT[ttCLAIM_REWARD]); + hookObj[jss::HookOnOutgoing] = to_string(uint256{}); + env(hook(issuer, {{hookObj}}, 0), fee(XRP(1))); + env.close(); + + env(reward::claim(alice), + reward::issuer(issuer), + ter(tecNO_TARGET)); + } + // Vaild HookOn + { + auto hookObj = hso(jtx::genesis::AcceptHook, overrideFlag); + hookObj[jss::HookOn] = to_string(~UINT256_BIT[ttCLAIM_REWARD]); + env(hook(issuer, {{hookObj}}, 0), fee(XRP(1))); + env.close(); + + env(reward::claim(alice), + reward::issuer(issuer), + fee(XRP(1)), + ter(tesSUCCESS)); + } + // Vaild IncomingHookOn + { + auto hookObj = hso(jtx::genesis::AcceptHook, overrideFlag); + hookObj.removeMember(jss::HookOn); + hookObj[jss::HookOnIncoming] = + to_string(~UINT256_BIT[ttCLAIM_REWARD]); + hookObj[jss::HookOnOutgoing] = to_string(uint256{}); + env(hook(issuer, {{hookObj}}, 0), fee(XRP(1))); + env.close(); + + env(reward::claim(alice), + reward::issuer(issuer), + fee(XRP(1)), + ter(tesSUCCESS)); + } + // Invalid Hooks Array + { + auto hookObj = hso(jtx::genesis::AcceptHook, overrideFlag); + hookObj[jss::HookOn] = to_string(UINT256_BIT[ttCLAIM_REWARD]); + env(hook( + issuer, + {{ + hookObj, + hookObj, + hookObj, + hookObj, + hookObj, + hookObj, + hookObj, + hookObj, + hookObj, + hookObj, + }}, + 0), + fee(XRP(1))); + env.close(); + + env(reward::claim(alice), + reward::issuer(issuer), + fee(XRP(1)), + ter(tecNO_TARGET)); + } + // Vaild Hooks Array + { + auto hookObj = hso(jtx::genesis::AcceptHook, overrideFlag); + hookObj[jss::HookOn] = to_string(UINT256_BIT[ttCLAIM_REWARD]); + auto hookObj2 = hso(jtx::genesis::AcceptHook, overrideFlag); + hookObj2[jss::HookOn] = to_string(~UINT256_BIT[ttCLAIM_REWARD]); + env(hook( + issuer, + {{ + hookObj, + hookObj, + hookObj, + hookObj, + hookObj, + hookObj, + hookObj, + hookObj, + hookObj, + hookObj2, + }}, + 0), + fee(XRP(1))); + env.close(); + + env(reward::claim(alice), + reward::issuer(issuer), + fee(XRP(1)), + ter(tesSUCCESS)); + } + } + + // tecNO_LINE + // trustline does not exist. + { + test::jtx::Env env{*this, network::makeNetworkConfig(21337)}; + + auto const alice = Account("alice"); + + auto const gw = Account("gw"); + env.fund(XRP(1000), alice, gw); + env.close(); + + env(hook(gw, {{hso(jtx::genesis::AcceptHook)}}, 0), fee(XRP(1))); + env.close(); + + jtx::IOU const USD = gw["USD"]; + + env(reward::claim(alice), + reward::issuer(gw), + reward::claimCurrency(USD), + fee(XRP(1)), + ter(tecNO_LINE)); + } } void @@ -335,9 +683,17 @@ struct ClaimReward_test : public beast::unit_test::suite test::jtx::Env env{*this, network::makeNetworkConfig(21337)}; auto const alice = Account("alice"); + auto const bob = Account("bob"); + auto const gw = Account("gw"); auto const issuer = Account("issuer"); - env.fund(XRP(1000), alice, issuer); + env.fund(XRP(1000), alice, bob, gw, issuer); + env.close(); + + env(hook(issuer, {{hso(jtx::genesis::AcceptHook)}}, 0), fee(XRP(1))); + env.close(); + env(hook(Account::master, {{hso(jtx::genesis::AcceptHook)}}, 0), + fee(XRP(1))); env.close(); // test claim rewards - no opt out @@ -352,7 +708,7 @@ struct ClaimReward_test : public beast::unit_test::suite .count(); auto tx = reward::claim(alice); - env(tx, reward::issuer(issuer), ter(tesSUCCESS)); + env(tx, reward::issuer(Account::master), fee(XRP(1)), ter(tesSUCCESS)); env.close(); BEAST_EXPECT( @@ -365,6 +721,51 @@ struct ClaimReward_test : public beast::unit_test::suite env.close(); BEAST_EXPECT(expectNoRewards(env, alice) == true); + + // test iou claim rewards + { + // set trustline + env(trust(bob, gw["USD"](10000))); + env.close(); + + // opt in + auto const currentLedger = env.current()->seq(); + auto const currentTime = + std::chrono::duration_cast( + env.app() + .getLedgerMaster() + .getValidatedLedger() + ->info() + .parentCloseTime.time_since_epoch()) + .count(); + + auto tx = reward::claim(bob); + env(tx, + reward::issuer(issuer), + reward::claimCurrency(gw["USD"]), + fee(XRP(1)), + ter(tesSUCCESS)); + env.close(); + + BEAST_EXPECT( + expectRewardsIOU( + env, + bob, + gw["USD"], + currentLedger, + currentLedger, + gw["USD"](0), + currentTime) == true); + + // opt out + env(reward::claim(bob), + reward::claimCurrency(gw["USD"]), + txflags(tfOptOut), + ter(tesSUCCESS)); + env.close(); + + BEAST_EXPECT(expectNoRewardsIOU(env, bob, gw["USD"]) == true); + } } void @@ -375,16 +776,20 @@ struct ClaimReward_test : public beast::unit_test::suite using namespace std::literals::chrono_literals; Env env{*this, features}; auto const alice = Account("alice"); - auto const issuer = Account("issuer"); - env.fund(XRP(10000), alice, issuer); + auto const issuer = Account::master; + env.fund(XRP(10000), alice); std::uint32_t aliceTicketSeq{env.seq(alice) + 1}; env(ticket::create(alice, 10)); std::uint32_t const aliceSeq{env.seq(alice)}; env.require(owners(alice, 10)); + env(hook(issuer, {{hso(jtx::genesis::AcceptHook)}}, 0), fee(XRP(1))); + env.close(); + env(reward::claim(alice), reward::issuer(issuer), ticket::use(aliceTicketSeq++), + fee(XRP(1)), ter(tesSUCCESS)); env.require(tickets(alice, env.seq(alice) - aliceTicketSeq)); @@ -392,6 +797,317 @@ struct ClaimReward_test : public beast::unit_test::suite env.require(owners(alice, 9)); } + void + testBalanceChanges(FeatureBitset features) + { + testcase("balance changes"); + using namespace jtx; + using namespace std::literals::chrono_literals; + + auto const getCurrentTime = [&](Env& env) { + return std::chrono::duration_cast( + env.app() + .getLedgerMaster() + .getValidatedLedger() + ->info() + .parentCloseTime.time_since_epoch()) + .count(); + }; + + // Native Reward Claim + { + Env env{*this, features}; + auto const alice = Account("alice"); + auto const gw = Account("gw"); + + auto const issuer = Account::master; + env.fund(XRP(10001), alice, gw); + env.close(); + + env(hook(issuer, {{hso(jtx::genesis::AcceptHook)}}, 0), + fee(XRP(1))); + env.close(); + + auto const currentTime = getCurrentTime(env); + auto const currentLedger = env.current()->seq(); + + env(reward::claim(alice), reward::issuer(issuer), fee(XRP(1))); + env.close(); + + env(fset(alice, 0)); + env.close(); + + BEAST_EXPECT( + expectRewards( + env, + alice, + currentLedger, + currentLedger + 1, + 10000, // 10000 XAH * time 1 + currentTime) == true); + } + + // IOU Reward Claim + for (bool const fromHighAccount : {true, false}) + { + Env env{*this, features}; + auto const alice = Account("alice"); + auto const bob = Account("bob"); + auto const issuer = Account("issuer"); + + auto const user = fromHighAccount ? alice : bob; + auto const gw = fromHighAccount ? bob : alice; + + if (fromHighAccount) + BEAST_EXPECT(user.id() < gw.id()); + else + BEAST_EXPECT(user.id() > gw.id()); + + env.fund(XRP(10000), user, gw, issuer); + env(fset(gw, asfDefaultRipple)); + + auto hookObj = hso(jtx::genesis::AcceptHook); + hookObj[jss::HookOn] = to_string(~UINT256_BIT[ttCLAIM_REWARD]); + env(hook(gw, {{hookObj}}, 0), fee(XRP(1))); + env.close(); + + env(trust(user, gw["USD"](1000000)), fee(XRP(1))); + env.close(); + env(pay(gw, user, gw["USD"](10000))); + env.close(); + + auto currentTime = getCurrentTime(env); + auto currentLedger = env.current()->seq(); + + env(reward::claim(user), + reward::issuer(gw), + reward::claimCurrency(gw["USD"]), + fee(XRP(1))); + env.close(); + + env(pay(user, gw, gw["USD"](10000))); + env.close(); + + BEAST_EXPECT( + expectRewardsIOU( + env, + user, + gw["USD"], + currentLedger, + currentLedger + 1, + user["USD"](10000), // 10000 USD * time 1 + currentTime) == true); + + env(pay(gw, user, gw["USD"](1))); + env.close(); + + // check Balance == 0 + BEAST_EXPECT( + expectRewardsIOU( + env, + user, + gw["USD"], + currentLedger, + currentLedger + 2, + user["USD"](10000), // 10000 USD * time 1 + 0 USD * time 1 + currentTime) == true); + } + + // Check Balance minus -> plus, plus -> minus + for (bool const fromHighAccount : {true, false}) + { + Env env{*this, features}; + auto const alice = Account("alice"); + auto const bob = Account("bob"); + auto const issuer = Account("issuer"); + + auto const user = fromHighAccount ? alice : bob; + auto const gw = fromHighAccount ? bob : alice; + + if (fromHighAccount) + BEAST_EXPECT(user.id() < gw.id()); + else + BEAST_EXPECT(user.id() > gw.id()); + + env.fund(XRP(10000), user, gw, issuer); + env(fset(gw, asfDefaultRipple)); + env.close(); + + auto hookObj = hso(jtx::genesis::AcceptHook); + hookObj[jss::HookOn] = to_string(~UINT256_BIT[ttCLAIM_REWARD]); + env(hook(gw, {{hookObj}}, 0), fee(XRP(1))); + env.close(); + + env(trust(user, gw["USD"](1000000))); + env.close(); + env(trust(gw, user["USD"](1000000))); + env(pay(gw, user, gw["USD"](10000))); + env.close(); + + auto currentTime = getCurrentTime(env); + auto currentLedger = env.current()->seq(); + + env(reward::claim(user), + reward::issuer(gw), + reward::claimCurrency(gw["USD"]), + fee(XRP(1))); + env.close(); + + env(pay(user, gw, gw["USD"](20000))); + env.close(); + + env(pay(user, gw, gw["USD"](1))); + env.close(); + + BEAST_EXPECT( + expectRewardsIOU( + env, + user, + gw["USD"], + currentLedger, + currentLedger + 2, + user["USD"](10000), // 10000 USD * time 1 + 0 USD * time 1 + currentTime) == true); + } + + // test with escrow (locked balance) + for (bool const fromHighAccount : {true, false}) + { + for (bool const hasEscrow : {true, false}) + { + Env env{*this, features}; + auto const alice = Account("alice"); + auto const bob = Account("bob"); + auto const issuer = Account("issuer"); + + auto const user = fromHighAccount ? alice : bob; + auto const gw = fromHighAccount ? bob : alice; + + if (fromHighAccount) + BEAST_EXPECT(user.id() < gw.id()); + else + BEAST_EXPECT(user.id() > gw.id()); + + env.fund(XRP(10000), user, gw, issuer); + env(fset(gw, asfDefaultRipple)); + + auto hookObj = hso(jtx::genesis::AcceptHook); + hookObj[jss::HookOn] = to_string(~UINT256_BIT[ttCLAIM_REWARD]); + env(hook(gw, {{hookObj}}, 0), fee(XRP(1))); + env.close(); + + env(trust(user, gw["USD"](1000000)), fee(XRP(1))); + env.close(); + env(pay(gw, user, gw["USD"](10000))); + env.close(); + + if (hasEscrow) + { + env(escrow(user, user, gw["USD"](2000)), + finish_time(env.now() + 1s), + fee(XRP(1))); + env.close(); + } + + auto currentTime = getCurrentTime(env); + auto currentLedger = env.current()->seq(); + + env(reward::claim(user), + reward::issuer(gw), + reward::claimCurrency(gw["USD"]), + fee(XRP(1))); + env.close(); + env(pay(user, gw, gw["USD"](5000))); + env.close(); + + BEAST_EXPECT( + expectRewardsIOU( + env, + user, + gw["USD"], + currentLedger, + currentLedger + 1, + user["USD"](10000), // 10000 USD * time 1 + currentTime) == true); + + env(pay(gw, user, gw["USD"](1))); + env.close(); + + // check Balance == 0 + BEAST_EXPECT( + expectRewardsIOU( + env, + user, + gw["USD"], + currentLedger, + currentLedger + 2, + user["USD"]( + 15000), // 10000 USD * time 1 + 5000 USD * time 1 + currentTime) == true); + } + } + + // STAmount overflow in reward accumulation should not cause + // transaction failure (tefEXCEPTION). The overflow should be + // gracefully skipped via try-catch. + for (bool const fromHighAccount : {true, false}) + { + Env env{*this, features}; + auto const alice = Account("alice"); + auto const bob = Account("bob"); + auto const issuer = Account("issuer"); + + auto const user = fromHighAccount ? alice : bob; + auto const gw = fromHighAccount ? bob : alice; + + env.fund(XRP(10000), user, gw, issuer); + env(fset(gw, asfDefaultRipple)); + env.close(); + + auto hookObj = hso(jtx::genesis::AcceptHook); + hookObj[jss::HookOn] = to_string(~UINT256_BIT[ttCLAIM_REWARD]); + env(hook(gw, {{hookObj}}, 0), fee(XRP(1))); + env.close(); + + // Use a near-max IOU balance at exponent 80. When + // multiply(balance, STAmount(lgrElapsed), issue) is called + // with lgrElapsed >= 2, the result exponent exceeds + // cMaxOffset(80), causing IOUAmount::normalize to throw + // std::overflow_error("value overflow"). + auto const bigUSD = STAmount{ + gw["USD"].issue(), std::uint64_t(5000000000000000ull), 80}; + // Payment amount must be large enough to register a + // balance change given STAmount's 16-digit precision. + auto const payBackUSD = STAmount{ + gw["USD"].issue(), std::uint64_t(1000000000000000ull), 80}; + + env(trust(user, bigUSD)); + env.close(); + env(pay(gw, user, bigUSD)); + env.close(); + + // Claim IOU reward to initialize reward tracking + env(reward::claim(user), + reward::issuer(gw), + reward::claimCurrency(gw["USD"]), + fee(XRP(1))); + env.close(); + + // Advance ledger so lgrElapsed >= 2. With lgrElapsed=1 + // the multiply result is exactly at cMaxOffset boundary + // (no overflow). With lgrElapsed >= 2, the result exponent + // exceeds cMaxOffset and triggers the overflow. + env.close(); + + // This payment modifies the trustline balance, triggering + // reward accumulation in Transactor. Without the try-catch + // fix, multiply() throws std::overflow_error("value overflow") + // and the transaction fails with tefEXCEPTION. + env(pay(user, gw, payBackUSD), ter(tesSUCCESS)); + env.close(); + } + } + void testWithFeats(FeatureBitset features) { @@ -400,6 +1116,7 @@ struct ClaimReward_test : public beast::unit_test::suite testInvalidPreclaim(features); testValidNoHook(features); testUsingTickets(features); + testBalanceChanges(features); } public: diff --git a/src/test/app/Escrow_test.cpp b/src/test/app/Escrow_test.cpp index 778e0bed7..bc3bf161c 100644 --- a/src/test/app/Escrow_test.cpp +++ b/src/test/app/Escrow_test.cpp @@ -18,6 +18,7 @@ //============================================================================== #include +#include #include #include #include @@ -4316,6 +4317,93 @@ struct Escrow_test : public beast::unit_test::suite env.close(); } + void + testIOUAMM(FeatureBitset features) + { + testcase("IOU AMM"); + using namespace test::jtx; + using namespace std::chrono; + + Account alice{"alice"}; + Account bob{"bob"}; + Account gw{"gw"}; + + auto const USD = gw["USD"]; + + // AMMCreate fails - insufficient balance + Env env(*this, features | featureAMM | featureAMMClawback); + env.fund(XRP(10000), alice, bob, gw); + env.close(); + + env.trust(USD(100000), alice); + env.close(); + env(pay(gw, alice, USD(1000))); + env.close(); + + env(escrow(alice, bob, USD(1000)), finish_time(env.now() + 1s)); + env.close(); + + // AMMCreate fails - insufficient balance + AMM ammFail(env, alice, XRP(1000), USD(1000), ter(tecUNFUNDED_AMM)); + + env(pay(gw, alice, USD(1000))); + env.close(); + + AMM ammAlice(env, alice, XRP(1000), USD(1000)); + BEAST_EXPECT(ammAlice.ammExists()); + + // Single Asset Deposit fails - insufficient balance + ammAlice.deposit( + alice, + USD(1000), + std::nullopt, + std::nullopt, + std::nullopt, + ter(tecUNFUNDED_AMM)); + + // Double Asset Deposit fails - insufficient balance + ammAlice.deposit( + alice, + USD(1000), + XRP(1000), + std::nullopt, + std::nullopt, + ter(tecUNFUNDED_AMM)); + + auto const lptoken = ammAlice.getLPTokensBalance(alice); + // lock all LP tokens + env(escrow(alice, bob, STAmount{lptoken, ammAlice.lptIssue()}), + finish_time(env.now() + 1s)); + env.close(); + + // Withdraw + ammAlice.withdraw( + alice, USD(1000), std::nullopt, std::nullopt, ter(tecAMM_BALANCE)); + ammAlice.withdrawAll(alice, USD(1000), ter(tecAMM_BALANCE)); + + env(ammAlice.bid(BidArg{ + .account = alice, + .bidMax = 100, + .assets = {{USD, XRP}}, + }), + ter(tecAMM_INVALID_TOKENS)); + + ammAlice.vote( + alice, + 1'000, + std::nullopt, + std::nullopt, + {{USD, XRP}}, + ter(tecAMM_INVALID_TOKENS)); + + // Cannot escrow clawbackable tokens, so we cannot ammClawback escrowed + // tokens + // env(amm::ammClawback(gw, alice, USD, XRP, USD(100)), + // ter(tecAMM_BALANCE)); + // env(amm::ammClawback(gw, alice, USD, XRP, std::nullopt), + // ter(tecAMM_BALANCE)); + } + static uint256 getEscrowIndex(AccountID const& account, std::uint32_t uSequence) { @@ -4729,6 +4817,7 @@ struct Escrow_test : public beast::unit_test::suite testIOUTLINSF(features); testIOUPrecisionLoss(features); testIOUClawback(features); + testIOUAMM(features); } public: diff --git a/src/test/app/HookAPI_test.cpp b/src/test/app/HookAPI_test.cpp index c50cef346..131520dbe 100644 --- a/src/test/app/HookAPI_test.cpp +++ b/src/test/app/HookAPI_test.cpp @@ -50,6 +50,7 @@ private: } public: + using enum hook_api::hook_return_code; void test_accept(FeatureBitset features) { @@ -3631,8 +3632,9 @@ public: auto const alice = Account{"alice"}; auto const bob = Account{"bob"}; + auto const claire = Account{"claire"}; Env env{*this, features}; - env.fund(XRP(100000), alice, bob); + env.fund(XRP(100000), alice, bob, claire); env.close(); // Compute hook hash for the accept hook @@ -3772,6 +3774,84 @@ public: BEAST_EXPECT(result2.has_value()); BEAST_EXPECT(result2.value() == newData.size()); } + + { + // fixHookMap: foreign state set without grant after state previous + // modified + + HookStateMap stateMap; + auto hookCtx = makeStubHookContext( + applyCtx, alice.id(), bob.id(), {}, stateMap); + + AccountID const aliceid = alice.id(); + + // Pre-populate stateMap + stateMap[alice.id()] = { + 100, // availableForReserves + 1, // namespaceCount + 1, // hookStateScale + {}}; + + auto& api = hookCtx.api(); + + // setup a hook on alice, and on claire, no grants + env(hook(alice, {{hso(genesis::AcceptHook)}}, 0), fee(XRP(1))); + env(hook(claire, {{hso(genesis::AcceptHook)}}, 0), fee(XRP(1))); + env.close(); + + // First modification + auto result1 = + api.state_foreign_set(testKey, testNs, aliceid, testData); + BEAST_EXPECT(result1.has_value()); + + // Second modification this time using bob as hookacc (should hit + // cache) + auto hookCtx2 = makeStubHookContext( + applyCtx, claire.id(), bob.id(), {}, stateMap); + + // check the state entry is carried into the second context + + // does the map contain the account? + BEAST_EXPECT( + hookCtx2.result.stateMap.find(aliceid) != + hookCtx2.result.stateMap.end()); + + // the name space? + BEAST_EXPECT( + std::get<3>(hookCtx2.result.stateMap[aliceid]).find(testNs) != + std::get<3>(hookCtx2.result.stateMap[aliceid]).end()); + + // the key entry? + BEAST_EXPECT( + std::get<3>(hookCtx2.result.stateMap[aliceid])[testNs].find( + testKey) != + std::get<3>(hookCtx2.result.stateMap[aliceid])[testNs].end()); + + // is the entry marked as modified? + BEAST_EXPECT( + std::get<3>(hookCtx2.result.stateMap[aliceid])[testNs][testKey] + .first); + + auto& api2 = hookCtx2.api(); + Bytes newData{0x04, 0x05}; + auto result2 = + api2.state_foreign_set(testKey, testNs, aliceid, newData); + + if (features[fixHookMap]) + { + // new behaviour: grant is missing, cannot write + BEAST_EXPECT(!result2.has_value()); + BEAST_EXPECT(result2.error() == NOT_AUTHORIZED); + BEAST_EXPECT(hookCtx2.result.foreignStateSetDisabled); + } + else + { + // old behaviour: allow this illegal write due to the entry + // being modified previously in the map + BEAST_EXPECT(result2.has_value()); + BEAST_EXPECT(result2.value() == newData.size()); + } + } } void @@ -4513,6 +4593,249 @@ public: BEAST_EXPECT(ok.value()); } + void + testHookFloat(FeatureBitset features) + { + // testcase("Test hook float"); + using namespace jtx; + using namespace hook::hook_float; + + { + // get_exponent + testcase << "get_exponent"; + BEAST_EXPECT(get_exponent(-1).error() == INVALID_FLOAT); + BEAST_EXPECT(get_exponent(0).value() == 0); + // 1234567891000000 * 10^(-5) + BEAST_EXPECT(get_exponent(6270245249190730432).value() == -5); + } + + { + // get_mantissa + testcase << "get_mantissa"; + BEAST_EXPECT(get_mantissa(-1).error() == INVALID_FLOAT); + BEAST_EXPECT(get_mantissa(0).value() == 0); + // 1234567891000000 * 10^(-5) + BEAST_EXPECT( + get_mantissa(6270245249190730432).value() == 1234567891000000); + } + + { + // is_negative + testcase << "is_negative"; + // 1234567891000000 * 10^(-5) + BEAST_EXPECT(is_negative(6270245249190730432) == false); + // -1234567891000000 * 10^(-5) + BEAST_EXPECT(is_negative(1658559230763342528) == true); + } + + { + // invert_sign + testcase << "invert_sign"; + BEAST_EXPECT( + invert_sign(6270245249190730432) == 1658559230763342528); + BEAST_EXPECT( + invert_sign(1658559230763342528) == 6270245249190730432); + } + + { + // set_sign + testcase << "set_sign"; + BEAST_EXPECT( + set_sign(6270245249190730432, true) == 1658559230763342528); + BEAST_EXPECT( + set_sign(1658559230763342528, false) == 6270245249190730432); + BEAST_EXPECT( + set_sign(6270245249190730432, false) == 6270245249190730432); + BEAST_EXPECT( + set_sign(1658559230763342528, true) == 1658559230763342528); + } + + { + // set_mantissa + testcase << "set_mantissa"; + + BEAST_EXPECT( + set_mantissa(6270245249190730432, maxMantissa + 1).error() == + MANTISSA_OVERSIZED); + BEAST_EXPECT( + set_mantissa(6270245249190730432, minMantissa - 1).error() == + MANTISSA_UNDERSIZED); + + BEAST_EXPECT( + set_mantissa(6270245249190730432, 1234567891000000).value() == + 6270245249190730432); + BEAST_EXPECT( + set_mantissa(1658559230763342528, 1234567891000000).value() == + 1658559230763342528); + + BEAST_EXPECT( + set_mantissa(6270245249190730432, 1098765432100000).value() == + 6270109446731830432); + BEAST_EXPECT( + set_mantissa(1658559230763342528, 1098765432100000).value() == + 1658423428304442528); + } + + { + // set_exponent + testcase << "set_exponent"; + BEAST_EXPECT( + set_exponent(6270245249190730432, maxExponent + 1).error() == + EXPONENT_OVERSIZED); + BEAST_EXPECT( + set_exponent(6270245249190730432, minExponent - 1).error() == + EXPONENT_UNDERSIZED); + + BEAST_EXPECT( + set_exponent(6270245249190730432, 40).value() == + 7080893182117419712); + BEAST_EXPECT( + set_exponent(6270245249190730432, -40).value() == + 5639741301358860992); + } + + { + // make_float + testcase << "make_float"; + + BEAST_EXPECT(make_float(0, -5, false).value() == 0); + + // invalid mantissa + BEAST_EXPECT( + make_float(maxMantissa + 1, -5, false).error() == + MANTISSA_OVERSIZED); + BEAST_EXPECT( + make_float(minMantissa - 1, -5, false).error() == + MANTISSA_UNDERSIZED); + + // invalid exponent + BEAST_EXPECT( + make_float(1234567891000000, maxExponent + 1, false).error() == + EXPONENT_OVERSIZED); + BEAST_EXPECT( + make_float(1234567891000000, minExponent - 1, false).error() == + EXPONENT_UNDERSIZED); + + BEAST_EXPECT( + make_float(1234567891000000, -5, false).value() == + 6270245249190730432); + BEAST_EXPECT( + make_float(1234567891000000, -5, true).value() == + 1658559230763342528); + } + + { + // normalize_xfl + testcase << "normalize_xfl"; + + { + // zero + uint64_t mantissa = 0; + int32_t exponent = -5; + + BEAST_EXPECT( + normalize_xfl(mantissa, exponent, false).value() == 0); + BEAST_EXPECT(mantissa == 0); + BEAST_EXPECT(exponent == -5); + } + + { + // scale the mantissa up into the canonical range + uint64_t mantissa = 12345; + int32_t exponent = 0; + + BEAST_EXPECT( + normalize_xfl(mantissa, exponent, false).value() == + make_float(1234500000000000ULL, -11, false).value()); + BEAST_EXPECT(mantissa == 1234500000000000ULL); + BEAST_EXPECT(exponent == -11); + } + + { + // scale the mantissa down into the canonical range + uint64_t mantissa = 1234567891000000000ULL; + int32_t exponent = -5; + + BEAST_EXPECT( + normalize_xfl(mantissa, exponent, false).value() == + make_float(1234567891000000ULL, -2, false).value()); + BEAST_EXPECT(mantissa == 1234567891000000ULL); + BEAST_EXPECT(exponent == -2); + } + + { + // explicit negative sign + uint64_t mantissa = 12345; + int32_t exponent = 0; + + BEAST_EXPECT( + normalize_xfl(mantissa, exponent, true).value() == + make_float(1234500000000000ULL, -11, true).value()); + BEAST_EXPECT(mantissa == 1234500000000000ULL); + BEAST_EXPECT(exponent == -11); + } + + { + // signed negative mantissa + int64_t mantissa = -12345; + int32_t exponent = 0; + + BEAST_EXPECT( + normalize_xfl(mantissa, exponent).value() == + make_float(1234500000000000ULL, -11, true).value()); + BEAST_EXPECT(mantissa == -1234500000000000LL); + BEAST_EXPECT(exponent == -11); + } + + { + // signed minimum is adjusted before taking the absolute value + int64_t mantissa = std::numeric_limits::min(); + int32_t exponent = 0; + + BEAST_EXPECT( + normalize_xfl(mantissa, exponent).value() == + make_float(9223372036854775ULL, 3, true).value()); + BEAST_EXPECT(mantissa == -9223372036854775LL); + BEAST_EXPECT(exponent == 3); + } + + { + // one below the minimum mantissa is rounded into range + uint64_t mantissa = minMantissa - 1; + int32_t exponent = -5; + + BEAST_EXPECT( + normalize_xfl(mantissa, exponent, false).value() == + make_float(minMantissa, -5, false).value()); + BEAST_EXPECT(mantissa == minMantissa); + BEAST_EXPECT(exponent == -5); + } + + { + // underflow returns canonical zero + uint64_t mantissa = 1; + int32_t exponent = minExponent; + + BEAST_EXPECT( + normalize_xfl(mantissa, exponent, false).value() == 0); + BEAST_EXPECT(mantissa == 0); + BEAST_EXPECT(exponent == 0); + } + + { + // exponent overflow is reported + uint64_t mantissa = minMantissa; + int32_t exponent = maxExponent + 1; + + BEAST_EXPECT( + normalize_xfl(mantissa, exponent, false).error() == + XFL_OVERFLOW); + BEAST_EXPECT(mantissa == minMantissa); + BEAST_EXPECT(exponent == maxExponent + 1); + } + } + } + void testWithFeatures(FeatureBitset features) { @@ -4588,6 +4911,7 @@ public: test_state(features); test_state_foreign(features); + test_state_foreign_set(features - fixHookMap); test_state_foreign_set(features); test_state_foreign_set_max(features); test_state_set(features); @@ -4615,6 +4939,7 @@ public: { using namespace test::jtx; testWithFeatures(supported_amendments()); + testHookFloat(supported_amendments()); } }; diff --git a/src/test/app/Import_test.cpp b/src/test/app/Import_test.cpp index 96dbddfe5..bc146cc03 100644 --- a/src/test/app/Import_test.cpp +++ b/src/test/app/Import_test.cpp @@ -2667,8 +2667,9 @@ class Import_test : public beast::unit_test::suite } for (bool const withFixImportIssuer : {true, false}) { + auto const withAMM = features | featureAMM; auto const amend = - withFixImportIssuer ? features : features - fixImportIssuer; + withFixImportIssuer ? withAMM : withAMM - fixImportIssuer; test::jtx::Env env{ *this, network::makeNetworkVLConfig(21337, keys), amend}; env.fund(XRP(1000), alice, issuer); diff --git a/src/test/app/Invoke_test.cpp b/src/test/app/Invoke_test.cpp index 703237fe5..09801b9ae 100644 --- a/src/test/app/Invoke_test.cpp +++ b/src/test/app/Invoke_test.cpp @@ -173,7 +173,10 @@ class Invoke_test : public beast::unit_test::suite // tecNO_PERMISSION // issuer is an AMM account { - test::jtx::Env env{*this, network::makeNetworkConfig(21337)}; + test::jtx::Env env{ + *this, + network::makeNetworkConfig(21337), + features | featureAMM}; auto const alice = Account("alice"); auto const issuer = Account("issuer"); diff --git a/src/test/app/LPTokenTransfer_test.cpp b/src/test/app/LPTokenTransfer_test.cpp index 5fd505b5f..9fe7996f8 100644 --- a/src/test/app/LPTokenTransfer_test.cpp +++ b/src/test/app/LPTokenTransfer_test.cpp @@ -388,7 +388,7 @@ public: void run() override { - FeatureBitset const all{jtx::supported_amendments()}; + FeatureBitset const all{jtx::supported_amendments() | featureAMM}; testDirectStep(all); testBookStep(all); diff --git a/src/test/app/LedgerMaster_test.cpp b/src/test/app/LedgerMaster_test.cpp index 8a1f90857..eaa2100c0 100644 --- a/src/test/app/LedgerMaster_test.cpp +++ b/src/test/app/LedgerMaster_test.cpp @@ -20,6 +20,8 @@ #include #include #include +#include +#include #include namespace ripple { @@ -117,13 +119,224 @@ class LedgerMaster_test : public beast::unit_test::suite } } + void + testCanSkipPinnedGap() + { + // detail::canSkipPinnedGap is the lifted decision used by + // findNewLedgersToPublish to decide whether pubSeq can leap + // across a gap in the publish-target set without violating + // contiguous publication of non-pinned ledgers. In production + // this guard rarely fires (pinned ranges are old historical + // catalogue data, pubSeq is at the live tip) — it matters in + // test/standalone catalogue-load scenarios where pinned seqs + // can sit near pubSeq. The invariant is small and crisp; + // unit-testing it directly is far cheaper than reconstructing + // a believable findNewLedgersToPublish state through Env. + testcase("detail::canSkipPinnedGap"); + + using detail::canSkipPinnedGap; + + // No gap (pubSeq is already at intervalStart) — always OK. + { + RangeSet empty; + BEAST_EXPECT(canSkipPinnedGap(10, 10, empty)); + BEAST_EXPECT(canSkipPinnedGap(10, 5, empty)); // past + } + + // One-element gap, exactly pinned: can skip. + { + RangeSet pinned; + pinned.insert(range(10u, 10u)); + BEAST_EXPECT(canSkipPinnedGap(10, 11, pinned)); + } + + // One-element gap, NOT pinned: must halt. + { + RangeSet empty; + BEAST_EXPECT(!canSkipPinnedGap(10, 11, empty)); + } + + // Multi-element gap entirely covered by pinned: can skip. + { + RangeSet pinned; + pinned.insert(range(11u, 15u)); + BEAST_EXPECT(canSkipPinnedGap(11, 16, pinned)); + } + + // Multi-element gap, pinned covers most but leaves one + // non-pinned seq exposed: must halt. This is the bug the + // guard exists to prevent — silently publishing seq 16 + // while seq 15 (non-pinned) is unfetched. + { + RangeSet pinned; + pinned.insert(range(11u, 14u)); // missing 15 + BEAST_EXPECT(!canSkipPinnedGap(11, 16, pinned)); + } + + // Pinned set extends beyond the gap — only the gap window + // matters for the decision. + { + RangeSet pinned; + pinned.insert(range(0u, 1000u)); + BEAST_EXPECT(canSkipPinnedGap(50, 100, pinned)); + } + + // Empty pinned, with a gap: must halt. + { + RangeSet empty; + BEAST_EXPECT(!canSkipPinnedGap(50, 100, empty)); + } + + // Disjoint pinned that doesn't intersect the gap: must halt. + { + RangeSet pinned; + pinned.insert(range(200u, 300u)); + BEAST_EXPECT(!canSkipPinnedGap(50, 100, pinned)); + } + + // Pinned partially intersects gap from the left only. + { + RangeSet pinned; + pinned.insert(range(50u, 75u)); + BEAST_EXPECT(!canSkipPinnedGap(50, 100, pinned)); + } + + // Pinned partially intersects gap from the right only. + { + RangeSet pinned; + pinned.insert(range(75u, 99u)); + BEAST_EXPECT(!canSkipPinnedGap(50, 100, pinned)); + } + } + + void + testPinUnpinSymmetry() + { + testcase("pin/unpin symmetry"); + + using namespace test::jtx; + Env env{*this, makeNetworkConfig(11111)}; + + auto const alice = Account("alice"); + env.fund(XRP(1000), alice); + env.close(); + + auto& lm = env.app().getLedgerMaster(); + + // The most recently closed ledger is the one we'll pin. + auto const ledger = lm.getClosedLedger(); + BEAST_EXPECT(ledger); + if (!ledger) + return; + auto const seq = ledger->info().seq; + BEAST_EXPECT(seq > 0); + + // Baseline — not pinned. + BEAST_EXPECT(!lm.isPinned(seq)); + + // Pin via storeLedger(ledger, pin=true). This is the same path + // catalogue_load uses just before saveValidatedLedger, which is why + // a save failure must roll back via unpinLedger() to keep the + // in-memory pinned-set consistent with state.db. + lm.storeLedger(ledger, /*pin=*/true); + BEAST_EXPECT(lm.isPinned(seq)); + BEAST_EXPECT(boost::icl::contains(lm.getPinnedLedgersRangeSet(), seq)); + + // Unpin reverses the pin (the rollback contract used by + // catalogue_load's RAII guard on save failure). + lm.unpinLedger(seq); + BEAST_EXPECT(!lm.isPinned(seq)); + BEAST_EXPECT(!boost::icl::contains(lm.getPinnedLedgersRangeSet(), seq)); + + // Unpinning a seq that isn't pinned is a no-op (idempotent). + lm.unpinLedger(seq); + BEAST_EXPECT(!lm.isPinned(seq)); + } + + void + testSetPinnedRangesImmediateMerge() + { + // Regression test: when setPinnedLedgersRangeSet runs and + // mCompleteLedgers is already non-empty (LOAD/standalone startup + // already called setFullLedger), the deferred merge in + // setFullLedger never fires because the one-shot guard relies on + // a flag, not on emptiness. Without the immediate-merge branch + // in setPinnedLedgersRangeSet, pinned ranges restored from + // state.db would never appear in mCompleteLedgers and historical + // pinned ledgers would not be queryable until the next + // validation. This test asserts the immediate-merge behavior. + testcase("setPinnedLedgersRangeSet immediate merge"); + + using namespace test::jtx; + Env env{*this, makeNetworkConfig(11111)}; + + auto const alice = Account("alice"); + env.fund(XRP(1000), alice); + env.close(); + + auto& lm = env.app().getLedgerMaster(); + + // Why mCompleteLedgers is non-empty here: + // env.close() above runs synchronous standalone consensus. + // That hits LedgerMaster::switchLCL, which (in standalone) + // calls setFullLedger directly on the calling thread, which + // inserts the closed ledger's seq into mCompleteLedgers. + // So by the time we get here, mCompleteLedgers has the + // genesis + funded seqs in it. + // + // This is the precondition that triggers the immediate-merge + // branch in setPinnedLedgersRangeSet (the LOAD/standalone + // startup case). On a NORMAL/NETWORK production startup, + // mCompleteLedgers would still be empty when state.db is read + // and setPinnedLedgersRangeSet is called, and the merge would + // be deferred to the first validated ledger via setFullLedger. + // We're specifically testing the *other* branch here. + auto const completeBefore = lm.getCompleteLedgersRangeSet(); + BEAST_EXPECT(!completeBefore.empty()); + + // Pinned ranges are far outside the existing range so we can + // unambiguously detect them being merged in. + std::uint32_t const pinLo = 100000; + std::uint32_t const pinHi = 100099; + RangeSet pinned; + pinned.insert(range(pinLo, pinHi)); + + // Restore pinned ranges (mirrors what SHAMapStoreImp::start does + // after reading state.db's PinnedLedgers row). + lm.setPinnedLedgersRangeSet(pinned); + + // Pinned set is restored. + BEAST_EXPECT(lm.isPinned(pinLo)); + BEAST_EXPECT(lm.isPinned(pinHi)); + BEAST_EXPECT(lm.isPinned((pinLo + pinHi) / 2)); + + // Critical invariant: pinned ranges are now in mCompleteLedgers + // immediately, not deferred to the first validated ledger. + // This is what makes haveLedger() return true for pinned seqs + // after a LOAD/standalone restart, without which the historical + // pinned data is on disk but the system reports "we don't have + // it". + BEAST_EXPECT(lm.haveLedger(pinLo)); + BEAST_EXPECT(lm.haveLedger(pinHi)); + BEAST_EXPECT(lm.haveLedger((pinLo + pinHi) / 2)); + + // The pre-existing complete range is still complete (merge, + // not replace). + for (auto const& interval : completeBefore) + for (auto s = interval.lower(); s <= interval.upper(); ++s) + BEAST_EXPECT(lm.haveLedger(s)); + } + public: void run() override { using namespace test::jtx; FeatureBitset const all{supported_amendments() - featureXahauGenesis}; + testCanSkipPinnedGap(); testWithFeats(all); + testPinUnpinSymmetry(); + testSetPinnedRangesImmediateMerge(); } void diff --git a/src/test/app/MPToken_test.cpp b/src/test/app/MPToken_test.cpp index 9c843586d..0d89008ae 100644 --- a/src/test/app/MPToken_test.cpp +++ b/src/test/app/MPToken_test.cpp @@ -1878,6 +1878,13 @@ class MPToken_test : public beast::unit_test::suite [jss::Amount] = mpt.getJson(JsonOptions::none); test(jv, sfAmounts.jsonName.c_str()); } + // ClaimReward + { + Json::Value jv = reward::claim(alice); + jv[sfClaimCurrency.jsonName][jss::mpt_issuance_id] = + to_string(issue); + test(jv, sfClaimCurrency.jsonName.c_str()); + } } for (const auto& str : txWithAmounts) printf("%s\n", str.c_str()); diff --git a/src/test/app/Oracle_test.cpp b/src/test/app/Oracle_test.cpp index 44eeb1c9f..42fc81841 100644 --- a/src/test/app/Oracle_test.cpp +++ b/src/test/app/Oracle_test.cpp @@ -568,6 +568,46 @@ private: Oracle oracle(env, {.owner = owner}); oracle.set(UpdateArg{.series = {{"XRP", "USD", 742, 2}}}); } + + // Should be same order as creation + Env env(*this); + auto const baseFee = + static_cast(env.current()->fees().base.drops()); + + auto test = [&](Env& env, DataSeries const& series) { + env.fund(XRP(1'000), owner); + Oracle oracle( + env, {.owner = owner, .series = series, .fee = baseFee}); + BEAST_EXPECT(oracle.exists()); + auto sle = env.le(keylet::oracle(owner, oracle.documentID())); + BEAST_EXPECT( + sle->getFieldArray(sfPriceDataSeries).size() == series.size()); + + auto const beforeQuoteAssetName1 = + sle->getFieldArray(sfPriceDataSeries)[0] + .getFieldCurrency(sfQuoteAsset) + .getText(); + auto const beforeQuoteAssetName2 = + sle->getFieldArray(sfPriceDataSeries)[1] + .getFieldCurrency(sfQuoteAsset) + .getText(); + + oracle.set(UpdateArg{.series = series, .fee = baseFee}); + sle = env.le(keylet::oracle(owner, oracle.documentID())); + + auto const afterQuoteAssetName1 = + sle->getFieldArray(sfPriceDataSeries)[0] + .getFieldCurrency(sfQuoteAsset) + .getText(); + auto const afterQuoteAssetName2 = + sle->getFieldArray(sfPriceDataSeries)[1] + .getFieldCurrency(sfQuoteAsset) + .getText(); + + BEAST_EXPECT(afterQuoteAssetName1 == beforeQuoteAssetName1); + BEAST_EXPECT(afterQuoteAssetName2 == beforeQuoteAssetName2); + }; + test(env, {{"XRP", "USD", 742, 2}, {"XRP", "EUR", 711, 2}}); } void diff --git a/src/test/app/Remit_test.cpp b/src/test/app/Remit_test.cpp index 3825aeb82..7ebe42ebd 100644 --- a/src/test/app/Remit_test.cpp +++ b/src/test/app/Remit_test.cpp @@ -413,7 +413,7 @@ struct Remit_test : public beast::unit_test::suite // tecNO_PERMISSION - inform account is an AMM { - Env env{*this, features}; + Env env{*this, features | featureAMM}; env.fund(XRP(1000), gw, alice, bob); env.close(); @@ -2902,7 +2902,7 @@ struct Remit_test : public beast::unit_test::suite auto const gw = Account("gw"); auto const USD = gw["USD"]; - Env env{*this, features}; + Env env{*this, features | featureAMM}; env.fund(XRP(10'000'000), alice, gw); env.close(); diff --git a/src/test/app/SHAMapStore_test.cpp b/src/test/app/SHAMapStore_test.cpp index 134f72a00..2ea3d2412 100644 --- a/src/test/app/SHAMapStore_test.cpp +++ b/src/test/app/SHAMapStore_test.cpp @@ -22,10 +22,14 @@ #include #include #include +#include +#include #include #include #include +#include #include +#include namespace ripple { namespace test { @@ -51,6 +55,25 @@ class SHAMapStore_test : public beast::unit_test::suite return cfg; } + static auto + pinnedPersistent( + std::unique_ptr cfg, + std::string const& dbPath, + std::string const& nodePath, + std::string const& pinnedPath) + { + cfg = onlineDelete(std::move(cfg)); + cfg->legacy("database_path", dbPath); + cfg->overwrite(SECTION_RELATIONAL_DB, "backend", "sqlite"); + + auto& section = cfg->section(ConfigSection::nodeDatabase()); + section.set("type", "rwdb"); + section.set("path", nodePath); + section.set("pinned_type", "nudb"); + section.set("pinned_path", pinnedPath); + return cfg; + } + bool goodLedger( jtx::Env& env, @@ -648,13 +671,191 @@ public: BEAST_EXPECT(dbr->getName() == "3"); } + void + testPinnedRangeRestoreRequiresPinnedData() + { + testcase("pinned range restore requires pinned data"); + + using namespace jtx; + + beast::temp_dir tempDir; + auto const tempPath = boost::filesystem::path(tempDir.path()); + auto const dbPath = (tempPath / "db").string(); + auto const nodePath = (tempPath / "node").string(); + auto const pinnedPath = (tempPath / "pinned").string(); + + boost::filesystem::create_directories(dbPath); + boost::filesystem::create_directories(nodePath); + boost::filesystem::create_directories(pinnedPath); + + { + Env env( + *this, + envconfig(pinnedPersistent, dbPath, nodePath, pinnedPath)); + + RangeSet ranges; + ranges.insert(range(100u, 200u)); + env.app().getSHAMapStore().setPinnedRanges(ranges); + + NodeStoreScheduler scheduler(env.app().getJobQueue()); + auto const journal = env.app().journal("SHAMapStoreTest"); + + try + { + SHAMapStoreImp restoreCheck(env.app(), scheduler, journal); + restoreCheck.start(); + fail( + "Expected startup failure for stale persisted pinned " + "ranges"); + } + catch (std::runtime_error const& e) + { + BEAST_EXPECT( + std::string(e.what()).find( + "Persisted pinned interval 100-200") != + std::string::npos); + } + } + } + + // Helper: compare two RangeSet for set-equality. The + // test cares about which sequences are in the result, not the + // particular interval representation. + static bool + sameRanges( + RangeSet const& a, + RangeSet const& b) + { + return a == b; + } + + void + testComputeOnlineDeleteTargets() + { + // detail::computeOnlineDeleteTargets is the lifted interval + // arithmetic used by SHAMapStoreImp::clearSqlRanges to decide + // which ledger sequences to actually delete during online + // delete. The base window is [minSeq, lastRotated - 1] and + // pinned ranges are subtracted from it. The result must: + // (a) be empty when the base is empty or fully pinned, + // (b) preserve all pinned seqs (they survive rotation), + // (c) cover every non-pinned seq in the base. + // Lifting the math out keeps the contract testable without + // standing up a full SHAMapStoreImp + database fixture. + testcase("detail::computeOnlineDeleteTargets"); + + using detail::computeOnlineDeleteTargets; + + // Empty base window (minSeq >= lastRotated): no work. + { + RangeSet empty; + BEAST_EXPECT(computeOnlineDeleteTargets(100, 100, empty).empty()); + BEAST_EXPECT(computeOnlineDeleteTargets(101, 100, empty).empty()); + } + + // No pins: the entire base window is the target. + { + RangeSet empty; + RangeSet expected; + expected.insert(range(50u, 99u)); // [50, 100-1] + BEAST_EXPECT(sameRanges( + computeOnlineDeleteTargets(50, 100, empty), expected)); + } + + // Pins fully cover the base window: nothing to delete. This + // is the bug the pinning feature exists to prevent — pinned + // ranges getting silently rotated away. + { + RangeSet pinned; + pinned.insert(range(50u, 99u)); + BEAST_EXPECT(computeOnlineDeleteTargets(50, 100, pinned).empty()); + } + + // Pins extend beyond the base window: still nothing to delete. + { + RangeSet pinned; + pinned.insert(range(0u, 1000u)); + BEAST_EXPECT(computeOnlineDeleteTargets(50, 100, pinned).empty()); + } + + // Pins cover only the leading section of the base. Result is + // the trailing remainder. + { + RangeSet pinned; + pinned.insert(range(50u, 70u)); + RangeSet expected; + expected.insert(range(71u, 99u)); + BEAST_EXPECT(sameRanges( + computeOnlineDeleteTargets(50, 100, pinned), expected)); + } + + // Pins cover only the trailing section. Result is leading. + { + RangeSet pinned; + pinned.insert(range(80u, 99u)); + RangeSet expected; + expected.insert(range(50u, 79u)); + BEAST_EXPECT(sameRanges( + computeOnlineDeleteTargets(50, 100, pinned), expected)); + } + + // Pins cut a hole in the middle. Result is two disjoint + // intervals — both get deleted, the hole is preserved. + { + RangeSet pinned; + pinned.insert(range(60u, 80u)); + RangeSet expected; + expected.insert(range(50u, 59u)); + expected.insert(range(81u, 99u)); + BEAST_EXPECT(sameRanges( + computeOnlineDeleteTargets(50, 100, pinned), expected)); + } + + // Multiple disjoint pinned ranges all inside the base. Result + // is the base minus all pinned holes. + { + RangeSet pinned; + pinned.insert(range(55u, 60u)); + pinned.insert(range(70u, 75u)); + pinned.insert(range(90u, 95u)); + RangeSet expected; + expected.insert(range(50u, 54u)); + expected.insert(range(61u, 69u)); + expected.insert(range(76u, 89u)); + expected.insert(range(96u, 99u)); + BEAST_EXPECT(sameRanges( + computeOnlineDeleteTargets(50, 100, pinned), expected)); + } + + // Pinned ranges entirely outside the base window: no effect. + { + RangeSet pinned; + pinned.insert(range(0u, 49u)); + pinned.insert(range(100u, 200u)); + RangeSet expected; + expected.insert(range(50u, 99u)); + BEAST_EXPECT(sameRanges( + computeOnlineDeleteTargets(50, 100, pinned), expected)); + } + + // Single-sequence base window with a single-sequence pin + // matching exactly: empty result. + { + RangeSet pinned; + pinned.insert(range(99u, 99u)); + BEAST_EXPECT(computeOnlineDeleteTargets(99, 100, pinned).empty()); + } + } + void run() override { + testComputeOnlineDeleteTargets(); testClear(); testAutomatic(); testCanDelete(); testRotate(); + testPinnedRangeRestoreRequiresPinnedData(); } }; diff --git a/src/test/app/SetHookTSH_test.cpp b/src/test/app/SetHookTSH_test.cpp index 9cf94e067..f4ef10ed8 100644 --- a/src/test/app/SetHookTSH_test.cpp +++ b/src/test/app/SetHookTSH_test.cpp @@ -878,7 +878,7 @@ private: test::jtx::Env env{ *this, network::makeNetworkConfig(21337, "10", "1000000", "200000"), - features}; + features | featureAMM}; auto const issuer = Account("gw"); auto const account = Account("alice"); @@ -928,7 +928,7 @@ private: test::jtx::Env env{ *this, network::makeNetworkConfig(21337, "10", "1000000", "200000"), - features}; + features | featureAMM}; auto const issuer = Account("gw"); auto const account = Account("alice"); @@ -983,7 +983,7 @@ private: cfg->FEES.reference_fee = XRPAmount(1); return cfg; }), - features); + features | featureAMM); auto const issuer = Account("gw"); auto const account = Account("alice"); @@ -1043,7 +1043,7 @@ private: test::jtx::Env env{ *this, network::makeNetworkConfig(21337, "10", "1000000", "200000"), - features}; + features | featureAMM | featureAMMClawback}; auto const issuer = Account("gw"); auto const account = Account("alice"); @@ -1101,7 +1101,7 @@ private: test::jtx::Env env{ *this, network::makeNetworkConfig(21337, "10", "1000000", "200000"), - features}; + features | featureAMM}; auto const issuer = Account("gw"); auto const account = Account("alice"); @@ -1156,7 +1156,7 @@ private: test::jtx::Env env{ *this, network::makeNetworkConfig(21337, "10", "1000000", "200000"), - features}; + features | featureAMM}; auto const issuer = Account("gw"); auto const account = Account("alice"); @@ -1203,7 +1203,7 @@ private: test::jtx::Env env{ *this, network::makeNetworkConfig(21337, "10", "1000000", "200000"), - features}; + features | featureAMM}; auto const issuer = Account("gw"); auto const account = Account("alice"); @@ -1647,8 +1647,12 @@ private: features}; auto const account = Account("alice"); - auto const issuer = Account("issuer"); - env.fund(XRP(1000), account, issuer); + auto const issuer = Account::master; + env.fund(XRP(1000), account); + env.close(); + + env(hook(issuer, {{hso(jtx::genesis::AcceptHook)}}, 0), + fee(XRP(1))); env.close(); // set tsh collect @@ -1680,8 +1684,8 @@ private: features}; auto const account = Account("alice"); - auto const issuer = Account("issuer"); - env.fund(XRP(1000), account, issuer); + auto const issuer = Account::master; + env.fund(XRP(1000), account); env.close(); // set tsh collect diff --git a/src/test/app/SetHook_test.cpp b/src/test/app/SetHook_test.cpp index b36d18989..8c60f3862 100644 --- a/src/test/app/SetHook_test.cpp +++ b/src/test/app/SetHook_test.cpp @@ -690,6 +690,8 @@ public: bool const hasHookCanEmit = env.current()->rules().enabled(featureHookCanEmit); + bool const hasNamedHooks = + env.current()->rules().enabled(featureNamedHooks); auto const alice = Account{"alice"}; env.fund(XRP(10000), alice); @@ -726,7 +728,8 @@ public: } // grants, parameters, hookon, hookonincoming, hookonoutgoing, - // hookcanemit, hookapiversion, hooknamespace keys must be absent + // hookcanemit, hookapiversion, hooknamespace, hookname keys must be + // absent for (auto const& [key, value] : JSSMap{ {jss::HookGrants, Json::arrayValue}, {jss::HookParameters, Json::arrayValue}, @@ -743,11 +746,16 @@ public: "000000000000000000000000000000000000000000000000000000000000" "0000"}, {jss::HookApiVersion, "0"}, - {jss::HookNamespace, to_string(uint256{beast::zero})}}) + {jss::HookNamespace, to_string(uint256{beast::zero})}, + {jss::HookName, strHex(std::string{"DEADBEEF"})}, + }) { if (!hasHookCanEmit && key == jss::HookCanEmit) continue; + if (!hasNamedHooks && key == jss::HookName) + continue; + Json::Value iv; iv[jss::CreateCode] = ""; iv[key] = value; @@ -755,7 +763,7 @@ public: env(jv, M("Hook DELETE operation cannot include: grants, params, " "hookon, HookOnIncoming, HookOnOutgoing, hookcanemit, " - "apiversion, namespace"), + "apiversion, namespace, hookname"), HSFEE, ter(temMALFORMED)); env.close(); @@ -894,6 +902,8 @@ public: bool const fixNS = env.current()->rules().enabled(fixNSDelete); bool const hasHookCanEmit = env.current()->rules().enabled(featureHookCanEmit); + bool const hasNamedHooks = + env.current()->rules().enabled(featureNamedHooks); auto const alice = Account{"alice"}; env.fund(XRP(10000), alice); @@ -926,11 +936,15 @@ public: "000000000000000000000000000000000000000000000000000000000000" "0000"}, {jss::HookApiVersion, "0"}, + {jss::HookName, strHex(std::string{"DEADBEEF"})}, }) { if (!hasHookCanEmit && key == jss::HookCanEmit) continue; + if (!hasNamedHooks && key == jss::HookName) + continue; + Json::Value iv; iv[key] = value; iv[jss::Flags] = hsfNSDELETE; @@ -939,7 +953,7 @@ public: env(jv, M("Hook NSDELETE operation cannot include: grants, params, " "hookon, hookonincoming, hookonoutgoing, hookcanemit, " - "apiversion"), + "apiversion, hookname"), HSFEE, ter(temMALFORMED)); env.close(); @@ -1684,6 +1698,340 @@ public: } } + void + testHookName(FeatureBitset features) + { + testcase("Test hook name"); + using namespace jtx; + + auto const alice = Account{"alice"}; + auto const bob = Account{"bob"}; + auto const charlie = Account{"charlie"}; + auto const USD = alice["USD"]; + + Env env{*this, features}; + + env.fund(XRP(10000), alice, bob); + env.close(); + + // Invalid hook name (length=3,17, not utf-8) + for (auto const name : { + "414243", // ABC (length=3) + "4142434445464748494A4B4C4D4E4F5051", // ABCDEFGHIJKLMNOPQ + // (length=17) + "DEADBEEF", // not utf-8 + }) + { + auto jvh = hso(accept_wasm); + jvh[jss::HookName] = name; + env(ripple::test::jtx::hook(alice, {{jvh}}, 0), + M("Hook name must be between 8 and 32 hex characters and be a " + "valid UTF-8 string"), + HSFEE, + features[featureNamedHooks] ? ter(temMALFORMED) + : ter(temDISABLED)); + + auto jvi = invoke::invoke(alice); + jvi[jss::HookName] = name; + env(jvi, + M("Call named hook with the invalid hook name"), + HSFEE, + ter(temMALFORMED)); + } + + if (!features[featureNamedHooks]) + return; + + { + /// Create, Install, Update named Hook + Env env{*this, features}; + + env.fund(XRP(10000), alice, bob, charlie); + env.close(); + + // Create with hook name + auto jvh = hso(accept_wasm); + jvh[jss::HookName] = "41424344"; + env(ripple::test::jtx::hook(alice, {{jvh}}, 0), + M("Create named hook"), + HSFEE); + env.close(); + + // Check hook definition + { + auto const hookDef = + env.le(keylet::hookDefinition(accept_hash)); + BEAST_EXPECT(hookDef); + BEAST_EXPECT(!hookDef->isFieldPresent(sfHookName)); + } + // Check Hook + { + auto const hooks = env.le(keylet::hook(alice)); + BEAST_EXPECT(hooks && hooks->isFieldPresent(sfHooks)); + auto const& hooksArray = hooks->getFieldArray(sfHooks); + BEAST_EXPECT(hooksArray.size() == 1); + BEAST_EXPECT(hooksArray[0].isFieldPresent(sfHookName)); + BEAST_EXPECT( + strHex(hooksArray[0].getFieldVL(sfHookName)) == "41424344"); + } + + // install with hook name + jvh[jss::HookName] = "41424344"; + env(ripple::test::jtx::hook(bob, {{jvh}}, 0), + M("Install named hook"), + HSFEE); + env.close(); + // Check Hook + { + auto const hooks = env.le(keylet::hook(bob)); + BEAST_EXPECT(hooks && hooks->isFieldPresent(sfHooks)); + auto const& hooksArray = hooks->getFieldArray(sfHooks); + BEAST_EXPECT(hooksArray.size() == 1); + BEAST_EXPECT(hooksArray[0].isFieldPresent(sfHookName)); + BEAST_EXPECT( + strHex(hooksArray[0].getFieldVL(sfHookName)) == "41424344"); + } + + // install without hook name + jvh.removeMember(jss::HookName); + env(ripple::test::jtx::hook(charlie, {{jvh}}, 0), + M("Install non-named hook"), + HSFEE); + env.close(); + // Check Hook + { + auto const hooks = env.le(keylet::hook(charlie)); + BEAST_EXPECT(hooks && hooks->isFieldPresent(sfHooks)); + auto const& hooksArray = hooks->getFieldArray(sfHooks); + BEAST_EXPECT(hooksArray.size() == 1); + BEAST_EXPECT(!hooksArray[0].isFieldPresent(sfHookName)); + } + + // Update named hook to non-named hook + jvh[jss::HookName] = ""; + jvh[jss::Flags] = hsfOVERRIDE; + env(ripple::test::jtx::hook(alice, {{jvh}}, 0), + M("Update named hook to non-named hook"), + HSFEE); + env.close(); + // Check Hook + { + auto const hooks = env.le(keylet::hook(alice)); + BEAST_EXPECT(hooks && hooks->isFieldPresent(sfHooks)); + auto const& hooksArray = hooks->getFieldArray(sfHooks); + BEAST_EXPECT(hooksArray.size() == 1); + BEAST_EXPECT(!hooksArray[0].isFieldPresent(sfHookName)); + } + + // Update named hook to named hook + jvh[jss::HookName] = "4142434445"; + jvh[jss::Flags] = hsfOVERRIDE; + env(ripple::test::jtx::hook(bob, {{jvh}}, 0), + M("Update non-named hook to named hook"), + HSFEE); + env.close(); + // Check Hook + { + auto const hooks = env.le(keylet::hook(bob)); + BEAST_EXPECT(hooks && hooks->isFieldPresent(sfHooks)); + auto const& hooksArray = hooks->getFieldArray(sfHooks); + BEAST_EXPECT(hooksArray.size() == 1); + BEAST_EXPECT(hooksArray[0].isFieldPresent(sfHookName)); + BEAST_EXPECT( + strHex(hooksArray[0].getFieldVL(sfHookName)) == + "4142434445"); + } + + // Update non-named hook to named hook + jvh[jss::HookName] = "41424344"; + jvh[jss::Flags] = hsfOVERRIDE; + env(ripple::test::jtx::hook(charlie, {{jvh}}, 0), + M("Update non-named hook to named hook"), + HSFEE); + env.close(); + // Check Hook + { + auto const hooks = env.le(keylet::hook(charlie)); + BEAST_EXPECT(hooks && hooks->isFieldPresent(sfHooks)); + auto const& hooksArray = hooks->getFieldArray(sfHooks); + BEAST_EXPECT(hooksArray.size() == 1); + BEAST_EXPECT(hooksArray[0].isFieldPresent(sfHookName)); + BEAST_EXPECT( + strHex(hooksArray[0].getFieldVL(sfHookName)) == "41424344"); + } + } + + // Install named hook + auto jvh = hso(accept_wasm); + jvh[jss::HookName] = "41424344"; + jvh[jss::Flags] = hsfCOLLECT; + auto jvh2 = hso(accept2_wasm); + jvh2[jss::Flags] = hsfCOLLECT; + env(ripple::test::jtx::hook(alice, {{jvh, jvh2}}, 0), + M("Install named hook"), + HSFEE); + env.close(); + + // + // Test Strong + // + // Call named hook without specifying the hook name + { + auto jv = invoke::invoke(alice); + auto expectedFee = + calculateBaseFee(*env.current(), *env.jt(jv).stx); + BEAST_EXPECT(expectedFee == drops(19)); + env(jv, + M("Call named hook without specifying the hook name"), + HSFEE); + env.close(); + // execute only non-named hook + auto const hookExecutions = + env.meta()->getFieldArray(sfHookExecutions); + BEAST_EXPECT(hookExecutions.size() == 1); + BEAST_EXPECT( + hookExecutions[0].getFieldH256(sfHookHash) == accept2_hash); + } + + // Call named hook with the wrong hook name + { + auto jv = invoke::invoke(alice); + jv[jss::HookName] = "41424345"; + auto expectedFee = + calculateBaseFee(*env.current(), *env.jt(jv).stx); + BEAST_EXPECT(expectedFee == drops(19)); + env(jv, M("Call named hook with the wrong hook name"), HSFEE); + env.close(); + // execute only non-named hook + auto const hookExecutions = + env.meta()->getFieldArray(sfHookExecutions); + BEAST_EXPECT(hookExecutions.size() == 1); + BEAST_EXPECT( + hookExecutions[0].getFieldH256(sfHookHash) == accept2_hash); + } + + // Call named hook with the correct hook name + { + auto jv = invoke::invoke(alice); + jv[jss::HookName] = "41424344"; + auto expectedFee = + calculateBaseFee(*env.current(), *env.jt(jv).stx); + BEAST_EXPECT(expectedFee == drops(28)); + env(jv, M("Call named hook with the correct hook name"), HSFEE); + env.close(); + // execute both named and non-named hooks + auto const hookExecutions = + env.meta()->getFieldArray(sfHookExecutions); + BEAST_EXPECT(hookExecutions.size() == 2); + BEAST_EXPECT( + hookExecutions[0].getFieldH256(sfHookHash) == accept_hash); + BEAST_EXPECT( + hookExecutions[1].getFieldH256(sfHookHash) == accept2_hash); + } + + // + // Test Weak + // + env(fset(alice, asfTshCollect), fee(XRP(1))); + env.close(); + // Call named hook without specifying the hook name + { + auto jv = trust(bob, USD(1000)); + env(jv, + M("Call named hook without specifying the hook name"), + HSFEE); + env.close(); + // execute only non-named hook + auto const hookExecutions = + env.meta()->getFieldArray(sfHookExecutions); + BEAST_EXPECT(hookExecutions.size() == 1); + BEAST_EXPECT( + hookExecutions[0].getFieldH256(sfHookHash) == accept2_hash); + } + + // Call named hook with the wrong hook name + { + auto jv = trust(bob, USD(1000)); + jv[jss::HookName] = "41424345"; + env(jv, M("Call named hook with the wrong hook name"), HSFEE); + env.close(); + // execute only non-named hook + auto const hookExecutions = + env.meta()->getFieldArray(sfHookExecutions); + BEAST_EXPECT(hookExecutions.size() == 1); + BEAST_EXPECT( + hookExecutions[0].getFieldH256(sfHookHash) == accept2_hash); + } + + // Call named hook with the correct hook name + { + auto jv = invoke::invoke(alice); + jv[jss::HookName] = "41424344"; + env(jv, M("Call named hook with the correct hook name"), HSFEE); + env.close(); + // execute both named and non-named hooks + auto const hookExecutions = + env.meta()->getFieldArray(sfHookExecutions); + BEAST_EXPECT(hookExecutions.size() == 2); + BEAST_EXPECT( + hookExecutions[0].getFieldH256(sfHookHash) == accept_hash); + BEAST_EXPECT( + hookExecutions[1].getFieldH256(sfHookHash) == accept2_hash); + } + env(fclear(alice, asfTshCollect), fee(XRP(1))); + env.close(); + + // + // Callback Execution + // + jvh = hso(emit_invoke_wasm); + jvh[jss::HookName] = "41424344"; + jvh[jss::Flags] = hsfOVERRIDE; + env(ripple::test::jtx::hook(alice, {{jvh}}, 0), + M("Install named callback hook"), + HSFEE); + env.close(); + // Call named hook without specifying the hook name + { + auto jv = invoke::invoke(alice); + env(jv, + M("Call named hook without specifying the hook name"), + HSFEE); + env.close(); + // execute only non-named hook + BEAST_EXPECT(!env.meta()->isFieldPresent(sfHookEmissions)); + } + + // Call named hook with the wrong hook name + { + auto jv = invoke::invoke(alice); + jv[jss::HookName] = "41424345"; + env(jv, M("Call named hook with the wrong hook name"), HSFEE); + env.close(); + // execute only non-named hook + BEAST_EXPECT(!env.meta()->isFieldPresent(sfHookEmissions)); + } + + // Call named hook with the correct hook name + { + auto jv = invoke::invoke(alice); + jv[jss::HookName] = "41424344"; + env(jv, M("Call named hook with the correct hook name"), HSFEE); + env.close(); + // execute both named and non-named hooks + BEAST_EXPECT(env.meta()->isFieldPresent(sfHookEmissions)); + auto const hookEmissions = + env.meta()->getFieldArray(sfHookEmissions)[0]; + auto const etxn = hookEmissions.getFieldH256(sfEmittedTxnID); + env.close(); + auto const tx = env.closed()->txRead(etxn); + BEAST_EXPECT(tx.first && tx.second); + BEAST_EXPECT(!tx.first->isFieldPresent(sfHookName)); + // Callback transaction doesn't need to have hook name + BEAST_EXPECT(tx.second->isFieldPresent(sfHookExecutions)); + } + } + void testFillCopy(FeatureBitset features) { @@ -1741,6 +2089,8 @@ public: bool const hasHookCanEmit = env.current()->rules().enabled(featureHookCanEmit); + bool const hasNamedHooks = + env.current()->rules().enabled(featureNamedHooks); auto const bob = Account{"bob"}; env.fund(XRP(10000), bob); @@ -1789,6 +2139,8 @@ public: iv[jss::HookCanEmit] = "0000000000000000000000000000000000000000000000000000000000" "000000"; + if (hasNamedHooks) + iv[jss::HookName] = strHex(std::string{"DEADBEEF"}); jv[jss::Hooks][0U] = Json::Value{}; jv[jss::Hooks][0U][jss::Hook] = iv; @@ -1811,6 +2163,8 @@ public: iv[jss::HookCanEmit] = "0000000000000000000000000000000000000000000000000000000000" "000000"; + if (hasNamedHooks) + iv[jss::HookName] = strHex(std::string{"DEADBEEF"}); jv[jss::Hooks][0U] = Json::Value{}; jv[jss::Hooks][0U][jss::Hook] = iv; @@ -1834,6 +2188,8 @@ public: iv[jss::HookCanEmit] = "0000000000000000000000000000000000000000000000000000000000" "000000"; + if (hasNamedHooks) + iv[jss::HookName] = strHex(std::string{"DEADBEEF"}); jv[jss::Hooks][0U] = Json::Value{}; jv[jss::Hooks][0U][jss::Hook] = iv; @@ -1854,6 +2210,8 @@ public: iv[jss::HookCanEmit] = "0000000000000000000000000000000000000000000000000000000000" "000000"; + if (hasNamedHooks) + iv[jss::HookName] = strHex(std::string{"DEADBEEF"}); jv[jss::Hooks][0U] = Json::Value{}; jv[jss::Hooks][0U][jss::Hook] = iv; @@ -2002,6 +2360,8 @@ public: bool const hasHookCanEmit = env.current()->rules().enabled(featureHookCanEmit); + bool const hasNamedHooks = + env.current()->rules().enabled(featureNamedHooks); auto const alice = Account{"alice"}; env.fund(XRP(10000), alice); @@ -2028,6 +2388,8 @@ public: iv[jss::HookCanEmit] = "0000000000000000000000000000000000000000000000000000000000" "000000"; + if (hasNamedHooks) + iv[jss::HookName] = strHex(std::string{"DEADBEEF"}); iv[jss::HookParameters] = Json::Value{Json::arrayValue}; iv[jss::HookParameters][0U] = Json::Value{}; iv[jss::HookParameters][0U][jss::HookParameter] = Json::Value{}; @@ -2116,11 +2478,16 @@ public: "CAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFE" "CAFECAFE"}, {jss::HookParameters, params}, - {jss::HookGrants, grants}}) + {jss::HookGrants, grants}, + {jss::HookName, strHex(std::string{"DEADBEEF"})}, + }) { if (!hasHookCanEmit && key == jss::HookCanEmit) continue; + if (!hasNamedHooks && key == jss::HookName) + continue; + Json::Value iv; iv[key] = value; jv[jss::Hooks][0U] = Json::Value{}; @@ -2192,11 +2559,16 @@ public: {jss::HookCanEmit, "00000000000000000000000000000000000000000000000000000000" "00000000"}, - {jss::HookNamespace, to_string(uint256{beast::zero})}}) + {jss::HookNamespace, to_string(uint256{beast::zero})}, + {jss::HookName, strHex(std::string{"DEADBEEF"})}, + }) { if (key == jss::HookCanEmit && !hasHookCanEmit) continue; + if (!hasNamedHooks && key == jss::HookName) + continue; + Json::Value iv; iv[key] = value; jv[jss::Hooks][0U] = Json::Value{}; @@ -2445,11 +2817,16 @@ public: "CAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFE" "CAFECAFE"}, {jss::HookParameters, params}, - {jss::HookGrants, grants}}) + {jss::HookGrants, grants}, + {jss::HookName, strHex(std::string{"DEADBEEF"})}, + }) { if (key == jss::HookCanEmit && !hasHookCanEmit) continue; + if (!hasNamedHooks && key == jss::HookName) + continue; + Json::Value iv; iv[key] = value; jv[jss::Hooks][0U] = Json::Value{}; @@ -13122,7 +13499,7 @@ public: 0,0 ))); - ASSERT(34 == (e=util_keylet(buf, 34, KEYLET_AMM, + ASSERT(INVALID_ARGUMENT == (e=util_keylet(buf, 34, KEYLET_AMM, SBUF(asset1), SBUF(asset2), 0,0 ))); @@ -14721,6 +15098,7 @@ public: testPageCap(features); testHookOnV2(features); + testHookName(features); testFillCopy(features); @@ -14824,7 +15202,7 @@ public: using namespace test::jtx; static FeatureBitset const all{supported_amendments()}; - static std::array const feats{ + static std::array const feats{ all, all - fixXahauV2, all - fixXahauV1 - fixXahauV2, @@ -14834,6 +15212,7 @@ public: featureHookCanEmit, all - fixXahauV1 - fixXahauV2 - fixNSDelete - fixPageCap - featureExtendedHookState, + all - featureNamedHooks, }; if (BEAST_EXPECT(instance < feats.size())) @@ -14985,6 +15364,98 @@ private: )[test.hook]"]; HASH_WASM(accept2); + + // This hook is used to test Callback + TestHook emit_invoke_wasm = // WASM: 7 + wasm[ + R"[test.hook]( + #include + extern int64_t accept (uint32_t read_ptr, uint32_t read_len, int64_t error_code); + extern int64_t rollback (uint32_t read_ptr, uint32_t read_len, int64_t error_code); + extern int64_t emit (uint32_t write_ptr, uint32_t write_len, uint32_t read_ptr, uint32_t read_len); + extern int64_t hook_account(uint32_t write_ptr, uint32_t write_len); + extern int64_t etxn_reserve(uint32_t); + extern int64_t etxn_fee_base (uint32_t read_ptr, uint32_t read_len); + extern int64_t etxn_details (uint32_t write_ptr, uint32_t write_len); + extern int64_t ledger_seq (void); + + #define SBUF(x) (uint32_t)x,sizeof(x) + // clang-format off + uint8_t txn[229] = + { + /* size, upto, field name */ + /* 3, 0, tt = Invoke */ 0x12U, 0x00U, 0x63U, + /* 5, 3, flags */ 0x22U, 0x00U, 0x00U, 0x00U, 0x00U, + /* 5, 8, sequence */ 0x24U, 0x00U, 0x00U, 0x00U, 0x00U, + /* 6, 13, firstledgersequence */ 0x20U, 0x1AU, 0x00U, 0x00U, 0x00U, 0x00U, + /* 6, 19, lastledgersequence */ 0x20U, 0x1BU, 0x00U, 0x00U, 0x00U, 0x00U, + /* 9, 25, fee */ 0x68U, 0x40U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, + /* 35, 34, signingpubkey */ 0x73U, 0x21U, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + /* 22, 69, account */ 0x81U, 0x14U, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + /* 116, 91, emit details */ + /* 0, 229, */ + }; + // clang-format on + + // TX BUILDER + #define FLAGS_OUT (txn + 4U) + #define FLS_OUT (txn + 15U) + #define LLS_OUT (txn + 21U) + #define FEE_OUT (txn + 26U) + #define ACCOUNT_OUT (txn + 71U) + #define EMIT_OUT (txn + 91U) + + #define FLIP_ENDIAN_32(value) \ + (uint32_t)(((value & 0xFFU) << 24) | ((value & 0xFF00U) << 8) | \ + ((value & 0xFF0000U) >> 8) | ((value & 0xFF000000U) >> 24)) + + #define SET_UINT32(ptr, value) *((uint32_t *)(ptr)) = FLIP_ENDIAN_32(value); + + #define SET_NATIVE_AMOUNT(ptr, amount) \ + do { \ + uint8_t *b = (ptr); \ + *b++ = 0b01000000 + ((amount >> 56) & 0b00111111); \ + *b++ = (amount >> 48) & 0xFFU; \ + *b++ = (amount >> 40) & 0xFFU; \ + *b++ = (amount >> 32) & 0xFFU; \ + *b++ = (amount >> 24) & 0xFFU; \ + *b++ = (amount >> 16) & 0xFFU; \ + *b++ = (amount >> 8) & 0xFFU; \ + *b++ = (amount >> 0) & 0xFFU; \ + } while (0) + + #define PREPARE_TXN() \ + do { \ + etxn_reserve(1); \ + uint32_t fls = (uint32_t)ledger_seq() + 1; \ + SET_UINT32(FLS_OUT, fls); \ + SET_UINT32(LLS_OUT, fls + 4); \ + hook_account(ACCOUNT_OUT, 20); \ + etxn_details(EMIT_OUT, 138U); \ + int64_t fee = etxn_fee_base(SBUF(txn)); \ + SET_NATIVE_AMOUNT(FEE_OUT, fee); \ + } while (0) + + + int64_t cbak(uint32_t r) + { + _g(1,1); + return accept(0,0,0); + } + + int64_t hook(uint32_t reserved) + { + _g(1,1); + PREPARE_TXN(); + uint8_t emithash[32]; + int64_t emit_result = emit(SBUF(emithash), SBUF(txn)); + if (emit_result < 0) + return rollback(SBUF("Emit failed."), __LINE__); + return accept(SBUF("Emit succeeded."), __LINE__); + } + )[test.hook]"]; + + HASH_WASM(emit_invoke); }; #define SETHOOK_TEST(i, last) \ @@ -15002,7 +15473,8 @@ SETHOOK_TEST(2, false) SETHOOK_TEST(3, false) SETHOOK_TEST(4, false) SETHOOK_TEST(5, false) -SETHOOK_TEST(6, true) +SETHOOK_TEST(6, false) +SETHOOK_TEST(7, true) BEAST_DEFINE_TESTSUITE_PRIO(SetHook0, app, ripple, 2); BEAST_DEFINE_TESTSUITE_PRIO(SetHook1, app, ripple, 2); @@ -15011,6 +15483,7 @@ BEAST_DEFINE_TESTSUITE_PRIO(SetHook3, app, ripple, 2); BEAST_DEFINE_TESTSUITE_PRIO(SetHook4, app, ripple, 2); BEAST_DEFINE_TESTSUITE_PRIO(SetHook5, app, ripple, 2); BEAST_DEFINE_TESTSUITE_PRIO(SetHook6, app, ripple, 2); +BEAST_DEFINE_TESTSUITE_PRIO(SetHook7, app, ripple, 2); } // namespace test } // namespace ripple #undef M diff --git a/src/test/app/SetHook_wasm.h b/src/test/app/SetHook_wasm.h index b54e41550..5a8d43cf3 100644 --- a/src/test/app/SetHook_wasm.h +++ b/src/test/app/SetHook_wasm.h @@ -20453,7 +20453,7 @@ std::map> wasm = { 0,0 ))); - ASSERT(34 == (e=util_keylet(buf, 34, KEYLET_AMM, + ASSERT(INVALID_ARGUMENT == (e=util_keylet(buf, 34, KEYLET_AMM, SBUF(asset1), SBUF(asset2), 0,0 ))); @@ -20534,9 +20534,9 @@ std::map> wasm = { 0x6CU, 0x62U, 0x61U, 0x63U, 0x6BU, 0x00U, 0x02U, 0x03U, 0x65U, 0x6EU, 0x76U, 0x06U, 0x61U, 0x63U, 0x63U, 0x65U, 0x70U, 0x74U, 0x00U, 0x02U, 0x03U, 0x02U, 0x01U, 0x03U, 0x05U, 0x03U, 0x01U, 0x00U, 0x02U, 0x06U, - 0x51U, 0x0DU, 0x7FU, 0x01U, 0x41U, 0xB0U, 0xA6U, 0x04U, 0x0BU, 0x7FU, - 0x00U, 0x41U, 0xB0U, 0x26U, 0x0BU, 0x7FU, 0x00U, 0x41U, 0x80U, 0x08U, - 0x0BU, 0x7FU, 0x00U, 0x41U, 0xB0U, 0xA6U, 0x04U, 0x0BU, 0x7FU, 0x00U, + 0x51U, 0x0DU, 0x7FU, 0x01U, 0x41U, 0xC0U, 0xA6U, 0x04U, 0x0BU, 0x7FU, + 0x00U, 0x41U, 0xBEU, 0x26U, 0x0BU, 0x7FU, 0x00U, 0x41U, 0x80U, 0x08U, + 0x0BU, 0x7FU, 0x00U, 0x41U, 0xC0U, 0xA6U, 0x04U, 0x0BU, 0x7FU, 0x00U, 0x41U, 0x80U, 0x08U, 0x0BU, 0x7FU, 0x00U, 0x41U, 0xF0U, 0x08U, 0x0BU, 0x7FU, 0x00U, 0x41U, 0x90U, 0x09U, 0x0BU, 0x7FU, 0x00U, 0x41U, 0xE4U, 0x08U, 0x0BU, 0x7FU, 0x00U, 0x41U, 0x80U, 0x08U, 0x0BU, 0x7FU, 0x00U, @@ -21016,64 +21016,64 @@ std::map> wasm = { 0x1AU, 0x0BU, 0x02U, 0x40U, 0x20U, 0x01U, 0x41U, 0x22U, 0x41U, 0x1BU, 0x41U, 0xB0U, 0x89U, 0x80U, 0x80U, 0x00U, 0x41U, 0x28U, 0x41U, 0xE0U, 0x89U, 0x80U, 0x80U, 0x00U, 0x41U, 0x28U, 0x41U, 0x00U, 0x41U, 0x00U, - 0x10U, 0x81U, 0x80U, 0x80U, 0x80U, 0x00U, 0x42U, 0x22U, 0x51U, 0x0DU, - 0x00U, 0x41U, 0x89U, 0x9FU, 0x80U, 0x80U, 0x00U, 0x41U, 0xCDU, 0x00U, + 0x10U, 0x81U, 0x80U, 0x80U, 0x80U, 0x00U, 0x42U, 0x79U, 0x51U, 0x0DU, + 0x00U, 0x41U, 0x89U, 0x9FU, 0x80U, 0x80U, 0x00U, 0x41U, 0xDBU, 0x00U, 0x42U, 0xD0U, 0x04U, 0x10U, 0x82U, 0x80U, 0x80U, 0x80U, 0x00U, 0x1AU, 0x0BU, 0x02U, 0x40U, 0x20U, 0x01U, 0x41U, 0x22U, 0x41U, 0x1CU, 0x41U, 0xF0U, 0x88U, 0x80U, 0x80U, 0x00U, 0x41U, 0x14U, 0x41U, 0x90U, 0x89U, 0x80U, 0x80U, 0x00U, 0x41U, 0x14U, 0x41U, 0x00U, 0x41U, 0x00U, 0x10U, 0x81U, 0x80U, 0x80U, 0x80U, 0x00U, 0x42U, 0x79U, 0x51U, 0x0DU, 0x00U, - 0x41U, 0xD6U, 0x9FU, 0x80U, 0x80U, 0x00U, 0x41U, 0xD4U, 0x00U, 0x42U, + 0x41U, 0xE4U, 0x9FU, 0x80U, 0x80U, 0x00U, 0x41U, 0xD4U, 0x00U, 0x42U, 0xD5U, 0x04U, 0x10U, 0x82U, 0x80U, 0x80U, 0x80U, 0x00U, 0x1AU, 0x0BU, 0x02U, 0x40U, 0x20U, 0x01U, 0x41U, 0x22U, 0x41U, 0x1DU, 0x41U, 0xF0U, 0x88U, 0x80U, 0x80U, 0x00U, 0x41U, 0x14U, 0x41U, 0x90U, 0x89U, 0x80U, 0x80U, 0x00U, 0x41U, 0x14U, 0x41U, 0x00U, 0x41U, 0x00U, 0x10U, 0x81U, 0x80U, 0x80U, 0x80U, 0x00U, 0x42U, 0x79U, 0x51U, 0x0DU, 0x00U, 0x41U, - 0xAAU, 0xA0U, 0x80U, 0x80U, 0x00U, 0x41U, 0xE3U, 0x00U, 0x42U, 0xDAU, + 0xB8U, 0xA0U, 0x80U, 0x80U, 0x00U, 0x41U, 0xE3U, 0x00U, 0x42U, 0xDAU, 0x04U, 0x10U, 0x82U, 0x80U, 0x80U, 0x80U, 0x00U, 0x1AU, 0x0BU, 0x02U, 0x40U, 0x20U, 0x01U, 0x41U, 0x22U, 0x41U, 0x1EU, 0x41U, 0xF0U, 0x88U, 0x80U, 0x80U, 0x00U, 0x41U, 0x14U, 0x41U, 0x90U, 0x89U, 0x80U, 0x80U, 0x00U, 0x41U, 0x14U, 0x41U, 0x00U, 0x41U, 0x00U, 0x10U, 0x81U, 0x80U, - 0x80U, 0x80U, 0x00U, 0x42U, 0x79U, 0x51U, 0x0DU, 0x00U, 0x41U, 0x8DU, + 0x80U, 0x80U, 0x00U, 0x42U, 0x79U, 0x51U, 0x0DU, 0x00U, 0x41U, 0x9BU, 0xA1U, 0x80U, 0x80U, 0x00U, 0x41U, 0xF2U, 0x00U, 0x42U, 0xDFU, 0x04U, 0x10U, 0x82U, 0x80U, 0x80U, 0x80U, 0x00U, 0x1AU, 0x0BU, 0x02U, 0x40U, 0x20U, 0x01U, 0x41U, 0x22U, 0x41U, 0x1FU, 0x41U, 0xF0U, 0x88U, 0x80U, 0x80U, 0x00U, 0x41U, 0x14U, 0x41U, 0x00U, 0x41U, 0x00U, 0x41U, 0x00U, 0x41U, 0x00U, 0x10U, 0x81U, 0x80U, 0x80U, 0x80U, 0x00U, 0x42U, 0x79U, - 0x51U, 0x0DU, 0x00U, 0x41U, 0xFFU, 0xA1U, 0x80U, 0x80U, 0x00U, 0x41U, + 0x51U, 0x0DU, 0x00U, 0x41U, 0x8DU, 0xA2U, 0x80U, 0x80U, 0x00U, 0x41U, 0xCDU, 0x00U, 0x42U, 0xE5U, 0x04U, 0x10U, 0x82U, 0x80U, 0x80U, 0x80U, 0x00U, 0x1AU, 0x0BU, 0x02U, 0x40U, 0x20U, 0x01U, 0x41U, 0x22U, 0x41U, 0x20U, 0x41U, 0xF0U, 0x88U, 0x80U, 0x80U, 0x00U, 0x41U, 0x14U, 0x41U, 0x03U, 0x41U, 0x00U, 0x41U, 0x00U, 0x41U, 0x00U, 0x10U, 0x81U, 0x80U, - 0x80U, 0x80U, 0x00U, 0x42U, 0x22U, 0x51U, 0x0DU, 0x00U, 0x41U, 0xCCU, + 0x80U, 0x80U, 0x00U, 0x42U, 0x22U, 0x51U, 0x0DU, 0x00U, 0x41U, 0xDAU, 0xA2U, 0x80U, 0x80U, 0x00U, 0x41U, 0xC3U, 0x00U, 0x42U, 0xEBU, 0x04U, 0x10U, 0x82U, 0x80U, 0x80U, 0x80U, 0x00U, 0x1AU, 0x0BU, 0x02U, 0x40U, 0x20U, 0x01U, 0x41U, 0x22U, 0x41U, 0x21U, 0x41U, 0xF0U, 0x88U, 0x80U, 0x80U, 0x00U, 0x41U, 0x14U, 0x41U, 0x00U, 0x41U, 0x00U, 0x41U, 0x00U, 0x41U, 0x00U, 0x10U, 0x81U, 0x80U, 0x80U, 0x80U, 0x00U, 0x42U, 0x79U, - 0x51U, 0x0DU, 0x00U, 0x41U, 0x8FU, 0xA3U, 0x80U, 0x80U, 0x00U, 0x41U, + 0x51U, 0x0DU, 0x00U, 0x41U, 0x9DU, 0xA3U, 0x80U, 0x80U, 0x00U, 0x41U, 0xDAU, 0x00U, 0x42U, 0xF1U, 0x04U, 0x10U, 0x82U, 0x80U, 0x80U, 0x80U, 0x00U, 0x1AU, 0x0BU, 0x02U, 0x40U, 0x20U, 0x01U, 0x41U, 0x22U, 0x41U, 0x22U, 0x41U, 0xF0U, 0x88U, 0x80U, 0x80U, 0x00U, 0x41U, 0x14U, 0x41U, 0x00U, 0x41U, 0x00U, 0x41U, 0x00U, 0x41U, 0x00U, 0x10U, 0x81U, 0x80U, - 0x80U, 0x80U, 0x00U, 0x42U, 0x79U, 0x51U, 0x0DU, 0x00U, 0x41U, 0xE9U, + 0x80U, 0x80U, 0x00U, 0x42U, 0x79U, 0x51U, 0x0DU, 0x00U, 0x41U, 0xF7U, 0xA3U, 0x80U, 0x80U, 0x00U, 0x41U, 0xD1U, 0x00U, 0x42U, 0xF7U, 0x04U, 0x10U, 0x82U, 0x80U, 0x80U, 0x80U, 0x00U, 0x1AU, 0x0BU, 0x02U, 0x40U, 0x20U, 0x01U, 0x41U, 0x22U, 0x41U, 0x23U, 0x41U, 0xF0U, 0x88U, 0x80U, 0x80U, 0x00U, 0x41U, 0x14U, 0x41U, 0x90U, 0x89U, 0x80U, 0x80U, 0x00U, 0x41U, 0x14U, 0x41U, 0x00U, 0x41U, 0x00U, 0x10U, 0x81U, 0x80U, 0x80U, - 0x80U, 0x00U, 0x42U, 0x79U, 0x51U, 0x0DU, 0x00U, 0x41U, 0xBAU, 0xA4U, + 0x80U, 0x00U, 0x42U, 0x79U, 0x51U, 0x0DU, 0x00U, 0x41U, 0xC8U, 0xA4U, 0x80U, 0x80U, 0x00U, 0x41U, 0xD8U, 0x00U, 0x42U, 0xFCU, 0x04U, 0x10U, 0x82U, 0x80U, 0x80U, 0x80U, 0x00U, 0x1AU, 0x0BU, 0x02U, 0x40U, 0x20U, 0x01U, 0x41U, 0x22U, 0x41U, 0x24U, 0x41U, 0xF0U, 0x88U, 0x80U, 0x80U, 0x00U, 0x41U, 0x14U, 0x41U, 0x00U, 0x41U, 0x00U, 0x41U, 0x00U, 0x41U, 0x00U, 0x10U, 0x81U, 0x80U, 0x80U, 0x80U, 0x00U, 0x42U, 0x79U, 0x51U, - 0x0DU, 0x00U, 0x41U, 0x92U, 0xA5U, 0x80U, 0x80U, 0x00U, 0x41U, 0xDDU, + 0x0DU, 0x00U, 0x41U, 0xA0U, 0xA5U, 0x80U, 0x80U, 0x00U, 0x41U, 0xDDU, 0x00U, 0x42U, 0x82U, 0x05U, 0x10U, 0x82U, 0x80U, 0x80U, 0x80U, 0x00U, 0x1AU, 0x0BU, 0x02U, 0x40U, 0x20U, 0x01U, 0x41U, 0x22U, 0x41U, 0x1AU, 0x41U, 0xF0U, 0x88U, 0x80U, 0x80U, 0x00U, 0x41U, 0x14U, 0x41U, 0x01U, 0x41U, 0x00U, 0x41U, 0x00U, 0x41U, 0x00U, 0x10U, 0x81U, 0x80U, 0x80U, - 0x80U, 0x00U, 0x42U, 0x22U, 0x51U, 0x0DU, 0x00U, 0x41U, 0xEFU, 0xA5U, + 0x80U, 0x00U, 0x42U, 0x22U, 0x51U, 0x0DU, 0x00U, 0x41U, 0xFDU, 0xA5U, 0x80U, 0x80U, 0x00U, 0x41U, 0xC1U, 0x00U, 0x42U, 0x84U, 0x05U, 0x10U, 0x82U, 0x80U, 0x80U, 0x80U, 0x00U, 0x1AU, 0x0BU, 0x02U, 0x40U, 0x02U, 0x40U, 0x20U, 0x01U, 0x29U, 0x03U, 0x00U, 0x42U, 0x80U, 0x82U, 0xDDU, @@ -21088,7 +21088,7 @@ std::map> wasm = { 0x80U, 0x80U, 0x80U, 0x00U, 0x1AU, 0x0BU, 0x41U, 0x00U, 0x41U, 0x00U, 0x42U, 0x00U, 0x10U, 0x83U, 0x80U, 0x80U, 0x80U, 0x00U, 0x1AU, 0x20U, 0x01U, 0x41U, 0x30U, 0x6AU, 0x24U, 0x80U, 0x80U, 0x80U, 0x80U, 0x00U, - 0x20U, 0x02U, 0x0BU, 0x0BU, 0xBFU, 0x1EU, 0x02U, 0x00U, 0x41U, 0x80U, + 0x20U, 0x02U, 0x0BU, 0x0BU, 0xCDU, 0x1EU, 0x02U, 0x00U, 0x41U, 0x80U, 0x08U, 0x0BU, 0x88U, 0x02U, 0xC5U, 0xD0U, 0xF3U, 0x4BU, 0x0AU, 0x19U, 0x05U, 0xBCU, 0x3BU, 0x29U, 0xAAU, 0x1BU, 0xE1U, 0x39U, 0xFEU, 0x04U, 0xD6U, 0x0CU, 0x86U, 0x94U, 0xD3U, 0x95U, 0x0AU, 0x8DU, 0x80U, 0x25U, @@ -21116,7 +21116,7 @@ std::map> wasm = { 0x45U, 0x48U, 0x52U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x3AU, 0x51U, 0x8AU, 0x22U, 0x53U, 0x81U, 0x60U, 0x84U, 0x1CU, 0x14U, 0x32U, 0xFEU, 0x6FU, 0x3EU, 0x6DU, 0x6EU, 0x76U, 0x29U, 0xFBU, 0xBAU, 0x00U, 0x41U, - 0x88U, 0x0AU, 0x0BU, 0xA8U, 0x1CU, 0x75U, 0x74U, 0x69U, 0x6CU, 0x5FU, + 0x88U, 0x0AU, 0x0BU, 0xB6U, 0x1CU, 0x75U, 0x74U, 0x69U, 0x6CU, 0x5FU, 0x6BU, 0x65U, 0x79U, 0x6CU, 0x65U, 0x74U, 0x28U, 0x31U, 0x30U, 0x30U, 0x30U, 0x30U, 0x30U, 0x30U, 0x2CU, 0x20U, 0x33U, 0x34U, 0x2CU, 0x20U, 0x4BU, 0x45U, 0x59U, 0x4CU, 0x45U, 0x54U, 0x5FU, 0x53U, 0x4BU, 0x49U, @@ -21385,41 +21385,31 @@ std::map> wasm = { 0x54U, 0x45U, 0x5FU, 0x44U, 0x49U, 0x52U, 0x2CU, 0x20U, 0x53U, 0x42U, 0x55U, 0x46U, 0x28U, 0x61U, 0x29U, 0x2CU, 0x20U, 0x53U, 0x42U, 0x55U, 0x46U, 0x28U, 0x6EU, 0x73U, 0x29U, 0x2CU, 0x20U, 0x30U, 0x2CU, 0x30U, - 0x20U, 0x29U, 0x29U, 0x00U, 0x33U, 0x34U, 0x20U, 0x3DU, 0x3DU, 0x20U, - 0x28U, 0x65U, 0x3DU, 0x75U, 0x74U, 0x69U, 0x6CU, 0x5FU, 0x6BU, 0x65U, - 0x79U, 0x6CU, 0x65U, 0x74U, 0x28U, 0x62U, 0x75U, 0x66U, 0x2CU, 0x20U, - 0x33U, 0x34U, 0x2CU, 0x20U, 0x4BU, 0x45U, 0x59U, 0x4CU, 0x45U, 0x54U, - 0x5FU, 0x41U, 0x4DU, 0x4DU, 0x2CU, 0x20U, 0x53U, 0x42U, 0x55U, 0x46U, - 0x28U, 0x61U, 0x73U, 0x73U, 0x65U, 0x74U, 0x31U, 0x29U, 0x2CU, 0x20U, - 0x53U, 0x42U, 0x55U, 0x46U, 0x28U, 0x61U, 0x73U, 0x73U, 0x65U, 0x74U, - 0x32U, 0x29U, 0x2CU, 0x20U, 0x30U, 0x2CU, 0x30U, 0x20U, 0x29U, 0x29U, - 0x00U, 0x49U, 0x4EU, 0x56U, 0x41U, 0x4CU, 0x49U, 0x44U, 0x5FU, 0x41U, - 0x52U, 0x47U, 0x55U, 0x4DU, 0x45U, 0x4EU, 0x54U, 0x20U, 0x3DU, 0x3DU, - 0x20U, 0x28U, 0x65U, 0x3DU, 0x75U, 0x74U, 0x69U, 0x6CU, 0x5FU, 0x6BU, - 0x65U, 0x79U, 0x6CU, 0x65U, 0x74U, 0x28U, 0x62U, 0x75U, 0x66U, 0x2CU, - 0x20U, 0x33U, 0x34U, 0x2CU, 0x20U, 0x4BU, 0x45U, 0x59U, 0x4CU, 0x45U, - 0x54U, 0x5FU, 0x42U, 0x52U, 0x49U, 0x44U, 0x47U, 0x45U, 0x2CU, 0x20U, - 0x53U, 0x42U, 0x55U, 0x46U, 0x28U, 0x61U, 0x29U, 0x2CU, 0x20U, 0x53U, - 0x42U, 0x55U, 0x46U, 0x28U, 0x62U, 0x29U, 0x2CU, 0x20U, 0x30U, 0x2CU, - 0x30U, 0x20U, 0x29U, 0x29U, 0x00U, 0x49U, 0x4EU, 0x56U, 0x41U, 0x4CU, - 0x49U, 0x44U, 0x5FU, 0x41U, 0x52U, 0x47U, 0x55U, 0x4DU, 0x45U, 0x4EU, - 0x54U, 0x20U, 0x3DU, 0x3DU, 0x20U, 0x28U, 0x65U, 0x3DU, 0x75U, 0x74U, - 0x69U, 0x6CU, 0x5FU, 0x6BU, 0x65U, 0x79U, 0x6CU, 0x65U, 0x74U, 0x28U, - 0x62U, 0x75U, 0x66U, 0x2CU, 0x20U, 0x33U, 0x34U, 0x2CU, 0x20U, 0x4BU, - 0x45U, 0x59U, 0x4CU, 0x45U, 0x54U, 0x5FU, 0x58U, 0x43U, 0x48U, 0x41U, - 0x49U, 0x4EU, 0x5FU, 0x4FU, 0x57U, 0x4EU, 0x45U, 0x44U, 0x5FU, 0x43U, - 0x4CU, 0x41U, 0x49U, 0x4DU, 0x5FU, 0x49U, 0x44U, 0x2CU, 0x20U, 0x53U, - 0x42U, 0x55U, 0x46U, 0x28U, 0x61U, 0x29U, 0x2CU, 0x20U, 0x53U, 0x42U, - 0x55U, 0x46U, 0x28U, 0x62U, 0x29U, 0x2CU, 0x20U, 0x30U, 0x2CU, 0x30U, 0x20U, 0x29U, 0x29U, 0x00U, 0x49U, 0x4EU, 0x56U, 0x41U, 0x4CU, 0x49U, 0x44U, 0x5FU, 0x41U, 0x52U, 0x47U, 0x55U, 0x4DU, 0x45U, 0x4EU, 0x54U, 0x20U, 0x3DU, 0x3DU, 0x20U, 0x28U, 0x65U, 0x3DU, 0x75U, 0x74U, 0x69U, 0x6CU, 0x5FU, 0x6BU, 0x65U, 0x79U, 0x6CU, 0x65U, 0x74U, 0x28U, 0x62U, 0x75U, 0x66U, 0x2CU, 0x20U, 0x33U, 0x34U, 0x2CU, 0x20U, 0x4BU, 0x45U, - 0x59U, 0x4CU, 0x45U, 0x54U, 0x5FU, 0x58U, 0x43U, 0x48U, 0x41U, 0x49U, - 0x4EU, 0x5FU, 0x4FU, 0x57U, 0x4EU, 0x45U, 0x44U, 0x5FU, 0x43U, 0x52U, - 0x45U, 0x41U, 0x54U, 0x45U, 0x5FU, 0x41U, 0x43U, 0x43U, 0x4FU, 0x55U, - 0x4EU, 0x54U, 0x5FU, 0x43U, 0x4CU, 0x41U, 0x49U, 0x4DU, 0x5FU, 0x49U, + 0x59U, 0x4CU, 0x45U, 0x54U, 0x5FU, 0x41U, 0x4DU, 0x4DU, 0x2CU, 0x20U, + 0x53U, 0x42U, 0x55U, 0x46U, 0x28U, 0x61U, 0x73U, 0x73U, 0x65U, 0x74U, + 0x31U, 0x29U, 0x2CU, 0x20U, 0x53U, 0x42U, 0x55U, 0x46U, 0x28U, 0x61U, + 0x73U, 0x73U, 0x65U, 0x74U, 0x32U, 0x29U, 0x2CU, 0x20U, 0x30U, 0x2CU, + 0x30U, 0x20U, 0x29U, 0x29U, 0x00U, 0x49U, 0x4EU, 0x56U, 0x41U, 0x4CU, + 0x49U, 0x44U, 0x5FU, 0x41U, 0x52U, 0x47U, 0x55U, 0x4DU, 0x45U, 0x4EU, + 0x54U, 0x20U, 0x3DU, 0x3DU, 0x20U, 0x28U, 0x65U, 0x3DU, 0x75U, 0x74U, + 0x69U, 0x6CU, 0x5FU, 0x6BU, 0x65U, 0x79U, 0x6CU, 0x65U, 0x74U, 0x28U, + 0x62U, 0x75U, 0x66U, 0x2CU, 0x20U, 0x33U, 0x34U, 0x2CU, 0x20U, 0x4BU, + 0x45U, 0x59U, 0x4CU, 0x45U, 0x54U, 0x5FU, 0x42U, 0x52U, 0x49U, 0x44U, + 0x47U, 0x45U, 0x2CU, 0x20U, 0x53U, 0x42U, 0x55U, 0x46U, 0x28U, 0x61U, + 0x29U, 0x2CU, 0x20U, 0x53U, 0x42U, 0x55U, 0x46U, 0x28U, 0x62U, 0x29U, + 0x2CU, 0x20U, 0x30U, 0x2CU, 0x30U, 0x20U, 0x29U, 0x29U, 0x00U, 0x49U, + 0x4EU, 0x56U, 0x41U, 0x4CU, 0x49U, 0x44U, 0x5FU, 0x41U, 0x52U, 0x47U, + 0x55U, 0x4DU, 0x45U, 0x4EU, 0x54U, 0x20U, 0x3DU, 0x3DU, 0x20U, 0x28U, + 0x65U, 0x3DU, 0x75U, 0x74U, 0x69U, 0x6CU, 0x5FU, 0x6BU, 0x65U, 0x79U, + 0x6CU, 0x65U, 0x74U, 0x28U, 0x62U, 0x75U, 0x66U, 0x2CU, 0x20U, 0x33U, + 0x34U, 0x2CU, 0x20U, 0x4BU, 0x45U, 0x59U, 0x4CU, 0x45U, 0x54U, 0x5FU, + 0x58U, 0x43U, 0x48U, 0x41U, 0x49U, 0x4EU, 0x5FU, 0x4FU, 0x57U, 0x4EU, + 0x45U, 0x44U, 0x5FU, 0x43U, 0x4CU, 0x41U, 0x49U, 0x4DU, 0x5FU, 0x49U, 0x44U, 0x2CU, 0x20U, 0x53U, 0x42U, 0x55U, 0x46U, 0x28U, 0x61U, 0x29U, 0x2CU, 0x20U, 0x53U, 0x42U, 0x55U, 0x46U, 0x28U, 0x62U, 0x29U, 0x2CU, 0x20U, 0x30U, 0x2CU, 0x30U, 0x20U, 0x29U, 0x29U, 0x00U, 0x49U, 0x4EU, @@ -21427,58 +21417,70 @@ std::map> wasm = { 0x4DU, 0x45U, 0x4EU, 0x54U, 0x20U, 0x3DU, 0x3DU, 0x20U, 0x28U, 0x65U, 0x3DU, 0x75U, 0x74U, 0x69U, 0x6CU, 0x5FU, 0x6BU, 0x65U, 0x79U, 0x6CU, 0x65U, 0x74U, 0x28U, 0x62U, 0x75U, 0x66U, 0x2CU, 0x20U, 0x33U, 0x34U, - 0x2CU, 0x20U, 0x4BU, 0x45U, 0x59U, 0x4CU, 0x45U, 0x54U, 0x5FU, 0x44U, - 0x49U, 0x44U, 0x2CU, 0x20U, 0x53U, 0x42U, 0x55U, 0x46U, 0x28U, 0x61U, - 0x29U, 0x2CU, 0x20U, 0x30U, 0x2CU, 0x30U, 0x2CU, 0x20U, 0x30U, 0x2CU, - 0x30U, 0x20U, 0x29U, 0x29U, 0x00U, 0x33U, 0x34U, 0x20U, 0x3DU, 0x3DU, - 0x20U, 0x28U, 0x65U, 0x3DU, 0x75U, 0x74U, 0x69U, 0x6CU, 0x5FU, 0x6BU, - 0x65U, 0x79U, 0x6CU, 0x65U, 0x74U, 0x28U, 0x62U, 0x75U, 0x66U, 0x2CU, - 0x20U, 0x33U, 0x34U, 0x2CU, 0x20U, 0x4BU, 0x45U, 0x59U, 0x4CU, 0x45U, - 0x54U, 0x5FU, 0x4FU, 0x52U, 0x41U, 0x43U, 0x4CU, 0x45U, 0x2CU, 0x20U, - 0x53U, 0x42U, 0x55U, 0x46U, 0x28U, 0x61U, 0x29U, 0x2CU, 0x20U, 0x33U, - 0x2CU, 0x20U, 0x30U, 0x2CU, 0x20U, 0x30U, 0x2CU, 0x30U, 0x20U, 0x29U, + 0x2CU, 0x20U, 0x4BU, 0x45U, 0x59U, 0x4CU, 0x45U, 0x54U, 0x5FU, 0x58U, + 0x43U, 0x48U, 0x41U, 0x49U, 0x4EU, 0x5FU, 0x4FU, 0x57U, 0x4EU, 0x45U, + 0x44U, 0x5FU, 0x43U, 0x52U, 0x45U, 0x41U, 0x54U, 0x45U, 0x5FU, 0x41U, + 0x43U, 0x43U, 0x4FU, 0x55U, 0x4EU, 0x54U, 0x5FU, 0x43U, 0x4CU, 0x41U, + 0x49U, 0x4DU, 0x5FU, 0x49U, 0x44U, 0x2CU, 0x20U, 0x53U, 0x42U, 0x55U, + 0x46U, 0x28U, 0x61U, 0x29U, 0x2CU, 0x20U, 0x53U, 0x42U, 0x55U, 0x46U, + 0x28U, 0x62U, 0x29U, 0x2CU, 0x20U, 0x30U, 0x2CU, 0x30U, 0x20U, 0x29U, 0x29U, 0x00U, 0x49U, 0x4EU, 0x56U, 0x41U, 0x4CU, 0x49U, 0x44U, 0x5FU, 0x41U, 0x52U, 0x47U, 0x55U, 0x4DU, 0x45U, 0x4EU, 0x54U, 0x20U, 0x3DU, 0x3DU, 0x20U, 0x28U, 0x65U, 0x3DU, 0x75U, 0x74U, 0x69U, 0x6CU, 0x5FU, 0x6BU, 0x65U, 0x79U, 0x6CU, 0x65U, 0x74U, 0x28U, 0x62U, 0x75U, 0x66U, 0x2CU, 0x20U, 0x33U, 0x34U, 0x2CU, 0x20U, 0x4BU, 0x45U, 0x59U, 0x4CU, - 0x45U, 0x54U, 0x5FU, 0x4DU, 0x50U, 0x54U, 0x4FU, 0x4BU, 0x45U, 0x4EU, - 0x5FU, 0x49U, 0x53U, 0x53U, 0x55U, 0x41U, 0x4EU, 0x43U, 0x45U, 0x2CU, + 0x45U, 0x54U, 0x5FU, 0x44U, 0x49U, 0x44U, 0x2CU, 0x20U, 0x53U, 0x42U, + 0x55U, 0x46U, 0x28U, 0x61U, 0x29U, 0x2CU, 0x20U, 0x30U, 0x2CU, 0x30U, + 0x2CU, 0x20U, 0x30U, 0x2CU, 0x30U, 0x20U, 0x29U, 0x29U, 0x00U, 0x33U, + 0x34U, 0x20U, 0x3DU, 0x3DU, 0x20U, 0x28U, 0x65U, 0x3DU, 0x75U, 0x74U, + 0x69U, 0x6CU, 0x5FU, 0x6BU, 0x65U, 0x79U, 0x6CU, 0x65U, 0x74U, 0x28U, + 0x62U, 0x75U, 0x66U, 0x2CU, 0x20U, 0x33U, 0x34U, 0x2CU, 0x20U, 0x4BU, + 0x45U, 0x59U, 0x4CU, 0x45U, 0x54U, 0x5FU, 0x4FU, 0x52U, 0x41U, 0x43U, + 0x4CU, 0x45U, 0x2CU, 0x20U, 0x53U, 0x42U, 0x55U, 0x46U, 0x28U, 0x61U, + 0x29U, 0x2CU, 0x20U, 0x33U, 0x2CU, 0x20U, 0x30U, 0x2CU, 0x20U, 0x30U, + 0x2CU, 0x30U, 0x20U, 0x29U, 0x29U, 0x00U, 0x49U, 0x4EU, 0x56U, 0x41U, + 0x4CU, 0x49U, 0x44U, 0x5FU, 0x41U, 0x52U, 0x47U, 0x55U, 0x4DU, 0x45U, + 0x4EU, 0x54U, 0x20U, 0x3DU, 0x3DU, 0x20U, 0x28U, 0x65U, 0x3DU, 0x75U, + 0x74U, 0x69U, 0x6CU, 0x5FU, 0x6BU, 0x65U, 0x79U, 0x6CU, 0x65U, 0x74U, + 0x28U, 0x62U, 0x75U, 0x66U, 0x2CU, 0x20U, 0x33U, 0x34U, 0x2CU, 0x20U, + 0x4BU, 0x45U, 0x59U, 0x4CU, 0x45U, 0x54U, 0x5FU, 0x4DU, 0x50U, 0x54U, + 0x4FU, 0x4BU, 0x45U, 0x4EU, 0x5FU, 0x49U, 0x53U, 0x53U, 0x55U, 0x41U, + 0x4EU, 0x43U, 0x45U, 0x2CU, 0x20U, 0x53U, 0x42U, 0x55U, 0x46U, 0x28U, + 0x61U, 0x29U, 0x2CU, 0x20U, 0x30U, 0x2CU, 0x30U, 0x2CU, 0x20U, 0x30U, + 0x2CU, 0x30U, 0x20U, 0x29U, 0x29U, 0x00U, 0x49U, 0x4EU, 0x56U, 0x41U, + 0x4CU, 0x49U, 0x44U, 0x5FU, 0x41U, 0x52U, 0x47U, 0x55U, 0x4DU, 0x45U, + 0x4EU, 0x54U, 0x20U, 0x3DU, 0x3DU, 0x20U, 0x28U, 0x65U, 0x3DU, 0x75U, + 0x74U, 0x69U, 0x6CU, 0x5FU, 0x6BU, 0x65U, 0x79U, 0x6CU, 0x65U, 0x74U, + 0x28U, 0x62U, 0x75U, 0x66U, 0x2CU, 0x20U, 0x33U, 0x34U, 0x2CU, 0x20U, + 0x4BU, 0x45U, 0x59U, 0x4CU, 0x45U, 0x54U, 0x5FU, 0x4DU, 0x50U, 0x54U, + 0x4FU, 0x4BU, 0x45U, 0x4EU, 0x2CU, 0x20U, 0x53U, 0x42U, 0x55U, 0x46U, + 0x28U, 0x61U, 0x29U, 0x2CU, 0x20U, 0x30U, 0x2CU, 0x30U, 0x2CU, 0x20U, + 0x30U, 0x2CU, 0x30U, 0x20U, 0x29U, 0x29U, 0x00U, 0x49U, 0x4EU, 0x56U, + 0x41U, 0x4CU, 0x49U, 0x44U, 0x5FU, 0x41U, 0x52U, 0x47U, 0x55U, 0x4DU, + 0x45U, 0x4EU, 0x54U, 0x20U, 0x3DU, 0x3DU, 0x20U, 0x28U, 0x65U, 0x3DU, + 0x75U, 0x74U, 0x69U, 0x6CU, 0x5FU, 0x6BU, 0x65U, 0x79U, 0x6CU, 0x65U, + 0x74U, 0x28U, 0x62U, 0x75U, 0x66U, 0x2CU, 0x20U, 0x33U, 0x34U, 0x2CU, + 0x20U, 0x4BU, 0x45U, 0x59U, 0x4CU, 0x45U, 0x54U, 0x5FU, 0x43U, 0x52U, + 0x45U, 0x44U, 0x45U, 0x4EU, 0x54U, 0x49U, 0x41U, 0x4CU, 0x2CU, 0x20U, + 0x53U, 0x42U, 0x55U, 0x46U, 0x28U, 0x61U, 0x29U, 0x2CU, 0x20U, 0x53U, + 0x42U, 0x55U, 0x46U, 0x28U, 0x62U, 0x29U, 0x2CU, 0x20U, 0x30U, 0x2CU, + 0x30U, 0x20U, 0x29U, 0x29U, 0x00U, 0x49U, 0x4EU, 0x56U, 0x41U, 0x4CU, + 0x49U, 0x44U, 0x5FU, 0x41U, 0x52U, 0x47U, 0x55U, 0x4DU, 0x45U, 0x4EU, + 0x54U, 0x20U, 0x3DU, 0x3DU, 0x20U, 0x28U, 0x65U, 0x3DU, 0x75U, 0x74U, + 0x69U, 0x6CU, 0x5FU, 0x6BU, 0x65U, 0x79U, 0x6CU, 0x65U, 0x74U, 0x28U, + 0x62U, 0x75U, 0x66U, 0x2CU, 0x20U, 0x33U, 0x34U, 0x2CU, 0x20U, 0x4BU, + 0x45U, 0x59U, 0x4CU, 0x45U, 0x54U, 0x5FU, 0x50U, 0x45U, 0x52U, 0x4DU, + 0x49U, 0x53U, 0x53U, 0x49U, 0x4FU, 0x4EU, 0x45U, 0x44U, 0x5FU, 0x44U, + 0x4FU, 0x4DU, 0x41U, 0x49U, 0x4EU, 0x2CU, 0x20U, 0x53U, 0x42U, 0x55U, + 0x46U, 0x28U, 0x61U, 0x29U, 0x2CU, 0x20U, 0x30U, 0x2CU, 0x30U, 0x2CU, + 0x20U, 0x30U, 0x2CU, 0x30U, 0x20U, 0x29U, 0x29U, 0x00U, 0x33U, 0x34U, + 0x20U, 0x3DU, 0x3DU, 0x20U, 0x28U, 0x65U, 0x3DU, 0x75U, 0x74U, 0x69U, + 0x6CU, 0x5FU, 0x6BU, 0x65U, 0x79U, 0x6CU, 0x65U, 0x74U, 0x28U, 0x62U, + 0x75U, 0x66U, 0x2CU, 0x20U, 0x33U, 0x34U, 0x2CU, 0x20U, 0x4BU, 0x45U, + 0x59U, 0x4CU, 0x45U, 0x54U, 0x5FU, 0x43U, 0x52U, 0x4FU, 0x4EU, 0x2CU, 0x20U, 0x53U, 0x42U, 0x55U, 0x46U, 0x28U, 0x61U, 0x29U, 0x2CU, 0x20U, - 0x30U, 0x2CU, 0x30U, 0x2CU, 0x20U, 0x30U, 0x2CU, 0x30U, 0x20U, 0x29U, - 0x29U, 0x00U, 0x49U, 0x4EU, 0x56U, 0x41U, 0x4CU, 0x49U, 0x44U, 0x5FU, - 0x41U, 0x52U, 0x47U, 0x55U, 0x4DU, 0x45U, 0x4EU, 0x54U, 0x20U, 0x3DU, - 0x3DU, 0x20U, 0x28U, 0x65U, 0x3DU, 0x75U, 0x74U, 0x69U, 0x6CU, 0x5FU, - 0x6BU, 0x65U, 0x79U, 0x6CU, 0x65U, 0x74U, 0x28U, 0x62U, 0x75U, 0x66U, - 0x2CU, 0x20U, 0x33U, 0x34U, 0x2CU, 0x20U, 0x4BU, 0x45U, 0x59U, 0x4CU, - 0x45U, 0x54U, 0x5FU, 0x4DU, 0x50U, 0x54U, 0x4FU, 0x4BU, 0x45U, 0x4EU, - 0x2CU, 0x20U, 0x53U, 0x42U, 0x55U, 0x46U, 0x28U, 0x61U, 0x29U, 0x2CU, - 0x20U, 0x30U, 0x2CU, 0x30U, 0x2CU, 0x20U, 0x30U, 0x2CU, 0x30U, 0x20U, - 0x29U, 0x29U, 0x00U, 0x49U, 0x4EU, 0x56U, 0x41U, 0x4CU, 0x49U, 0x44U, - 0x5FU, 0x41U, 0x52U, 0x47U, 0x55U, 0x4DU, 0x45U, 0x4EU, 0x54U, 0x20U, - 0x3DU, 0x3DU, 0x20U, 0x28U, 0x65U, 0x3DU, 0x75U, 0x74U, 0x69U, 0x6CU, - 0x5FU, 0x6BU, 0x65U, 0x79U, 0x6CU, 0x65U, 0x74U, 0x28U, 0x62U, 0x75U, - 0x66U, 0x2CU, 0x20U, 0x33U, 0x34U, 0x2CU, 0x20U, 0x4BU, 0x45U, 0x59U, - 0x4CU, 0x45U, 0x54U, 0x5FU, 0x43U, 0x52U, 0x45U, 0x44U, 0x45U, 0x4EU, - 0x54U, 0x49U, 0x41U, 0x4CU, 0x2CU, 0x20U, 0x53U, 0x42U, 0x55U, 0x46U, - 0x28U, 0x61U, 0x29U, 0x2CU, 0x20U, 0x53U, 0x42U, 0x55U, 0x46U, 0x28U, - 0x62U, 0x29U, 0x2CU, 0x20U, 0x30U, 0x2CU, 0x30U, 0x20U, 0x29U, 0x29U, - 0x00U, 0x49U, 0x4EU, 0x56U, 0x41U, 0x4CU, 0x49U, 0x44U, 0x5FU, 0x41U, - 0x52U, 0x47U, 0x55U, 0x4DU, 0x45U, 0x4EU, 0x54U, 0x20U, 0x3DU, 0x3DU, - 0x20U, 0x28U, 0x65U, 0x3DU, 0x75U, 0x74U, 0x69U, 0x6CU, 0x5FU, 0x6BU, - 0x65U, 0x79U, 0x6CU, 0x65U, 0x74U, 0x28U, 0x62U, 0x75U, 0x66U, 0x2CU, - 0x20U, 0x33U, 0x34U, 0x2CU, 0x20U, 0x4BU, 0x45U, 0x59U, 0x4CU, 0x45U, - 0x54U, 0x5FU, 0x50U, 0x45U, 0x52U, 0x4DU, 0x49U, 0x53U, 0x53U, 0x49U, - 0x4FU, 0x4EU, 0x45U, 0x44U, 0x5FU, 0x44U, 0x4FU, 0x4DU, 0x41U, 0x49U, - 0x4EU, 0x2CU, 0x20U, 0x53U, 0x42U, 0x55U, 0x46U, 0x28U, 0x61U, 0x29U, - 0x2CU, 0x20U, 0x30U, 0x2CU, 0x30U, 0x2CU, 0x20U, 0x30U, 0x2CU, 0x30U, - 0x20U, 0x29U, 0x29U, 0x00U, 0x33U, 0x34U, 0x20U, 0x3DU, 0x3DU, 0x20U, - 0x28U, 0x65U, 0x3DU, 0x75U, 0x74U, 0x69U, 0x6CU, 0x5FU, 0x6BU, 0x65U, - 0x79U, 0x6CU, 0x65U, 0x74U, 0x28U, 0x62U, 0x75U, 0x66U, 0x2CU, 0x20U, - 0x33U, 0x34U, 0x2CU, 0x20U, 0x4BU, 0x45U, 0x59U, 0x4CU, 0x45U, 0x54U, - 0x5FU, 0x43U, 0x52U, 0x4FU, 0x4EU, 0x2CU, 0x20U, 0x53U, 0x42U, 0x55U, - 0x46U, 0x28U, 0x61U, 0x29U, 0x2CU, 0x20U, 0x31U, 0x2CU, 0x20U, 0x30U, - 0x2CU, 0x20U, 0x30U, 0x2CU, 0x20U, 0x30U, 0x29U, 0x29U, 0x00U, + 0x31U, 0x2CU, 0x20U, 0x30U, 0x2CU, 0x20U, 0x30U, 0x2CU, 0x20U, 0x30U, + 0x29U, 0x29U, 0x00U, }}, /* ==== WASM: 89 ==== */ @@ -34110,6 +34112,193 @@ std::map> wasm = { 0x0BU, }}, + /* ==== WASM: 100 ==== */ + {R"[test.hook]( + #include + extern int64_t accept (uint32_t read_ptr, uint32_t read_len, int64_t error_code); + extern int64_t rollback (uint32_t read_ptr, uint32_t read_len, int64_t error_code); + extern int64_t emit (uint32_t write_ptr, uint32_t write_len, uint32_t read_ptr, uint32_t read_len); + extern int64_t hook_account(uint32_t write_ptr, uint32_t write_len); + extern int64_t etxn_reserve(uint32_t); + extern int64_t etxn_fee_base (uint32_t read_ptr, uint32_t read_len); + extern int64_t etxn_details (uint32_t write_ptr, uint32_t write_len); + extern int64_t ledger_seq (void); + + #define SBUF(x) (uint32_t)x,sizeof(x) + // clang-format off + uint8_t txn[229] = + { + /* size, upto, field name */ + /* 3, 0, tt = Invoke */ 0x12U, 0x00U, 0x63U, + /* 5, 3, flags */ 0x22U, 0x00U, 0x00U, 0x00U, 0x00U, + /* 5, 8, sequence */ 0x24U, 0x00U, 0x00U, 0x00U, 0x00U, + /* 6, 13, firstledgersequence */ 0x20U, 0x1AU, 0x00U, 0x00U, 0x00U, 0x00U, + /* 6, 19, lastledgersequence */ 0x20U, 0x1BU, 0x00U, 0x00U, 0x00U, 0x00U, + /* 9, 25, fee */ 0x68U, 0x40U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, + /* 35, 34, signingpubkey */ 0x73U, 0x21U, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + /* 22, 69, account */ 0x81U, 0x14U, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + /* 116, 91, emit details */ + /* 0, 229, */ + }; + // clang-format on + + // TX BUILDER + #define FLAGS_OUT (txn + 4U) + #define FLS_OUT (txn + 15U) + #define LLS_OUT (txn + 21U) + #define FEE_OUT (txn + 26U) + #define ACCOUNT_OUT (txn + 71U) + #define EMIT_OUT (txn + 91U) + + #define FLIP_ENDIAN_32(value) \ + (uint32_t)(((value & 0xFFU) << 24) | ((value & 0xFF00U) << 8) | \ + ((value & 0xFF0000U) >> 8) | ((value & 0xFF000000U) >> 24)) + + #define SET_UINT32(ptr, value) *((uint32_t *)(ptr)) = FLIP_ENDIAN_32(value); + + #define SET_NATIVE_AMOUNT(ptr, amount) \ + do { \ + uint8_t *b = (ptr); \ + *b++ = 0b01000000 + ((amount >> 56) & 0b00111111); \ + *b++ = (amount >> 48) & 0xFFU; \ + *b++ = (amount >> 40) & 0xFFU; \ + *b++ = (amount >> 32) & 0xFFU; \ + *b++ = (amount >> 24) & 0xFFU; \ + *b++ = (amount >> 16) & 0xFFU; \ + *b++ = (amount >> 8) & 0xFFU; \ + *b++ = (amount >> 0) & 0xFFU; \ + } while (0) + + #define PREPARE_TXN() \ + do { \ + etxn_reserve(1); \ + uint32_t fls = (uint32_t)ledger_seq() + 1; \ + SET_UINT32(FLS_OUT, fls); \ + SET_UINT32(LLS_OUT, fls + 4); \ + hook_account(ACCOUNT_OUT, 20); \ + etxn_details(EMIT_OUT, 138U); \ + int64_t fee = etxn_fee_base(SBUF(txn)); \ + SET_NATIVE_AMOUNT(FEE_OUT, fee); \ + } while (0) + + + int64_t cbak(uint32_t r) + { + _g(1,1); + return accept(0,0,0); + } + + int64_t hook(uint32_t reserved) + { + _g(1,1); + PREPARE_TXN(); + uint8_t emithash[32]; + int64_t emit_result = emit(SBUF(emithash), SBUF(txn)); + if (emit_result < 0) + return rollback(SBUF("Emit failed."), __LINE__); + return accept(SBUF("Emit succeeded."), __LINE__); + } + )[test.hook]", + { + 0x00U, 0x61U, 0x73U, 0x6DU, 0x01U, 0x00U, 0x00U, 0x00U, 0x01U, 0x25U, + 0x06U, 0x60U, 0x02U, 0x7FU, 0x7FU, 0x01U, 0x7FU, 0x60U, 0x03U, 0x7FU, + 0x7FU, 0x7EU, 0x01U, 0x7EU, 0x60U, 0x01U, 0x7FU, 0x01U, 0x7EU, 0x60U, + 0x00U, 0x01U, 0x7EU, 0x60U, 0x02U, 0x7FU, 0x7FU, 0x01U, 0x7EU, 0x60U, + 0x04U, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x01U, 0x7EU, 0x02U, 0x8FU, 0x01U, + 0x09U, 0x03U, 0x65U, 0x6EU, 0x76U, 0x02U, 0x5FU, 0x67U, 0x00U, 0x00U, + 0x03U, 0x65U, 0x6EU, 0x76U, 0x06U, 0x61U, 0x63U, 0x63U, 0x65U, 0x70U, + 0x74U, 0x00U, 0x01U, 0x03U, 0x65U, 0x6EU, 0x76U, 0x0CU, 0x65U, 0x74U, + 0x78U, 0x6EU, 0x5FU, 0x72U, 0x65U, 0x73U, 0x65U, 0x72U, 0x76U, 0x65U, + 0x00U, 0x02U, 0x03U, 0x65U, 0x6EU, 0x76U, 0x0AU, 0x6CU, 0x65U, 0x64U, + 0x67U, 0x65U, 0x72U, 0x5FU, 0x73U, 0x65U, 0x71U, 0x00U, 0x03U, 0x03U, + 0x65U, 0x6EU, 0x76U, 0x0CU, 0x68U, 0x6FU, 0x6FU, 0x6BU, 0x5FU, 0x61U, + 0x63U, 0x63U, 0x6FU, 0x75U, 0x6EU, 0x74U, 0x00U, 0x04U, 0x03U, 0x65U, + 0x6EU, 0x76U, 0x0CU, 0x65U, 0x74U, 0x78U, 0x6EU, 0x5FU, 0x64U, 0x65U, + 0x74U, 0x61U, 0x69U, 0x6CU, 0x73U, 0x00U, 0x04U, 0x03U, 0x65U, 0x6EU, + 0x76U, 0x0DU, 0x65U, 0x74U, 0x78U, 0x6EU, 0x5FU, 0x66U, 0x65U, 0x65U, + 0x5FU, 0x62U, 0x61U, 0x73U, 0x65U, 0x00U, 0x04U, 0x03U, 0x65U, 0x6EU, + 0x76U, 0x04U, 0x65U, 0x6DU, 0x69U, 0x74U, 0x00U, 0x05U, 0x03U, 0x65U, + 0x6EU, 0x76U, 0x08U, 0x72U, 0x6FU, 0x6CU, 0x6CU, 0x62U, 0x61U, 0x63U, + 0x6BU, 0x00U, 0x01U, 0x03U, 0x03U, 0x02U, 0x02U, 0x02U, 0x05U, 0x03U, + 0x01U, 0x00U, 0x02U, 0x06U, 0x27U, 0x06U, 0x7FU, 0x01U, 0x41U, 0x90U, + 0x8AU, 0x04U, 0x0BU, 0x7FU, 0x00U, 0x41U, 0x82U, 0x0AU, 0x0BU, 0x7FU, + 0x00U, 0x41U, 0x80U, 0x08U, 0x0BU, 0x7FU, 0x00U, 0x41U, 0x90U, 0x8AU, + 0x04U, 0x0BU, 0x7FU, 0x00U, 0x41U, 0x80U, 0x08U, 0x0BU, 0x7FU, 0x00U, + 0x41U, 0x80U, 0x08U, 0x0BU, 0x07U, 0x0FU, 0x02U, 0x04U, 0x63U, 0x62U, + 0x61U, 0x6BU, 0x00U, 0x09U, 0x04U, 0x68U, 0x6FU, 0x6FU, 0x6BU, 0x00U, + 0x0AU, 0x0AU, 0xA8U, 0x83U, 0x00U, 0x02U, 0x99U, 0x80U, 0x00U, 0x00U, + 0x41U, 0x01U, 0x41U, 0x01U, 0x10U, 0x80U, 0x80U, 0x80U, 0x80U, 0x00U, + 0x1AU, 0x41U, 0x00U, 0x41U, 0x00U, 0x42U, 0x00U, 0x10U, 0x81U, 0x80U, + 0x80U, 0x80U, 0x00U, 0x0BU, 0x88U, 0x83U, 0x00U, 0x02U, 0x03U, 0x7FU, + 0x01U, 0x7EU, 0x23U, 0x80U, 0x80U, 0x80U, 0x80U, 0x00U, 0x41U, 0x20U, + 0x6BU, 0x22U, 0x01U, 0x24U, 0x80U, 0x80U, 0x80U, 0x80U, 0x00U, 0x41U, + 0x01U, 0x41U, 0x01U, 0x10U, 0x80U, 0x80U, 0x80U, 0x80U, 0x00U, 0x1AU, + 0x41U, 0x01U, 0x10U, 0x82U, 0x80U, 0x80U, 0x80U, 0x00U, 0x1AU, 0x41U, + 0x00U, 0x10U, 0x83U, 0x80U, 0x80U, 0x80U, 0x00U, 0xA7U, 0x22U, 0x02U, + 0x41U, 0x05U, 0x6AU, 0x22U, 0x03U, 0x41U, 0x18U, 0x74U, 0x20U, 0x03U, + 0x41U, 0x08U, 0x74U, 0x41U, 0x80U, 0x80U, 0xFCU, 0x07U, 0x71U, 0x72U, + 0x20U, 0x03U, 0x41U, 0x08U, 0x76U, 0x41U, 0x80U, 0xFEU, 0x03U, 0x71U, + 0x20U, 0x03U, 0x41U, 0x18U, 0x76U, 0x72U, 0x72U, 0x36U, 0x02U, 0x95U, + 0x88U, 0x80U, 0x80U, 0x00U, 0x41U, 0x00U, 0x20U, 0x02U, 0x41U, 0x01U, + 0x6AU, 0x22U, 0x03U, 0x41U, 0x18U, 0x74U, 0x20U, 0x03U, 0x41U, 0x08U, + 0x74U, 0x41U, 0x80U, 0x80U, 0xFCU, 0x07U, 0x71U, 0x72U, 0x20U, 0x03U, + 0x41U, 0x08U, 0x76U, 0x41U, 0x80U, 0xFEU, 0x03U, 0x71U, 0x20U, 0x03U, + 0x41U, 0x18U, 0x76U, 0x72U, 0x72U, 0x36U, 0x02U, 0x8FU, 0x88U, 0x80U, + 0x80U, 0x00U, 0x41U, 0xC7U, 0x88U, 0x80U, 0x80U, 0x00U, 0x41U, 0x14U, + 0x10U, 0x84U, 0x80U, 0x80U, 0x80U, 0x00U, 0x1AU, 0x41U, 0xDBU, 0x88U, + 0x80U, 0x80U, 0x00U, 0x41U, 0x8AU, 0x01U, 0x10U, 0x85U, 0x80U, 0x80U, + 0x80U, 0x00U, 0x1AU, 0x41U, 0x00U, 0x41U, 0x80U, 0x88U, 0x80U, 0x80U, + 0x00U, 0x41U, 0xE5U, 0x01U, 0x10U, 0x86U, 0x80U, 0x80U, 0x80U, 0x00U, + 0x22U, 0x04U, 0x3CU, 0x00U, 0xA1U, 0x88U, 0x80U, 0x80U, 0x00U, 0x41U, + 0x00U, 0x20U, 0x04U, 0x42U, 0x08U, 0x88U, 0x3CU, 0x00U, 0xA0U, 0x88U, + 0x80U, 0x80U, 0x00U, 0x41U, 0x00U, 0x20U, 0x04U, 0x42U, 0x10U, 0x88U, + 0x3CU, 0x00U, 0x9FU, 0x88U, 0x80U, 0x80U, 0x00U, 0x41U, 0x00U, 0x20U, + 0x04U, 0x42U, 0x18U, 0x88U, 0x3CU, 0x00U, 0x9EU, 0x88U, 0x80U, 0x80U, + 0x00U, 0x41U, 0x00U, 0x20U, 0x04U, 0x42U, 0x20U, 0x88U, 0x3CU, 0x00U, + 0x9DU, 0x88U, 0x80U, 0x80U, 0x00U, 0x41U, 0x00U, 0x20U, 0x04U, 0x42U, + 0x28U, 0x88U, 0x3CU, 0x00U, 0x9CU, 0x88U, 0x80U, 0x80U, 0x00U, 0x41U, + 0x00U, 0x20U, 0x04U, 0x42U, 0x30U, 0x88U, 0x3CU, 0x00U, 0x9BU, 0x88U, + 0x80U, 0x80U, 0x00U, 0x41U, 0x00U, 0x20U, 0x04U, 0x42U, 0x38U, 0x88U, + 0xA7U, 0x41U, 0x3FU, 0x71U, 0x41U, 0xC0U, 0x00U, 0x72U, 0x3AU, 0x00U, + 0x9AU, 0x88U, 0x80U, 0x80U, 0x00U, 0x02U, 0x40U, 0x02U, 0x40U, 0x20U, + 0x01U, 0x41U, 0x20U, 0x41U, 0x80U, 0x88U, 0x80U, 0x80U, 0x00U, 0x41U, + 0xE5U, 0x01U, 0x10U, 0x87U, 0x80U, 0x80U, 0x80U, 0x00U, 0x42U, 0x7FU, + 0x55U, 0x0DU, 0x00U, 0x41U, 0xE5U, 0x89U, 0x80U, 0x80U, 0x00U, 0x41U, + 0x0DU, 0x42U, 0xD3U, 0x00U, 0x10U, 0x88U, 0x80U, 0x80U, 0x80U, 0x00U, + 0x21U, 0x04U, 0x0CU, 0x01U, 0x0BU, 0x41U, 0xF2U, 0x89U, 0x80U, 0x80U, + 0x00U, 0x41U, 0x10U, 0x42U, 0xD4U, 0x00U, 0x10U, 0x81U, 0x80U, 0x80U, + 0x80U, 0x00U, 0x21U, 0x04U, 0x0BU, 0x20U, 0x01U, 0x41U, 0x20U, 0x6AU, + 0x24U, 0x80U, 0x80U, 0x80U, 0x80U, 0x00U, 0x20U, 0x04U, 0x0BU, 0x0BU, + 0x90U, 0x02U, 0x02U, 0x00U, 0x41U, 0x80U, 0x08U, 0x0BU, 0xE5U, 0x01U, + 0x12U, 0x00U, 0x63U, 0x22U, 0x00U, 0x00U, 0x00U, 0x00U, 0x24U, 0x00U, + 0x00U, 0x00U, 0x00U, 0x20U, 0x1AU, 0x00U, 0x00U, 0x00U, 0x00U, 0x20U, + 0x1BU, 0x00U, 0x00U, 0x00U, 0x00U, 0x68U, 0x40U, 0x00U, 0x00U, 0x00U, + 0x00U, 0x00U, 0x00U, 0x00U, 0x73U, 0x21U, 0x00U, 0x00U, 0x00U, 0x00U, + 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, + 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, + 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x81U, + 0x14U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, + 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, + 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, + 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, + 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, + 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, + 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, + 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, + 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, + 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, + 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, + 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, + 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, + 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, + 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, + 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, 0x00U, + 0x41U, 0xE5U, 0x09U, 0x0BU, 0x1DU, 0x45U, 0x6DU, 0x69U, 0x74U, 0x20U, + 0x66U, 0x61U, 0x69U, 0x6CU, 0x65U, 0x64U, 0x2EU, 0x00U, 0x45U, 0x6DU, + 0x69U, 0x74U, 0x20U, 0x73U, 0x75U, 0x63U, 0x63U, 0x65U, 0x65U, 0x64U, + 0x65U, 0x64U, 0x2EU, 0x00U, + }}, + }; } } // namespace ripple diff --git a/src/test/app/Touch_test.cpp b/src/test/app/Touch_test.cpp index f3a545f5d..bcb7f246c 100644 --- a/src/test/app/Touch_test.cpp +++ b/src/test/app/Touch_test.cpp @@ -207,19 +207,25 @@ private: test::jtx::Env env{*this, envconfig(), features}; auto const alice = Account("alice"); - auto const issuer = Account("issuer"); - env.fund(XRP(1000), alice, issuer); + auto const issuer = Account::master; + env.fund(XRP(1000), alice); + env.close(); + + env(hook(issuer, {{hso(jtx::genesis::AcceptHook)}}, 0), fee(XRP(1))); env.close(); // claim reward - env(reward::claim(alice), reward::issuer(issuer), ter(tesSUCCESS)); + env(reward::claim(alice), + reward::issuer(issuer), + fee(XRP(1)), + ter(tesSUCCESS)); env.close(); // verify touch validateTouch(env, alice, {"ClaimReward", "tesSUCCESS"}); auto const tt = env.current()->rules().enabled(featureTouch) ? "ClaimReward" - : "AccountSet"; + : "SetHook"; validateTouch(env, issuer, {tt, "tesSUCCESS"}); } diff --git a/src/test/core/Config_test.cpp b/src/test/core/Config_test.cpp index a99d48f3a..7214d7764 100644 --- a/src/test/core/Config_test.cpp +++ b/src/test/core/Config_test.cpp @@ -1513,6 +1513,133 @@ r.ripple.com:51235 } } + void + testDatabasePinnedValidation() + { + testcase("DatabasePinned config validation"); + + // Test 1: pinned_type=rwdb in non-standalone mode (should throw) + { + Config c; + std::string toLoad = + "[node_db]\n" + "type=NuDB\n" + "path=main\n" + "online_delete=256\n" + "pinned_type=rwdb\n" + "pinned_path=pinned\n"; + c.setupControl(true, true, false); // standalone = false + try + { + c.loadFromString(toLoad); + fail("Expected exception for non-durable pinned_type"); + } + catch (std::runtime_error const& e) + { + BEAST_EXPECT( + std::string(e.what()).find("durable backend") != + std::string::npos); + pass(); + } + } + + // Test 2: pinned_type=none in non-standalone mode (should throw) + { + Config c; + std::string toLoad = + "[node_db]\n" + "type=NuDB\n" + "path=main\n" + "online_delete=256\n" + "pinned_type=none\n" + "pinned_path=pinned\n"; + c.setupControl(true, true, false); // standalone = false + try + { + c.loadFromString(toLoad); + fail("Expected exception for non-durable pinned_type"); + } + catch (std::runtime_error const& e) + { + BEAST_EXPECT( + std::string(e.what()).find("durable backend") != + std::string::npos); + pass(); + } + } + + // Test 3: pinned_type=rwdb in standalone mode (should succeed + // — standalone is unrestricted) + { + Config c; + std::string toLoad = + "[node_db]\n" + "type=NuDB\n" + "path=main\n" + "online_delete=256\n" + "pinned_type=rwdb\n" + "pinned_path=pinned\n"; + c.setupControl(true, true, true); // standalone = true + try + { + c.loadFromString(toLoad); + pass(); + } + catch (std::runtime_error const& e) + { + fail("Should not throw in standalone mode"); + } + } + + // Test 4: pinned_type=NuDB in non-standalone mode (should succeed) + { + Config c; + std::string toLoad = + "[node_db]\n" + "type=NuDB\n" + "path=main\n" + "online_delete=256\n" + "pinned_type=NuDB\n" + "pinned_path=pinned\n"; + c.setupControl(true, true, false); // standalone = false + try + { + c.loadFromString(toLoad); + pass(); + } + catch (std::runtime_error const& e) + { + fail("Should not throw for durable pinned_type"); + } + } + + // Test 5: pinned_type without online_delete (should throw) + { + Config c; + std::string toLoad = + "[node_db]\n" + "type=NuDB\n" + "path=main\n" + "pinned_type=NuDB\n" + "pinned_path=pinned\n"; + c.setupControl(true, true, false); + try + { + c.loadFromString(toLoad); + fail( + "Expected exception for pinned_type without " + "online_delete"); + } + catch (std::runtime_error const& e) + { + BEAST_EXPECT( + std::string(e.what()).find("online_delete") != + std::string::npos); + pass(); + } + } + } + void testOverlay() { @@ -1604,6 +1731,7 @@ r.ripple.com:51235 testGetters(); testAmendment(); testRWDBOnlineDelete(); + testDatabasePinnedValidation(); testOverlay(); testNetworkID(); } diff --git a/src/test/jtx/AMMTest.h b/src/test/jtx/AMMTest.h index 4902e5f49..ba1bc4b2b 100644 --- a/src/test/jtx/AMMTest.h +++ b/src/test/jtx/AMMTest.h @@ -93,7 +93,8 @@ protected: std::optional> const& pool = std::nullopt, std::uint16_t tfee = 0, std::optional const& ter = std::nullopt, - std::vector const& features = {supported_amendments()}); + std::vector const& features = { + supported_amendments() | featureAMM | featureAMMClawback}); void testAMM( diff --git a/src/test/jtx/impl/AMM.cpp b/src/test/jtx/impl/AMM.cpp index 66a866ea0..acfa3aef1 100644 --- a/src/test/jtx/impl/AMM.cpp +++ b/src/test/jtx/impl/AMM.cpp @@ -31,7 +31,7 @@ namespace ripple { namespace test { namespace jtx { -static Number +[[maybe_unused]] static Number number(STAmount const& a) { if (isXRP(a)) diff --git a/src/test/jtx/impl/AMMTest.cpp b/src/test/jtx/impl/AMMTest.cpp index eb80d9ce7..9224de213 100644 --- a/src/test/jtx/impl/AMMTest.cpp +++ b/src/test/jtx/impl/AMMTest.cpp @@ -197,7 +197,8 @@ AMMTest::pathTestEnv() cfg->PATH_SEARCH_MAX = 10; return cfg; }), - supported_amendments() - featureXahauGenesis - featureTouch); + (supported_amendments() | featureAMM | featureAMMClawback) - + featureXahauGenesis - featureTouch); } Json::Value diff --git a/src/test/jtx/impl/reward.cpp b/src/test/jtx/impl/reward.cpp index 5b48a0e64..132a5ddce 100644 --- a/src/test/jtx/impl/reward.cpp +++ b/src/test/jtx/impl/reward.cpp @@ -49,6 +49,13 @@ issuer::operator()(Env& env, JTx& jt) const jt.jv[sfIssuer.jsonName] = to_string(issuer_); } +void +claimCurrency::operator()(Env& env, JTx& jt) const +{ + jt.jv[sfClaimCurrency.jsonName] = + STIssue{sfClaimCurrency, claimCurrency_}.getJson(JsonOptions::none); +} + } // namespace reward } // namespace jtx diff --git a/src/test/jtx/reward.h b/src/test/jtx/reward.h index bb0e0c74b..183dd2b95 100644 --- a/src/test/jtx/reward.h +++ b/src/test/jtx/reward.h @@ -56,6 +56,21 @@ public: operator()(Env&, JTx& jtx) const; }; +/** Sets the optional ClaimCurrency on a JTx. */ +class claimCurrency +{ +private: + Issue claimCurrency_; + +public: + explicit claimCurrency(Issue const& issue) : claimCurrency_(issue) + { + } + + void + operator()(Env&, JTx& jtx) const; +}; + } // namespace reward } // namespace jtx diff --git a/src/test/ledger/Invariants_test.cpp b/src/test/ledger/Invariants_test.cpp index 54bd6c93e..adb6652d0 100644 --- a/src/test/ledger/Invariants_test.cpp +++ b/src/test/ledger/Invariants_test.cpp @@ -78,7 +78,7 @@ class Invariants_test : public beast::unit_test::suite { using namespace test::jtx; FeatureBitset amendments = - supported_amendments() | featureInvariantsV1_1; + supported_amendments() | featureInvariantsV1_1 | featureAMM; Env env{*this, amendments}; Account const A1{"A1"}; diff --git a/src/test/nodestore/DatabasePinned_test.cpp b/src/test/nodestore/DatabasePinned_test.cpp new file mode 100644 index 000000000..c937241b5 --- /dev/null +++ b/src/test/nodestore/DatabasePinned_test.cpp @@ -0,0 +1,195 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2026 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ripple { +namespace test { + +class DatabasePinned_test : public beast::unit_test::suite +{ + // Build an Env whose nodestore is wired up as DatabasePinnedImp. + // pinned_type=rwdb is allowed in standalone mode (the catalogue + // tests do the same). + static std::unique_ptr + pinnedEnvconfig() + { + auto cfg = jtx::envconfig(); + auto& nodeDb = cfg->section(ConfigSection::nodeDatabase()); + nodeDb.set("pinned_type", "rwdb"); + nodeDb.set("online_delete", "256"); + return cfg; + } + + // Make a NodeObject with deterministic but unique-per-call hash so + // we can store it and fetch it back. + static std::shared_ptr + makeObject(NodeObjectType type, std::uint8_t marker) + { + Blob data(64, marker); + // Hash is just a stable function of (type, marker) so each + // (type, marker) pair has a distinct key. + Serializer s; + s.add32(static_cast(type)); + s.add8(marker); + auto const hash = sha512Half(makeSlice(s.peekData())); + return NodeObject::createObject(type, std::move(data), hash); + } + + // Borrow the running app's nodestore. With pinned_type configured + // it must be a DatabasePinnedImp; otherwise the test environment + // is set up wrong and we can't proceed. + NodeStore::DatabasePinnedImp* + getPinnedDb(jtx::Env& env) + { + auto* db = dynamic_cast( + &env.app().getNodeStore()); + BEAST_EXPECT(db != nullptr); + return db; + } + + void + testStoreRoutingByType() + { + // The architectural core of DatabasePinnedImp: store() routes + // by NodeObjectType. pinned* types go to the persistent + // backend (which never rotates); everything else goes to the + // rotating backend (which is rotated by online_delete). If + // routing breaks, pinned data ends up rotating away and the + // whole feature fails silently. + testcase("store routing by NodeObjectType"); + + using namespace jtx; + Env env{*this, pinnedEnvconfig()}; + auto* db = getPinnedDb(env); + if (!db) + return; + + auto const pinnedBefore = db->pinnedStoreCount(); + auto const hotBefore = db->hotStoreCount(); + + // Pinned types: should each increment pinnedStoreCount. + // The store() implementation also reduces them to their hot + // equivalents before serialization (single-byte on-disk type + // field), but the routing decision happens first. + for (auto const type : + {pinnedLEDGER, pinnedACCOUNT_NODE, pinnedTRANSACTION_NODE}) + { + db->store( + type, + Blob(64, 0xAB), + sha512Half( + makeSlice(std::string{"pin-"} + std::to_string(type))), + /*ledgerSeq=*/100); + } + + // Hot types: should each increment hotStoreCount. + for (auto const type : + {hotLEDGER, hotACCOUNT_NODE, hotTRANSACTION_NODE}) + { + db->store( + type, + Blob(64, 0xCD), + sha512Half( + makeSlice(std::string{"hot-"} + std::to_string(type))), + /*ledgerSeq=*/100); + } + + // 3 pinned stores routed to persistent_, 3 hot to rotating_. + BEAST_EXPECT(db->pinnedStoreCount() - pinnedBefore == 3u); + BEAST_EXPECT(db->hotStoreCount() - hotBefore == 3u); + } + + void + testStoreFetchRoundTrip() + { + // Verifies that data routed to either backend is retrievable + // through the unified Database interface (fetchNodeObject + // returns the right object regardless of which backend it + // landed in). This is the read-side counterpart to the + // routing test — confirms callers don't need to know about + // the dual-backend split. + testcase("store/fetch round-trip across both backends"); + + using namespace jtx; + Env env{*this, pinnedEnvconfig()}; + auto& nodeStore = env.app().getNodeStore(); + + // Store one object of each type with distinct content so the + // fetched data is unambiguously the one we stored. + struct Item + { + NodeObjectType type; + std::uint8_t marker; + std::shared_ptr obj; + }; + + std::vector items{ + {pinnedLEDGER, 0x01, nullptr}, + {pinnedACCOUNT_NODE, 0x02, nullptr}, + {pinnedTRANSACTION_NODE, 0x03, nullptr}, + {hotLEDGER, 0x04, nullptr}, + {hotACCOUNT_NODE, 0x05, nullptr}, + {hotTRANSACTION_NODE, 0x06, nullptr}, + }; + + for (auto& it : items) + { + it.obj = makeObject(it.type, it.marker); + // Database::store takes (type, data, hash, ledgerSeq). + // Pass a copy of the data because store() takes Blob&&. + nodeStore.store( + it.type, + Blob(it.obj->getData()), + it.obj->getHash(), + /*ledgerSeq=*/100); + } + + for (auto const& it : items) + { + auto const fetched = + nodeStore.fetchNodeObject(it.obj->getHash(), 100); + BEAST_EXPECT(fetched != nullptr); + if (!fetched) + continue; + BEAST_EXPECT(fetched->getData() == it.obj->getData()); + } + } + +public: + void + run() override + { + testStoreRoutingByType(); + testStoreFetchRoundTrip(); + } +}; + +BEAST_DEFINE_TESTSUITE(DatabasePinned, nodestore, ripple); + +} // namespace test +} // namespace ripple diff --git a/src/test/rdb/RelationalDatabase_test.cpp b/src/test/rdb/RelationalDatabase_test.cpp index bfacf2142..9edf1b2ea 100644 --- a/src/test/rdb/RelationalDatabase_test.cpp +++ b/src/test/rdb/RelationalDatabase_test.cpp @@ -541,6 +541,124 @@ public: } } + void + testRangeDeletionOperations( + std::string const& backend, + std::unique_ptr config) + { + testcase("Range deletion operations - " + backend); + + using namespace test::jtx; + config->LEDGER_HISTORY = 1000; + + Env env(*this, std::move(config)); + auto& db = env.app().getRelationalDatabase(); + + Account alice("alice"); + Account bob("bob"); + + env.fund(XRP(10000), alice, bob); + env.close(); + + auto* sqliteDb = getInterface(db); + BEAST_EXPECT(sqliteDb != nullptr); + if (!sqliteDb) + return; + + // Create 5 ledgers with transactions + for (int i = 0; i < 5; ++i) + { + env(pay(alice, bob, XRP(100 + i))); + env.close(); + } + + auto minSeq = sqliteDb->getMinLedgerSeq(); + auto maxSeq = sqliteDb->getMaxLedgerSeq(); + BEAST_EXPECT(minSeq && maxSeq); + if (!minSeq || !maxSeq) + return; + + auto initialTxCount = sqliteDb->getTransactionCount(); + auto initialAcctTxCount = sqliteDb->getAccountTransactionCount(); + + // Use a range that covers ledgers with transactions + // (funded ledger + 5 payment ledgers = sequences 3..7 typically) + LedgerIndex rangeStart = *minSeq; + LedgerIndex rangeEnd = *maxSeq; + + // Test deleteTransactionsInRange with limit — ensures forward + // progress (no infinite loop on partial deletes) + { + std::size_t totalDeleted = 0; + std::size_t iterations = 0; + std::size_t const limit = 2; + + while (true) + { + auto deleted = sqliteDb->deleteTransactionsInRange( + rangeStart, rangeEnd, limit); + if (deleted == 0) + break; + totalDeleted += deleted; + ++iterations; + // Safety: prevent infinite loop in case of bug + if (iterations > initialTxCount + 1) + { + fail( + "deleteTransactionsInRange: too many iterations, " + "possible infinite loop"); + break; + } + } + BEAST_EXPECT(totalDeleted > 0); + BEAST_EXPECT(totalDeleted <= initialTxCount); + auto afterTxCount = sqliteDb->getTransactionCount(); + BEAST_EXPECT(afterTxCount == initialTxCount - totalDeleted); + } + + // Test deleteAccountTransactionsInRange with limit + { + std::size_t totalDeleted = 0; + std::size_t iterations = 0; + std::size_t const limit = 2; + + while (true) + { + auto deleted = sqliteDb->deleteAccountTransactionsInRange( + rangeStart, rangeEnd, limit); + if (deleted == 0) + break; + totalDeleted += deleted; + ++iterations; + if (iterations > initialAcctTxCount + 1) + { + fail( + "deleteAccountTransactionsInRange: too many " + "iterations, possible infinite loop"); + break; + } + } + BEAST_EXPECT(totalDeleted > 0); + BEAST_EXPECT(totalDeleted <= initialAcctTxCount); + auto afterAcctTxCount = sqliteDb->getAccountTransactionCount(); + BEAST_EXPECT(afterAcctTxCount == initialAcctTxCount - totalDeleted); + } + + // Test deleteLedgersInRange + auto initialLedgerCount = sqliteDb->getLedgerCountMinMax(); + auto ledgersDeleted = + sqliteDb->deleteLedgersInRange(rangeStart + 1, rangeEnd - 1); + BEAST_EXPECT(ledgersDeleted > 0); + auto afterLedgerCount = sqliteDb->getLedgerCountMinMax(); + BEAST_EXPECT( + afterLedgerCount.numberOfRows == + initialLedgerCount.numberOfRows - ledgersDeleted); + + // Test range deletion with no matching range + auto emptyDeleted = sqliteDb->deleteLedgersInRange(999999, 999999); + BEAST_EXPECT(emptyDeleted == 0); + } + void testDatabaseSpaceOperations( std::string const& backend, @@ -744,6 +862,7 @@ public: testAccountTransactionOperations(backend, makeConfig(backend)); testAccountTransactionPaging(backend, makeConfig(backend)); testDeletionOperations(backend, makeConfig(backend)); + testRangeDeletionOperations(backend, makeConfig(backend)); testDatabaseSpaceOperations(backend, makeConfig(backend)); testTransactionMinLedgerSeq(backend, makeConfig(backend)); } diff --git a/src/test/rpc/AMMInfo_test.cpp b/src/test/rpc/AMMInfo_test.cpp index 6503de16a..c9face930 100644 --- a/src/test/rpc/AMMInfo_test.cpp +++ b/src/test/rpc/AMMInfo_test.cpp @@ -347,7 +347,8 @@ public: run() override { using namespace jtx; - auto const all = supported_amendments(); + auto const all = + supported_amendments() | featureAMM | featureAMMClawback; testErrors(); testSimpleRpc(); testVoteAndBid(all); diff --git a/src/test/rpc/AccountObjects_test.cpp b/src/test/rpc/AccountObjects_test.cpp index dea17a564..73090e1a2 100644 --- a/src/test/rpc/AccountObjects_test.cpp +++ b/src/test/rpc/AccountObjects_test.cpp @@ -577,8 +577,8 @@ public: Env env( *this, - features | featureXChainBridge | featureDID | featureCredentials | - featurePermissionedDomains); + features | featureAMM | featureXChainBridge | featureDID | + featureCredentials | featurePermissionedDomains); // Make a lambda we can use to get "account_objects" easily. auto acctObjs = [&env]( diff --git a/src/test/rpc/CatalogueStream_test.cpp b/src/test/rpc/CatalogueStream_test.cpp new file mode 100644 index 000000000..27e358bd4 --- /dev/null +++ b/src/test/rpc/CatalogueStream_test.cpp @@ -0,0 +1,322 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2025 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +namespace ripple { +namespace test { + +class CatalogueStream_test : public beast::unit_test::suite +{ + // The deserialize* public API takes `CatalogueInputStream&` which is just + // `boost::iostreams::filtering_istream`. So no production-code change is + // needed for fault-injection — we hand-build a byte vector that violates + // the wire format, push it through an `array_source`, and feed it to + // deserialize. Each test below targets one error branch in + // CatalogueStream.cpp. + // + // Wire format reminder (from serializeSHAMapToStream): + // leaf : [type:u8 ][key:32][size:u32][data:size] + // remove: [254 :u8 ][key:32] + // end : [255 :u8 ] + + using Bytes = std::vector; + + static void + appendByte(Bytes& b, std::uint8_t v) + { + b.push_back(v); + } + + static void + appendBytes(Bytes& b, void const* p, std::size_t n) + { + auto const* src = reinterpret_cast(p); + b.insert(b.end(), src, src + n); + } + + static void + appendKey(Bytes& b, std::uint8_t fillByte = 0xAA) + { + b.insert(b.end(), 32, fillByte); + } + + static void + appendU32(Bytes& b, std::uint32_t v) + { + // CatalogueStream writes raw bytes — no endian conversion. Match. + appendBytes(b, &v, sizeof(v)); + } + + // Build a fresh filtering_istream over `bytes`. The array_source is a + // value type stored by the filter chain, so `bytes` must outlive it. + static void + makeStream(Bytes const& bytes, RPC::CatalogueInputStream& out) + { + boost::iostreams::array_source src( + reinterpret_cast(bytes.data()), bytes.size()); + out.push(src); + } + + // Construct an empty SHAMap of the requested type using the test app's + // node family. Same pattern used in src/test/consensus/UNLReport_test.cpp. + static std::shared_ptr + makeMap(jtx::Env& env, SHAMapType type) + { + return std::make_shared(type, env.app().getNodeFamily()); + } + +public: + void + testTruncatedKey() + { + testcase("deserialize: truncated read of key"); + + jtx::Env env(*this); + auto map = makeMap(env, SHAMapType::STATE); + + // 1 byte (the node type) and nothing after — read of the 32-byte key + // hits EOF inside deserializeLeaf, triggers the "stream stopped while + // trying to read key" branch. + Bytes b; + appendByte(b, tnACCOUNT_STATE); + + RPC::CatalogueInputStream stream; + makeStream(b, stream); + + BEAST_EXPECT(!RPC::deserializeStateMapFromStream( + *map, stream, hotACCOUNT_NODE, env.app().journal("Test"))); + } + + void + testTruncatedSize() + { + testcase("deserialize: truncated read of size"); + + jtx::Env env(*this); + auto map = makeMap(env, SHAMapType::TRANSACTION); + + // type + key but no 4-byte size — triggers the "stream stopped while + // trying to read size" branch. Use tx map so the type byte isn't a + // remove (which would short-circuit before the size read). + Bytes b; + appendByte(b, tnTRANSACTION_MD); + appendKey(b); + + RPC::CatalogueInputStream stream; + makeStream(b, stream); + + BEAST_EXPECT(!RPC::deserializeTxMapFromStream( + *map, stream, hotTRANSACTION_NODE, env.app().journal("Test"))); + } + + void + testSizeTooLarge() + { + testcase("deserialize: size > 1 GiB rejected"); + + jtx::Env env(*this); + auto map = makeMap(env, SHAMapType::STATE); + + // type + key + size of 2 GiB — bails before any data read attempt. + Bytes b; + appendByte(b, tnACCOUNT_STATE); + appendKey(b); + appendU32(b, 2u * 1024u * 1024u * 1024u); + + RPC::CatalogueInputStream stream; + makeStream(b, stream); + + BEAST_EXPECT(!RPC::deserializeStateMapFromStream( + *map, stream, hotACCOUNT_NODE, env.app().journal("Test"))); + } + + void + testTruncatedData() + { + testcase("deserialize: truncated read of data payload"); + + jtx::Env env(*this); + auto map = makeMap(env, SHAMapType::STATE); + + // type + key + size=10 + only 5 bytes — hits the "Unexpected EOF + // while reading data" branch. + Bytes b; + appendByte(b, tnACCOUNT_STATE); + appendKey(b); + appendU32(b, 10); + for (int i = 0; i < 5; ++i) + appendByte(b, 0x42); + + RPC::CatalogueInputStream stream; + makeStream(b, stream); + + BEAST_EXPECT(!RPC::deserializeStateMapFromStream( + *map, stream, hotACCOUNT_NODE, env.app().journal("Test"))); + } + + void + testRemoveDisallowedInTxMap() + { + testcase( + "deserialize: remove sentinel rejected when allowRemoval=false"); + + jtx::Env env(*this); + auto map = makeMap(env, SHAMapType::TRANSACTION); + + // tx maps don't accept tnREMOVE — the deserialize for tx maps calls + // the inner helper with allowRemoval=false. We send a remove record + // and expect rejection at "unexpected removal in this map type". + Bytes b; + appendByte(b, tnREMOVE); + appendKey(b); + + RPC::CatalogueInputStream stream; + makeStream(b, stream); + + BEAST_EXPECT(!RPC::deserializeTxMapFromStream( + *map, stream, hotTRANSACTION_NODE, env.app().journal("Test"))); + } + + void + testRemoveOfAbsentKey() + { + testcase("deserialize: remove of absent key rejected"); + + jtx::Env env(*this); + auto map = makeMap(env, SHAMapType::STATE); + + // State map allows tnREMOVE, but the key has to be present first. + // Empty map + remove of any key → "key is already absent" branch. + Bytes b; + appendByte(b, tnREMOVE); + appendKey(b, 0x77); + + RPC::CatalogueInputStream stream; + makeStream(b, stream); + + BEAST_EXPECT(!RPC::deserializeStateMapFromStream( + *map, stream, hotACCOUNT_NODE, env.app().journal("Test"))); + } + + void + testMissingTerminator() + { + testcase("deserialize: EOF before terminal sentinel"); + + jtx::Env env(*this); + auto map = makeMap(env, SHAMapType::STATE); + + // One valid leaf, then EOF (no tnTERMINAL). The outer loop exits via + // stream.eof() with lastParsed = tnACCOUNT_STATE (not tnTERMINAL), + // so the "Unexpected EOF, terminal node not found" branch fires. + // Payload must be >= 12 bytes — SHAMapLeafNode asserts that for + // state-tree leaves. + Bytes b; + appendByte(b, tnACCOUNT_STATE); + appendKey(b, 0x11); + appendU32(b, 16); + for (int i = 0; i < 16; ++i) + appendByte(b, 0xCC); + + RPC::CatalogueInputStream stream; + makeStream(b, stream); + + BEAST_EXPECT(!RPC::deserializeStateMapFromStream( + *map, stream, hotACCOUNT_NODE, env.app().journal("Test"))); + } + + void + testEmptyStreamAcceptsTerminator() + { + testcase("deserialize: bare terminator → empty map, success"); + + jtx::Env env(*this); + auto map = makeMap(env, SHAMapType::STATE); + + Bytes b; + appendByte(b, tnTERMINAL); + + RPC::CatalogueInputStream stream; + makeStream(b, stream); + + BEAST_EXPECT(RPC::deserializeStateMapFromStream( + *map, stream, hotACCOUNT_NODE, env.app().journal("Test"))); + } + + void + testHappyPathRoundTrip() + { + testcase("deserialize: valid leaf + terminator → success"); + + jtx::Env env(*this); + auto map = makeMap(env, SHAMapType::STATE); + + // type + key + size=16 + 16 data bytes + terminator. Confirms the + // happy path through addGiveItem and flushDirty. Size must be >= 12 + // — SHAMapLeafNode asserts that for state-tree leaves. + Bytes b; + appendByte(b, tnACCOUNT_STATE); + appendKey(b, 0x55); + appendU32(b, 16); + for (int i = 0; i < 16; ++i) + appendByte(b, static_cast(i)); + appendByte(b, tnTERMINAL); + + RPC::CatalogueInputStream stream; + makeStream(b, stream); + + BEAST_EXPECT(RPC::deserializeStateMapFromStream( + *map, stream, hotACCOUNT_NODE, env.app().journal("Test"))); + + uint256 key; + std::memset(key.data(), 0x55, 32); + BEAST_EXPECT(map->hasItem(key)); + } + + void + run() override + { + testTruncatedKey(); + testTruncatedSize(); + testSizeTooLarge(); + testTruncatedData(); + testRemoveDisallowedInTxMap(); + testRemoveOfAbsentKey(); + testMissingTerminator(); + testEmptyStreamAcceptsTerminator(); + testHappyPathRoundTrip(); + } +}; + +BEAST_DEFINE_TESTSUITE(CatalogueStream, rpc, ripple); + +} // namespace test +} // namespace ripple diff --git a/src/test/rpc/Catalogue_test.cpp b/src/test/rpc/Catalogue_test.cpp index d2283f234..421f35564 100644 --- a/src/test/rpc/Catalogue_test.cpp +++ b/src/test/rpc/Catalogue_test.cpp @@ -19,15 +19,103 @@ #include #include +#include #include +#include #include #include #include +#include #include +#include #include namespace ripple { +namespace { + +// Parameters for polling ledger retrieval +struct LedgerRetryParams +{ + test::jtx::Env& env; + std::uint32_t seq = 0; + std::chrono::milliseconds pollInterval = std::chrono::milliseconds(500); + std::chrono::milliseconds maxWaitTime = std::chrono::milliseconds(10000); + bool requireValidated = false; +}; + +// Poll for ledger availability with retry logic +// Returns nullptr if ledger cannot be retrieved within the timeout period +std::shared_ptr +getLedgerWithRetry(const LedgerRetryParams& params) +{ + auto start = std::chrono::steady_clock::now(); + + while (true) + { + // First try to get the hash for this sequence + auto hash = params.env.app().getLedgerMaster().getHashBySeq(params.seq); + + if (hash.isNonZero()) + { + // Hash found, now try to get the ledger + auto ledger = + params.env.app().getLedgerMaster().getLedgerByHash(hash); + + if (ledger) + { + // If we don't require validation, or if it's already validated, + // return it + if (!params.requireValidated || ledger->info().validated) + { + return ledger; + } + // Otherwise continue polling until it becomes validated + } + } + + // Check if we've exceeded the timeout + auto elapsed = std::chrono::steady_clock::now() - start; + if (elapsed >= params.maxWaitTime) + { + return nullptr; + } + + // Wait before next attempt + std::this_thread::sleep_for(params.pollInterval); + } +} + +// Options for catalogueEnvconfig. Use designated initializers at call sites +// so each test reads as a small diff against the default — easy to scan and +// trivially extensible (add fields here, no call-site churn). +// catalogueEnvconfig() // happy path +// catalogueEnvconfig({.withPinnedType = false}) // exercise guard +struct CatalogueEnvOpts +{ + // Set [node_db] pinned_type=rwdb so catalogue_load passes its guard. + // false → omit pinned_type entirely (drives the rejection path). + bool withPinnedType = true; + + // [node_db] online_delete value (lifetime of rotating store, in ledgers). + std::string onlineDelete = "256"; +}; + +// envconfig with pinned_type set so catalogue_load passes the config guard. +// Uses rwdb for in-process speed (allowed in standalone). +inline std::unique_ptr +catalogueEnvconfig(CatalogueEnvOpts const& opts = {}) +{ + auto cfg = test::jtx::envconfig(); + auto& nodeDb = cfg->section(ConfigSection::nodeDatabase()); + if (opts.withPinnedType) + nodeDb.set("pinned_type", "rwdb"); + nodeDb.set("online_delete", opts.onlineDelete); + return cfg; +} + +} // anonymous namespace + #pragma pack(push, 1) // pack the struct tightly struct TestCATLHeader { @@ -215,7 +303,24 @@ class Catalogue_test : public beast::unit_test::suite { testcase("catalogue_load: Invalid parameters"); using namespace test::jtx; - Env env{*this, envconfig(), features}; + Env env{*this, catalogueEnvconfig(), features}; + + // Missing [node_db] pinned_type — catalogue_load rejects up front so + // loaded data isn't lost on the next rotation. Use a separate Env + // because this guard fires before any input_file parsing. + { + Env envNoPin{ + *this, catalogueEnvconfig({.withPinnedType = false}), features}; + Json::Value params{Json::objectValue}; + params[jss::input_file] = "/tmp/anything.catl"; + auto const result = + envNoPin.client().invoke("catalogue_load", params)[jss::result]; + BEAST_EXPECT(result[jss::error] == "invalidParams"); + BEAST_EXPECT(result[jss::status] == "error"); + BEAST_EXPECT( + result[jss::error_message].asString().find("pinned_type") != + std::string::npos); + } // No parameters { @@ -253,6 +358,42 @@ class Catalogue_test : public beast::unit_test::suite BEAST_EXPECT(result[jss::error] == "internal"); BEAST_EXPECT(result[jss::status] == "error"); } + + // Header has min_ledger > max_ledger + { + boost::filesystem::path tempDir = + boost::filesystem::temp_directory_path() / + boost::filesystem::unique_path(); + boost::filesystem::create_directories(tempDir); + + auto cataloguePath = (tempDir / "invalid-range.catl").string(); + + TestCATLHeader header; + header.min_ledger = 20; + header.max_ledger = 10; + header.version = 1; + header.network_id = env.app().config().NETWORK_ID; + header.filesize = sizeof(TestCATLHeader); + + std::ofstream outfile( + cataloguePath.c_str(), std::ios::out | std::ios::binary); + BEAST_EXPECT(outfile.good()); + outfile.write( + reinterpret_cast(&header), sizeof(header)); + outfile.close(); + + Json::Value params{Json::objectValue}; + params[jss::input_file] = cataloguePath; + auto const result = + env.client().invoke("catalogue_load", params)[jss::result]; + BEAST_EXPECT(result[jss::error] == "invalidParams"); + BEAST_EXPECT(result[jss::status] == "error"); + BEAST_EXPECT( + result[jss::error_message].asString().find( + "min_ledger must be <= max_ledger") != std::string::npos); + + boost::filesystem::remove_all(tempDir); + } } void @@ -262,9 +403,25 @@ class Catalogue_test : public beast::unit_test::suite using namespace test::jtx; // Create environment and test data - Env env{*this, envconfig(), features}; + Env env{*this, catalogueEnvconfig(), nullptr, beast::severities::kNone}; prepareLedgerData(env, 5); + auto noop = [](test::jtx::Env& env, + std::string partition, + std::string severity) { + Json::Value params{Json::objectValue}; + params[jss::severity] = severity; + params[jss::partition] = partition; + // env.client().invoke("log_level", params); + }; + + // Create journal for debugging + auto j = env.app().logs().journal("Catalogue_test"); + noop(env, "CatalogueTools", "trace"); + noop(env, "Catalogue_test", "trace"); + + JLOG(j.trace()) << "Test environment created, prepared ledger data"; + // Store some key state information before catalogue creation auto const sourceLedger = env.closed(); auto const bobKeylet = keylet::account(Account("bob").id()); @@ -274,6 +431,9 @@ class Catalogue_test : public beast::unit_test::suite Account("bob").id(), Currency(to_currency("EUR"))); + JLOG(j.trace()) << "Source ledger seq: " << sourceLedger->info().seq + << " hash: " << sourceLedger->info().hash; + // Get original state entries auto const bobAcct = sourceLedger->read(bobKeylet); auto const charlieAcct = sourceLedger->read(charlieKeylet); @@ -302,6 +462,10 @@ class Catalogue_test : public beast::unit_test::suite // First create a catalogue uint32_t minLedger = 3; uint32_t maxLedger = sourceLedger->info().seq; + + JLOG(j.trace()) << "Creating catalogue from ledger " << minLedger + << " to " << maxLedger; + { Json::Value params{Json::objectValue}; params[jss::min_ledger] = minLedger; @@ -311,27 +475,43 @@ class Catalogue_test : public beast::unit_test::suite auto const result = env.client().invoke("catalogue_create", params)[jss::result]; BEAST_EXPECT(result[jss::status] == jss::success); + + JLOG(j.trace()) << "Catalogue created: " << result.toStyledString(); } - // Create a new environment for loading with unique port + // Create a new environment for loading the catalogue + // We use a separate environment with incremented ports to avoid + // conflicts Note: The default RWDB backend works fine - the key insight + // is that pinned ledgers bypass the cache, so we need to poll for + // availability as the async publishAcqLedger jobs complete Env loadEnv{ *this, - test::jtx::envconfig(), + catalogueEnvconfig(), features, }; + noop(loadEnv, "CatalogueTools", "trace"); + noop(loadEnv, "Catalogue_test", "trace"); + + // Create journal for load environment + auto loadJ = loadEnv.app().logs().journal("Catalogue_test"); + // Now load the catalogue Json::Value params{Json::objectValue}; params[jss::input_file] = cataloguePath; + JLOG(loadJ.trace()) << "Loading catalogue from " << cataloguePath; + auto const result = loadEnv.client().invoke("catalogue_load", params)[jss::result]; + JLOG(loadJ.trace()) + << "Catalogue load result: " << result.toStyledString(); + BEAST_EXPECT(result[jss::status] == jss::success); BEAST_EXPECT(result[jss::ledger_min] == minLedger); BEAST_EXPECT(result[jss::ledger_max] == maxLedger); BEAST_EXPECT(result[jss::ledger_count] == (maxLedger - minLedger + 1)); - // Verify complete_ledgers reflects loaded ledgers auto const newCompleteLedgers = loadEnv.app().getLedgerMaster().getCompleteLedgers(); @@ -346,20 +526,36 @@ class Catalogue_test : public beast::unit_test::suite // Compare all ledgers from 3 to 16 inclusive for (std::uint32_t seq = 3; seq <= 16; ++seq) { - auto const sourceLedger = - env.app().getLedgerMaster().getLedgerByHash( - env.app().getLedgerMaster().getHashBySeq(seq)); + JLOG(j.trace()) << "Comparing ledger " << seq; - auto const loadedLedger = - loadEnv.app().getLedgerMaster().getLedgerByHash( - loadEnv.app().getLedgerMaster().getHashBySeq(seq)); + // Get the source ledger (doesn't need to be validated) + auto const sourceLedger = getLedgerWithRetry( + {.env = env, + .seq = seq, + .pollInterval = std::chrono::milliseconds(100), + .maxWaitTime = std::chrono::milliseconds(5000), + .requireValidated = false}); + + // Get the loaded ledger (must be validated) + auto const loadedLedger = getLedgerWithRetry( + {.env = loadEnv, + .seq = seq, + .pollInterval = std::chrono::milliseconds(500), + .maxWaitTime = std::chrono::milliseconds(30000), + .requireValidated = true}); if (!sourceLedger || !loadedLedger) { + JLOG(j.trace()) + << "Failed to get ledger " << seq + << " source: " << (sourceLedger ? "ok" : "missing") + << " loaded: " << (loadedLedger ? "ok" : "missing"); BEAST_EXPECT(false); // Test failure continue; } + JLOG(j.trace()) << "Got both ledgers for seq " << seq; + // Check basic ledger properties BEAST_EXPECT(sourceLedger->info().seq == loadedLedger->info().seq); BEAST_EXPECT( @@ -404,6 +600,10 @@ class Catalogue_test : public beast::unit_test::suite std::size_t sourceCount = std::ranges::distance(sourceLedger->sles); std::size_t loadedCount = std::ranges::distance(loadedLedger->sles); + JLOG(j.trace()) + << "Ledger " << seq << " SLE count - source: " << sourceCount + << " loaded: " << loadedCount; + BEAST_EXPECT(sourceCount == loadedCount); // Check existence of imported keylets @@ -411,6 +611,13 @@ class Catalogue_test : public beast::unit_test::suite { auto const key = sle->key(); bool exists = loadedLedger->exists(keylet::unchecked(key)); + + if (!exists) + { + JLOG(j.trace()) + << "Ledger " << seq << " missing key: " << key; + } + BEAST_EXPECT(exists); // If it exists, check the serialized form matches @@ -421,6 +628,13 @@ class Catalogue_test : public beast::unit_test::suite sle->add(s1); loadedSle->add(s2); bool serializedEqual = (s1.peekData() == s2.peekData()); + + if (!serializedEqual) + { + JLOG(j.trace()) + << "Ledger " << seq << " mismatch for key: " << key; + } + BEAST_EXPECT(serializedEqual); } } @@ -429,10 +643,20 @@ class Catalogue_test : public beast::unit_test::suite for (auto const& sle : loadedLedger->sles) { auto const key = sle->key(); - BEAST_EXPECT(sourceLedger->exists(keylet::unchecked(key))); + bool exists = sourceLedger->exists(keylet::unchecked(key)); + + if (!exists) + { + JLOG(j.trace()) + << "Ledger " << seq << " extra key in loaded: " << key; + } + + BEAST_EXPECT(exists); } } + JLOG(j.trace()) << "Ledger comparison complete"; + auto const loadedBobAcct = loadedLedger->read(bobKeylet); auto const loadedCharlieAcct = loadedLedger->read(charlieKeylet); auto const loadedEurTrust = loadedLedger->read(eurTrustKeylet); @@ -523,10 +747,11 @@ class Catalogue_test : public beast::unit_test::suite // Try to load catalogue in environment with different network ID Env env2{ *this, - envconfig([](std::unique_ptr cfg) { + [&]() { + auto cfg = catalogueEnvconfig(); cfg->NETWORK_ID = 456; return cfg; - }), + }(), features, }; @@ -553,7 +778,7 @@ class Catalogue_test : public beast::unit_test::suite // Create environment and test data Env env{ *this, - envconfig(), + catalogueEnvconfig(), features, nullptr, beast::severities::kDisabled, @@ -650,7 +875,7 @@ class Catalogue_test : public beast::unit_test::suite // Create environment and test data Env env{ *this, - envconfig(), + catalogueEnvconfig(), features, nullptr, beast::severities::kDisabled, @@ -729,7 +954,7 @@ class Catalogue_test : public beast::unit_test::suite using namespace test::jtx; // Create environment and test data - Env env{*this, envconfig(), features}; + Env env{*this, catalogueEnvconfig(), features}; prepareLedgerData(env, 5); boost::filesystem::path tempDir = @@ -818,7 +1043,7 @@ class Catalogue_test : public beast::unit_test::suite using namespace test::jtx; // Create environment - Env env{*this, envconfig(), features}; + Env env{*this, catalogueEnvconfig(), features}; boost::filesystem::path tempDir = boost::filesystem::temp_directory_path() / diff --git a/src/test/rpc/ServerDefinitions_test.cpp b/src/test/rpc/ServerDefinitions_test.cpp index 0529708f4..81fb4c8e3 100644 --- a/src/test/rpc/ServerDefinitions_test.cpp +++ b/src/test/rpc/ServerDefinitions_test.cpp @@ -55,9 +55,9 @@ public: using namespace test::jtx; + Env env(*this); + auto const result = env.rpc("server_definitions"); { - Env env(*this); - auto const result = env.rpc("server_definitions"); BEAST_EXPECT(!result[jss::result].isMember(jss::error)); BEAST_EXPECT(result[jss::result].isMember(jss::FIELDS)); BEAST_EXPECT(result[jss::result].isMember(jss::LEDGER_ENTRY_TYPES)); @@ -71,6 +71,38 @@ public: BEAST_EXPECT(result[jss::result].isMember(jss::hash)); BEAST_EXPECT(result[jss::result][jss::status] == "success"); } + + // check exception SFields + { + auto const fieldExists = [&](std::string name) { + for (auto& field : result[jss::result][jss::FIELDS]) + { + if (field[0u].asString() == name) + { + return true; + } + } + return false; + }; + BEAST_EXPECT(fieldExists("Generic")); + BEAST_EXPECT(fieldExists("Invalid")); + BEAST_EXPECT(fieldExists("ObjectEndMarker")); + BEAST_EXPECT(fieldExists("ArrayEndMarker")); + BEAST_EXPECT(fieldExists("taker_gets_funded")); + BEAST_EXPECT(fieldExists("taker_pays_funded")); + BEAST_EXPECT(fieldExists("hash")); + BEAST_EXPECT(fieldExists("index")); + } + + // verify no duplicate field names in FIELDS array + { + std::set fieldNames; + for (auto const& field : result[jss::result][jss::FIELDS]) + { + auto const name = field[0u].asString(); + BEAST_EXPECT(fieldNames.insert(name).second); + } + } } void diff --git a/src/xrpld/app/hook/HookAPI.h b/src/xrpld/app/hook/HookAPI.h index 91c23e091..abd07c053 100644 --- a/src/xrpld/app/hook/HookAPI.h +++ b/src/xrpld/app/hook/HookAPI.h @@ -3,11 +3,298 @@ #include #include +#include namespace hook { -using namespace ripple; + using HookReturnCode = hook_api::hook_return_code; +namespace hook_float { +using enum hook_api::hook_return_code; + +// power of 10 LUT for fast integer math +static int64_t power_of_ten[19] = { + 1LL, + 10LL, + 100LL, + 1000LL, + 10000LL, + 100000LL, + 1000000LL, + 10000000LL, + 100000000LL, + 1000000000LL, + 10000000000LL, + 100000000000LL, + 1000000000000LL, + 10000000000000LL, + 100000000000000LL, + 1000000000000000LL, // 15 + 10000000000000000LL, + 100000000000000000LL, + 1000000000000000000LL, +}; + +using namespace hook_api; +static int64_t const minMantissa = 1000000000000000ull; +static int64_t const maxMantissa = 9999999999999999ull; +static int32_t const minExponent = -96; +static int32_t const maxExponent = 80; + +inline Expected +get_exponent(int64_t float1) +{ + using enum hook_api::hook_return_code; + if (float1 < 0) + return Unexpected(INVALID_FLOAT); + if (float1 == 0) + return 0u; + uint64_t float_in = (uint64_t)float1; + float_in >>= 54U; + float_in &= 0xFFU; + return static_cast(float_in) - 97; +} + +inline Expected +get_mantissa(int64_t float1) +{ + using enum hook_api::hook_return_code; + if (float1 < 0) + return Unexpected(INVALID_FLOAT); + if (float1 == 0) + return 0ULL; + float1 -= ((((uint64_t)float1) >> 54U) << 54U); + return float1; +} + +inline bool +is_negative(int64_t float1) +{ + return ((float1 >> 62U) & 1ULL) == 0; +} + +inline int64_t +invert_sign(int64_t float1) +{ + int64_t r = (int64_t)(((uint64_t)float1) ^ (1ULL << 62U)); + return r; +} + +inline int64_t +set_sign(int64_t float1, bool set_negative) +{ + bool neg = is_negative(float1); + if ((neg && set_negative) || (!neg && !set_negative)) + return float1; + + return invert_sign(float1); +} + +inline Expected +set_mantissa(int64_t float1, uint64_t mantissa) +{ + using enum hook_api::hook_return_code; + if (mantissa > maxMantissa) + return Unexpected(MANTISSA_OVERSIZED); + if (mantissa < minMantissa) + return Unexpected(MANTISSA_UNDERSIZED); + return float1 - get_mantissa(float1).value() + mantissa; +} + +inline Expected +set_exponent(int64_t float1, int32_t exponent) +{ + using enum hook_api::hook_return_code; + if (exponent > maxExponent) + return Unexpected(EXPONENT_OVERSIZED); + if (exponent < minExponent) + return Unexpected(EXPONENT_UNDERSIZED); + + uint64_t exp = (exponent + 97); + exp <<= 54U; + float1 &= ~(0xFFLL << 54); + float1 += (int64_t)exp; + return float1; +} + +inline Expected +make_float(ripple::IOUAmount& amt) +{ + using enum hook_api::hook_return_code; + int64_t man_out = amt.mantissa(); + int64_t float_out = 0; + bool neg = man_out < 0; + if (neg) + man_out *= -1; + + float_out = set_sign(float_out, neg); + auto const mantissa = set_mantissa(float_out, (uint64_t)man_out); + if (!mantissa) + // TODO: This change requires the amendment. + // return Unexpected(mantissa.error()); + float_out = (int64_t)mantissa.error(); + else + float_out = mantissa.value(); + auto const exponent = set_exponent(float_out, amt.exponent()); + if (!exponent) + return Unexpected(exponent.error()); + float_out = exponent.value(); + return float_out; +} + +inline Expected +make_float(uint64_t mantissa, int32_t exponent, bool neg) +{ + using enum hook_api::hook_return_code; + if (mantissa == 0) + return 0ULL; + if (mantissa > maxMantissa) + return Unexpected(MANTISSA_OVERSIZED); + if (mantissa < minMantissa) + return Unexpected(MANTISSA_UNDERSIZED); + if (exponent > maxExponent) + return Unexpected(EXPONENT_OVERSIZED); + if (exponent < minExponent) + return Unexpected(EXPONENT_UNDERSIZED); + int64_t out = 0; + + auto const m = set_mantissa(out, mantissa); + if (!m) + return Unexpected(m.error()); // LCOV_EXCL_LINE + out = m.value(); + + auto const e = set_exponent(out, exponent); + if (!e) + return Unexpected(e.error()); // LCOV_EXCL_LINE + out = e.value(); + + out = set_sign(out, neg); + return out; +} + +/** + * This function normalizes the mantissa and exponent passed, if it can. + * It returns the XFL and mutates the supplied manitssa and exponent. + * If a negative mantissa is provided then the returned XFL has the negative + * flag set. If there is an overflow error return XFL_OVERFLOW. On underflow + * returns canonical 0 + */ +template +inline Expected +normalize_xfl(T& man, int32_t& exp, bool neg = false) +{ + if (man == 0) + return 0ULL; + + if (man == std::numeric_limits::min()) + man++; + + constexpr bool sman = std::is_same::value; + static_assert(sman || std::is_same()); + + if constexpr (sman) + { + if (man < 0) + { + man *= -1LL; + neg = true; + } + } + + // mantissa order + std::feclearexcept(FE_ALL_EXCEPT); + int32_t mo = log10(man); + // defensively ensure log10 produces a sane result; we'll borrow the + // overflow error code if it didn't + if (std::fetestexcept(FE_INVALID)) + return Unexpected(XFL_OVERFLOW); // LCOV_EXCL_LINE + + int32_t adjust = 15 - mo; + + if (adjust > 0) + { + // defensive check + if (adjust > 18) + return 0ULL; // LCOV_EXCL_LINE + man *= power_of_ten[adjust]; + exp -= adjust; + } + else if (adjust < 0) + { + // defensive check + if (-adjust > 18) + return Unexpected(XFL_OVERFLOW); // LCOV_EXCL_LINE + man /= power_of_ten[-adjust]; + exp -= adjust; + } + + if (man == 0) + { + exp = 0; + return 0ULL; + } + + // even after adjustment the mantissa can be outside the range by one place + // improving the math above would probably alleviate the need for these + // branches + if (man < minMantissa) + { + if (man == minMantissa - 1LL) + man += 1LL; + else + { + man *= 10LL; + exp--; + } + } + + if (man > maxMantissa) + { + if (man == maxMantissa + 1LL) + man -= 1LL; + else + { + man /= 10LL; + exp++; + } + } + + if (exp < minExponent) + { + man = 0; + exp = 0; + return 0ULL; + } + + if (man == 0) + { + exp = 0; + return 0ULL; + } + + if (exp > maxExponent) + return Unexpected(XFL_OVERFLOW); + + auto const ret = make_float((uint64_t)man, exp, neg); + if constexpr (sman) + { + if (neg) + man *= -1LL; + } + + if (!ret) + return Unexpected(ret.error()); + + return ret; +} + +const int64_t float_one_internal = + make_float(1000000000000000ull, -15, false).value(); + +} // namespace hook_float + +using namespace ripple; + using Bytes = std::vector; struct HookContext; // defined in applyHook.h @@ -170,7 +457,7 @@ public: Expected hook_hash(int32_t hook_no) const; - Expected + Expected hook_again() const; Expected @@ -320,7 +607,7 @@ private: bool modified) const; // these are only used by get_stobject_length below - enum parse_error { + enum class STOParseErrorCode { pe_unexpected_end = -1, pe_unknown_type_early = -2, // detected early pe_unknown_type_late = -3, // end of function @@ -329,8 +616,8 @@ private: }; inline Expected< - int32_t, - parse_error> + uint32_t, + STOParseErrorCode> get_stobject_length( unsigned char* start, // in - begin iterator unsigned char* maxptr, // in - end iterator diff --git a/src/xrpld/app/hook/applyHook.h b/src/xrpld/app/hook/applyHook.h index 2b77e4873..df553d2ea 100644 --- a/src/xrpld/app/hook/applyHook.h +++ b/src/xrpld/app/hook/applyHook.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -174,6 +175,8 @@ struct HookResult false; // hook_again allows strong pre-apply to nominate // additional weak post-apply execution std::shared_ptr provisionalMeta; + std::set> + foreignStateGrantCache; // add found grants here to avoid rechecking }; class HookExecutor; diff --git a/src/xrpld/app/hook/detail/HookAPI.cpp b/src/xrpld/app/hook/detail/HookAPI.cpp index a676230a7..dec51f781 100644 --- a/src/xrpld/app/hook/detail/HookAPI.cpp +++ b/src/xrpld/app/hook/detail/HookAPI.cpp @@ -5,285 +5,8 @@ #include #include #include -#include namespace hook { -namespace hook_float { - -// power of 10 LUT for fast integer math -static int64_t power_of_ten[19] = { - 1LL, - 10LL, - 100LL, - 1000LL, - 10000LL, - 100000LL, - 1000000LL, - 10000000LL, - 100000000LL, - 1000000000LL, - 10000000000LL, - 100000000000LL, - 1000000000000LL, - 10000000000000LL, - 100000000000000LL, - 1000000000000000LL, // 15 - 10000000000000000LL, - 100000000000000000LL, - 1000000000000000000LL, -}; - -using namespace hook_api; -static int64_t const minMantissa = 1000000000000000ull; -static int64_t const maxMantissa = 9999999999999999ull; -static int32_t const minExponent = -96; -static int32_t const maxExponent = 80; - -inline Expected -get_exponent(int64_t float1) -{ - if (float1 < 0) - return Unexpected(INVALID_FLOAT); - if (float1 == 0) - return 0; - uint64_t float_in = (uint64_t)float1; - float_in >>= 54U; - float_in &= 0xFFU; - return ((int32_t)float_in) - 97; -} - -inline Expected -get_mantissa(int64_t float1) -{ - if (float1 < 0) - return Unexpected(INVALID_FLOAT); - if (float1 == 0) - return 0; - float1 -= ((((uint64_t)float1) >> 54U) << 54U); - return float1; -} - -inline bool -is_negative(int64_t float1) -{ - return ((float1 >> 62U) & 1ULL) == 0; -} - -inline int64_t -invert_sign(int64_t float1) -{ - int64_t r = (int64_t)(((uint64_t)float1) ^ (1ULL << 62U)); - return r; -} - -inline int64_t -set_sign(int64_t float1, bool set_negative) -{ - bool neg = is_negative(float1); - if ((neg && set_negative) || (!neg && !set_negative)) - return float1; - - return invert_sign(float1); -} - -inline Expected -set_mantissa(int64_t float1, uint64_t mantissa) -{ - if (mantissa > maxMantissa) - return Unexpected(MANTISSA_OVERSIZED); - if (mantissa < minMantissa) - return Unexpected(MANTISSA_UNDERSIZED); - return float1 - get_mantissa(float1).value() + mantissa; -} - -inline Expected -set_exponent(int64_t float1, int32_t exponent) -{ - if (exponent > maxExponent) - return Unexpected(EXPONENT_OVERSIZED); - if (exponent < minExponent) - return Unexpected(EXPONENT_UNDERSIZED); - - uint64_t exp = (exponent + 97); - exp <<= 54U; - float1 &= ~(0xFFLL << 54); - float1 += (int64_t)exp; - return float1; -} - -inline Expected -make_float(ripple::IOUAmount& amt) -{ - int64_t man_out = amt.mantissa(); - int64_t float_out = 0; - bool neg = man_out < 0; - if (neg) - man_out *= -1; - - float_out = set_sign(float_out, neg); - auto const mantissa = set_mantissa(float_out, (uint64_t)man_out); - if (!mantissa) - // TODO: This change requires the amendment. - // return Unexpected(mantissa.error()); - float_out = mantissa.error(); - else - float_out = mantissa.value(); - auto const exponent = set_exponent(float_out, amt.exponent()); - if (!exponent) - return Unexpected(exponent.error()); - float_out = exponent.value(); - return float_out; -} - -inline Expected -make_float(uint64_t mantissa, int32_t exponent, bool neg) -{ - if (mantissa == 0) - return 0; - if (mantissa > maxMantissa) - return Unexpected(MANTISSA_OVERSIZED); - if (mantissa < minMantissa) - return Unexpected(MANTISSA_UNDERSIZED); - if (exponent > maxExponent) - return Unexpected(EXPONENT_OVERSIZED); - if (exponent < minExponent) - return Unexpected(EXPONENT_UNDERSIZED); - int64_t out = 0; - - auto const m = set_mantissa(out, mantissa); - if (!m) - return m.error(); - out = m.value(); - - auto const e = set_exponent(out, exponent); - if (!e) - return e.error(); - out = e.value(); - - out = set_sign(out, neg); - return out; -} - -/** - * This function normalizes the mantissa and exponent passed, if it can. - * It returns the XFL and mutates the supplied manitssa and exponent. - * If a negative mantissa is provided then the returned XFL has the negative - * flag set. If there is an overflow error return XFL_OVERFLOW. On underflow - * returns canonical 0 - */ -template -inline Expected -normalize_xfl(T& man, int32_t& exp, bool neg = false) -{ - if (man == 0) - return 0; - - if (man == std::numeric_limits::min()) - man++; - - constexpr bool sman = std::is_same::value; - static_assert(sman || std::is_same()); - - if constexpr (sman) - { - if (man < 0) - { - man *= -1LL; - neg = true; - } - } - - // mantissa order - std::feclearexcept(FE_ALL_EXCEPT); - int32_t mo = log10(man); - // defensively ensure log10 produces a sane result; we'll borrow the - // overflow error code if it didn't - if (std::fetestexcept(FE_INVALID)) - return Unexpected(XFL_OVERFLOW); - - int32_t adjust = 15 - mo; - - if (adjust > 0) - { - // defensive check - if (adjust > 18) - return 0; - man *= power_of_ten[adjust]; - exp -= adjust; - } - else if (adjust < 0) - { - // defensive check - if (-adjust > 18) - return Unexpected(XFL_OVERFLOW); - man /= power_of_ten[-adjust]; - exp -= adjust; - } - - if (man == 0) - { - exp = 0; - return 0; - } - - // even after adjustment the mantissa can be outside the range by one place - // improving the math above would probably alleviate the need for these - // branches - if (man < minMantissa) - { - if (man == minMantissa - 1LL) - man += 1LL; - else - { - man *= 10LL; - exp--; - } - } - - if (man > maxMantissa) - { - if (man == maxMantissa + 1LL) - man -= 1LL; - else - { - man /= 10LL; - exp++; - } - } - - if (exp < minExponent) - { - man = 0; - exp = 0; - return 0; - } - - if (man == 0) - { - exp = 0; - return 0; - } - - if (exp > maxExponent) - return Unexpected(XFL_OVERFLOW); - - auto const ret = make_float((uint64_t)man, exp, neg); - if constexpr (sman) - { - if (neg) - man *= -1LL; - } - - if (!ret) - return ret.error(); - - return ret; -} - -const int64_t float_one_internal = - make_float(1000000000000000ull, -15, false).value(); - -} // namespace hook_float using namespace ripple; using namespace hook_float; @@ -1178,7 +901,7 @@ HookAPI::etxn_details(uint8_t* out_ptr) const auto hash = etxn_nonce(); if (!hash.has_value()) - return INTERNAL_ERROR; + return Unexpected(INTERNAL_ERROR); memcpy(out, hash->data(), 32); @@ -1273,7 +996,7 @@ HookAPI::float_set(int32_t exponent, int64_t mantissa) const { if (normalized.error() == XFL_OVERFLOW) return Unexpected(INVALID_FLOAT); - return normalized.error(); + return Unexpected(normalized.error()); } if (normalized.value() == 0) return Unexpected(INVALID_FLOAT); @@ -1321,7 +1044,7 @@ HookAPI::float_mulratio( auto const result = make_float((uint64_t)man1, exp1, is_negative(float1)); if (!result) - return result.error(); + return Unexpected(result.error()); return result; } @@ -1932,7 +1655,7 @@ HookAPI::hook_hash(int32_t hook_no) const return hook.getFieldH256(sfHookHash); } -Expected +Expected HookAPI::hook_again() const { if (hookCtx.result.executeAgainAsWeak) @@ -1941,7 +1664,7 @@ HookAPI::hook_again() const if (hookCtx.result.isStrong) { hookCtx.result.executeAgainAsWeak = true; - return 1; + return 1ULL; } return Unexpected(PREREQUISITE_NOT_MET); @@ -2197,88 +1920,114 @@ HookAPI::state_foreign_set( if (hookCtx.result.foreignStateSetDisabled) return Unexpected(PREVIOUS_FAILURE_PREVENTS_RETRY); - // first check if we've already modified this state - auto cacheEntry = lookup_state_cache(account, ns, key); - if (cacheEntry && cacheEntry->get().first) - { - // if a cache entry already exists and it has already been modified - // don't check grants again - if (auto ret = set_state_cache(account, ns, key, data, true); - !ret.has_value()) - return Unexpected(ret.error()); + bool const hasFix = hookCtx.applyCtx.view().rules().enabled(fixHookMap); - return data.size(); + if (!hasFix) + { + // first check if we've already modified this state + auto cacheEntry = lookup_state_cache(account, ns, key); + if (cacheEntry && cacheEntry->get().first) + { + // if a cache entry already exists and it has already been modified + // don't check grants again + if (auto ret = set_state_cache(account, ns, key, data, true); + !ret.has_value()) + return Unexpected(ret.error()); + + return data.size(); + } } - - // cache miss or cache was present but entry was not marked as previously - // modified therefore before continuing we need to check grants - auto const sle = - hookCtx.applyCtx.view().read(ripple::keylet::hook(account)); - if (!sle) - return Unexpected(INTERNAL_ERROR); - - bool found_auth = false; - - // we do this by iterating the hooks installed on the foreign account and in - // turn their grants and namespaces - auto const& hooks = sle->getFieldArray(sfHooks); - for (auto const& hookObj : hooks) + // check if we've used a grant to modify this state entry before, if not + // look up possible grants + if (!hasFix || + hookCtx.result.foreignStateGrantCache.find({account, ns}) == + hookCtx.result.foreignStateGrantCache.end()) { - // skip blank entries - if (!hookObj.isFieldPresent(sfHookHash)) - continue; + auto const sle = + hookCtx.applyCtx.view().read(ripple::keylet::hook(account)); - if (!hookObj.isFieldPresent(sfHookGrants)) - continue; - - auto const& hookGrants = hookObj.getFieldArray(sfHookGrants); - - if (hookGrants.size() < 1) - continue; - - // the grant allows the hook to modify the granter's namespace only - if (hookObj.isFieldPresent(sfHookNamespace)) + if (!sle) { - if (hookObj.getFieldH256(sfHookNamespace) != ns) - continue; - } - else - { - // fetch the hook definition - auto const def = - hookCtx.applyCtx.view().read(ripple::keylet::hookDefinition( - hookObj.getFieldH256(sfHookHash))); - if (!def) // should never happen except in a rare race condition - continue; - if (def->getFieldH256(sfHookNamespace) != ns) - continue; - } - - // this is expensive search so we'll disallow after one failed attempt - for (auto const& hookGrantObj : hookGrants) - { - bool hasAuthorizedField = hookGrantObj.isFieldPresent(sfAuthorize); - - if (hookGrantObj.getFieldH256(sfHookHash) == - hookCtx.result.hookHash && - (!hasAuthorizedField || - hookGrantObj.getAccountID(sfAuthorize) == - hookCtx.result.account)) + if (hasFix) { - found_auth = true; - break; + hookCtx.result.foreignStateSetDisabled = true; + return Unexpected(NOT_AUTHORIZED); } + + return Unexpected(INTERNAL_ERROR); } - if (found_auth) - break; - } + // RH TODO: test this code path more completely - if (!found_auth) - { - // hook only gets one attempt - hookCtx.result.foreignStateSetDisabled = true; - return Unexpected(NOT_AUTHORIZED); + bool found_auth = false; + + // we do this by iterating the hooks installed on the foreign account + // and in turn their grants and namespaces + auto const& hooks = sle->getFieldArray(sfHooks); + for (auto const& hookObj : hooks) + { + // skip blank entries + if (!hookObj.isFieldPresent(sfHookHash)) + continue; + + if (!hookObj.isFieldPresent(sfHookGrants)) + continue; + + auto const& hookGrants = hookObj.getFieldArray(sfHookGrants); + + if (hookGrants.size() < 1) + continue; + + // the grant allows the hook to modify the granter's namespace only + if (hookObj.isFieldPresent(sfHookNamespace)) + { + if (hookObj.getFieldH256(sfHookNamespace) != ns) + continue; + } + else + { + // fetch the hook definition + auto const def = + hookCtx.applyCtx.view().read(ripple::keylet::hookDefinition( + hookObj.getFieldH256(sfHookHash))); + if (!def) // should never happen except in a rare race + // condition + continue; + if (def->getFieldH256(sfHookNamespace) != ns) + continue; + } + + // this is expensive search so we'll disallow after one failed + // attempt + for (auto const& hookGrantObj : hookGrants) + { + bool hasAuthorizedField = + hookGrantObj.isFieldPresent(sfAuthorize); + + if (hookGrantObj.getFieldH256(sfHookHash) == + hookCtx.result.hookHash && + (!hasAuthorizedField || + hookGrantObj.getAccountID(sfAuthorize) == + hookCtx.result.account)) + { + found_auth = true; + break; + } + } + + if (found_auth) + break; + } + + if (!found_auth) + { + // hook only gets one attempt + hookCtx.result.foreignStateSetDisabled = true; + return Unexpected(NOT_AUTHORIZED); + } + + // add the grant to the cache + hookCtx.result.foreignStateGrantCache.emplace(account, ns); } if (auto ret = set_state_cache(account, ns, key, data, true); @@ -2607,7 +2356,7 @@ HookAPI::slot_float(uint32_t slot_no) const normalized = ret.value(); } - if (normalized == EXPONENT_UNDERSIZED) + if (normalized == (int64_t)EXPONENT_UNDERSIZED) /* exponent undersized (underflow) */ return 0; // return 0 in this case return normalized; @@ -3134,8 +2883,8 @@ HookAPI::set_state_cache( // including header bytes (and footer bytes in the event of array or object) // negative indicates error inline Expected< - int32_t, - HookAPI::parse_error> + uint32_t, + HookAPI::STOParseErrorCode> HookAPI::get_stobject_length( unsigned char* start, // in - begin iterator unsigned char* maxptr, // in - end iterator @@ -3148,6 +2897,7 @@ HookAPI::get_stobject_length( int recursion_depth) // used internally const { + using enum HookAPI::STOParseErrorCode; if (recursion_depth > 10) return Unexpected(pe_excessive_nesting); @@ -3156,7 +2906,7 @@ HookAPI::get_stobject_length( : STI_VECTOR256; if (type > max_sti_type) - return pe_unknown_type_early; + return Unexpected(pe_unknown_type_early); unsigned char* end = maxptr; unsigned char* upto = start; @@ -3216,7 +2966,7 @@ HookAPI::get_stobject_length( // not supported types if (type == STI_NUMBER || type == STI_UINT96 || type == STI_UINT192 || type == STI_UINT384 || type == STI_UINT512) - return pe_unknown_type_early; + return Unexpected(pe_unknown_type_early); bool is_vl = (type == STI_ACCOUNT || type == STI_VL || @@ -3286,7 +3036,7 @@ HookAPI::get_stobject_length( length = 20; break; default: - return -1; + return Unexpected(pe_unknown_type_late); } } else if (type == STI_AMOUNT) /* AMOUNT */ @@ -3308,7 +3058,7 @@ HookAPI::get_stobject_length( int flag = *(upto + length++); // flag shoud be 0x01 or 0x10 or 0x20 or those union if (flag == 0 || flag & ~(0x01 | 0x10 | 0x20)) - return pe_unexpected_end; + return Unexpected(pe_unexpected_end); if (flag & 0x01) // account length += 20; if (flag & 0x10) // currency @@ -3329,10 +3079,10 @@ HookAPI::get_stobject_length( else if (lastflag == 0x00) break; // end byte else - return pe_unexpected_end; + return Unexpected(pe_unexpected_end); } if (upto >= end) - return pe_unexpected_end; + return Unexpected(pe_unexpected_end); } else if (type == STI_ISSUE) { diff --git a/src/xrpld/app/hook/detail/applyHook.cpp b/src/xrpld/app/hook/detail/applyHook.cpp index 42b91202a..14855b73c 100644 --- a/src/xrpld/app/hook/detail/applyHook.cpp +++ b/src/xrpld/app/hook/detail/applyHook.cpp @@ -14,8 +14,6 @@ #include #include #include -#include -#include #include #include #include @@ -598,160 +596,9 @@ getTransactionalStakeHolders(STTx const& tx, ReadView const& rv) } // namespace hook -namespace hook_float { - -using namespace hook_api; -static int64_t const minMantissa = 1000000000000000ull; -static int64_t const maxMantissa = 9999999999999999ull; -static int32_t const minExponent = -96; -static int32_t const maxExponent = 80; -inline int32_t -get_exponent(int64_t float1) -{ - if (float1 < 0) - return INVALID_FLOAT; - if (float1 == 0) - return 0; - uint64_t float_in = (uint64_t)float1; - float_in >>= 54U; - float_in &= 0xFFU; - return ((int32_t)float_in) - 97; -} - -inline int64_t -get_mantissa(int64_t float1) -{ - if (float1 < 0) - return INVALID_FLOAT; - if (float1 == 0) - return 0; - float1 -= ((((uint64_t)float1) >> 54U) << 54U); - return float1; -} - -inline bool -is_negative(int64_t float1) -{ - return ((float1 >> 62U) & 1ULL) == 0; -} - -inline int64_t -invert_sign(int64_t float1) -{ - int64_t r = (int64_t)(((uint64_t)float1) ^ (1ULL << 62U)); - return r; -} - -inline int64_t -set_sign(int64_t float1, bool set_negative) -{ - bool neg = is_negative(float1); - if ((neg && set_negative) || (!neg && !set_negative)) - return float1; - - return invert_sign(float1); -} - -inline int64_t -set_mantissa(int64_t float1, uint64_t mantissa) -{ - if (mantissa > maxMantissa) - return MANTISSA_OVERSIZED; - if (mantissa < minMantissa) - return MANTISSA_UNDERSIZED; - return float1 - get_mantissa(float1) + mantissa; -} - -inline int64_t -set_exponent(int64_t float1, int32_t exponent) -{ - if (exponent > maxExponent) - return EXPONENT_OVERSIZED; - if (exponent < minExponent) - return EXPONENT_UNDERSIZED; - - uint64_t exp = (exponent + 97); - exp <<= 54U; - float1 &= ~(0xFFLL << 54); - float1 += (int64_t)exp; - return float1; -} - -inline int64_t -make_float(ripple::IOUAmount& amt) -{ - int64_t man_out = amt.mantissa(); - int64_t float_out = 0; - bool neg = man_out < 0; - if (neg) - man_out *= -1; - - float_out = set_sign(float_out, neg); - float_out = set_mantissa(float_out, (uint64_t)man_out); - float_out = set_exponent(float_out, amt.exponent()); - return float_out; -} - -inline int64_t -make_float(uint64_t mantissa, int32_t exponent, bool neg) -{ - if (mantissa == 0) - return 0; - if (mantissa > maxMantissa) - return MANTISSA_OVERSIZED; - if (mantissa < minMantissa) - return MANTISSA_UNDERSIZED; - if (exponent > maxExponent) - return EXPONENT_OVERSIZED; - if (exponent < minExponent) - return EXPONENT_UNDERSIZED; - int64_t out = 0; - out = set_mantissa(out, mantissa); - out = set_exponent(out, exponent); - out = set_sign(out, neg); - return out; -} - -} // namespace hook_float -using namespace hook_float; +using namespace hook::hook_float; using hook::Bytes; -inline int32_t -no_free_slots(hook::HookContext& hookCtx) -{ - return hook_api::max_slots - hookCtx.slot.size() <= 0; -} - -inline std::optional -get_free_slot(hook::HookContext& hookCtx) -{ - // allocate a slot - int32_t slot_into = 0; - if (hookCtx.slot_free.size() > 0) - { - slot_into = hookCtx.slot_free.front(); - hookCtx.slot_free.pop(); - return slot_into; - } - - // no slots were available in the queue so increment slot counter until we - // find a free slot usually this will be the next available but the hook - // developer may have allocated any slot ahead of when the counter gets - // there - do - { - slot_into = ++hookCtx.slot_counter; - } while (hookCtx.slot.find(slot_into) != hookCtx.slot.end() && - // this condition should always be met, if for some reason, somehow - // it is not then we will return the final slot every time. - hookCtx.slot_counter <= hook_api::max_slots); - - if (hookCtx.slot_counter > hook_api::max_slots) - return {}; - - return slot_into; -} - // cu_ptr is a pointer into memory, bounds check is assumed to have already // happened inline std::optional @@ -807,7 +654,7 @@ parseCurrency(uint8_t* cu_ptr, uint32_t cu_len) return {}; } -inline int64_t +inline std::variant serialize_keylet( ripple::Keylet& kl, uint8_t* memory, @@ -815,7 +662,7 @@ serialize_keylet( uint32_t write_len) { if (write_len < 34) - return hook_api::TOO_SMALL; + return TOO_SMALL; memory[write_ptr + 0] = (kl.type >> 8) & 0xFFU; memory[write_ptr + 1] = (kl.type >> 0) & 0xFFU; @@ -823,7 +670,7 @@ serialize_keylet( for (int i = 0; i < 32; ++i) memory[write_ptr + 2 + i] = kl.key.data()[i]; - return 34; + return 34ULL; } std::optional @@ -866,19 +713,19 @@ hook::computeCreationFee(uint64_t byteCount) } // many datatypes can be encoded into an int64_t -inline int64_t +inline std::variant data_as_int64(void const* ptr_raw, uint32_t len) { if (len > 8) - return hook_api::hook_return_code::TOO_BIG; + return TOO_BIG; uint8_t const* ptr = reinterpret_cast(ptr_raw); uint64_t output = 0; for (int i = 0, j = (len - 1) * 8; i < len; ++i, j -= 8) output += (((uint64_t)ptr[i]) << j); if ((1ULL << 63U) & output) - return hook_api::hook_return_code::TOO_BIG; - return (int64_t)output; + return TOO_BIG; + return output; } /* returns true iff every even char is ascii and every odd char is 00 @@ -1265,7 +1112,7 @@ DEFINE_HOOK_FUNCTION( return OUT_OF_BOUNDS; if (!j.trace()) - return 0; + return 0ULL; if (read_len > 128) read_len = 128; @@ -1283,12 +1130,12 @@ DEFINE_HOOK_FUNCTION( (const char*)memory + read_ptr, read_len) << ": " << number; - return 0; + return 0ULL; } } j.trace() << "HookTrace[" << HC_ACC() << "]: " << number; - return 0; + return 0ULL; HOOK_TEARDOWN(); } @@ -1308,7 +1155,7 @@ DEFINE_HOOK_FUNCTION( return OUT_OF_BOUNDS; if (!j.trace()) - return 0; + return 0ULL; if (mread_len > 128) mread_len = 128; @@ -1371,7 +1218,7 @@ DEFINE_HOOK_FUNCTION( << std::string_view((const char*)output_storage, out_len); } - return 0; + return 0ULL; HOOK_TEARDOWN(); } @@ -1476,7 +1323,8 @@ DEFINE_HOOK_FUNCTION( auto const sleAccount = view.peek(hookCtx.result.accountKeylet); if (!sleAccount && view.rules().enabled(featureExtendedHookState)) - return tefINTERNAL; + // should return hook_api::hook_return_code + return static_cast(tefINTERNAL); uint16_t const hookStateScale = sleAccount->isFieldPresent(sfHookStateScale) ? sleAccount->getFieldU16(sfHookStateScale) @@ -1500,7 +1348,8 @@ DEFINE_HOOK_FUNCTION( { auto const sleAccount = view.peek(hookCtx.result.accountKeylet); if (!sleAccount) - return tefINTERNAL; + // should return hook_api::hook_return_code + return static_cast(tefINTERNAL); } if (!key) @@ -1715,7 +1564,8 @@ hook::finalizeHookResult( // add a metadata entry for this hook execution result { STObject meta{sfHookExecution}; - meta.setFieldU8(sfHookResult, hookResult.exitType); + meta.setFieldU8( + sfHookResult, static_cast(hookResult.exitType)); meta.setAccountID(sfHookAccount, hookResult.account); // RH NOTE: this is probably not necessary, a direct cast should always @@ -2225,7 +2075,7 @@ DEFINE_HOOK_FUNCTION(int64_t, slot_type, uint32_t slot_no, uint32_t flags) if (flags == 0) { auto const base = std::get<0>(*result); - return base.getFName().fieldCode; + return static_cast(base.getFName().fieldCode); } else { @@ -2836,7 +2686,8 @@ DEFINE_HOOK_FUNCTION( if (NOT_IN_BOUNDS(write_ptr, txID.size(), memory_length)) return OUT_OF_BOUNDS; - auto const write_txid = [&]() -> int64_t { + auto const write_txid = + [&]() -> std::variant { WRITE_WASM_MEMORY_AND_RETURN( write_ptr, txID.size(), @@ -2846,12 +2697,15 @@ DEFINE_HOOK_FUNCTION( memory_length); }; - int64_t result = write_txid(); + auto result = write_txid(); + if (std::holds_alternative(result)) + return std::get(result); - if (result == 32) + auto const value = std::get(result); + if (value == 32) hookCtx.result.emittedTxn.push(tpTrans); - return result; + return value; HOOK_TEARDOWN(); } @@ -3340,7 +3194,7 @@ DEFINE_HOOK_FUNCTION( uint32_t field_id) { // proxy only no setup or teardown - int64_t ret = sto_emplace( + auto ret = sto_emplace( hookCtx, frameCtx, write_ptr, @@ -3351,8 +3205,12 @@ DEFINE_HOOK_FUNCTION( 0, field_id); - if (ret > 0 && ret == read_len) - return DOESNT_EXIST; + if (std::holds_alternative(ret)) + { + auto const value = std::get(ret); + if (value > 0 && value == read_len) + return DOESNT_EXIST; + } return ret; } @@ -3375,7 +3233,7 @@ DEFINE_HOOK_FUNCTION( auto const result = api.sto_validate(data); if (!result) return result.error(); - return result.value() ? 1 : 0; + return result.value() ? 1ULL : 0ULL; HOOK_TEARDOWN(); } @@ -3411,7 +3269,7 @@ DEFINE_HOOK_FUNCTION( auto const result = api.util_verify(data, sig, key); if (!result) return result.error(); - return result.value() ? 1 : 0; + return result.value() ? 1ULL : 0ULL; HOOK_TEARDOWN(); } @@ -3506,26 +3364,30 @@ DEFINE_HOOK_FUNCTION(int32_t, _g, uint32_t id, uint32_t maxitr) << "Iterations: " << hookCtx.guard_map[id]; } hookCtx.result.exitType = hook_api::ExitType::ROLLBACK; - hookCtx.result.exitCode = GUARD_VIOLATION; + hookCtx.result.exitCode = (int64_t)GUARD_VIOLATION; return RC_ROLLBACK; } - return 1; + return 1U; HOOK_TEARDOWN(); } -#define RETURN_IF_INVALID_FLOAT(float1) \ - { \ - if (float1 < 0) \ - return hook_api::INVALID_FLOAT; \ - if (float1 != 0) \ - { \ - uint64_t mantissa = get_mantissa(float1); \ - int32_t exponent = get_exponent(float1); \ - if (mantissa < minMantissa || mantissa > maxMantissa || \ - exponent > maxExponent || exponent < minExponent) \ - return INVALID_FLOAT; \ - } \ +#define RETURN_IF_INVALID_FLOAT(float1) \ + { \ + if (float1 < 0) \ + return INVALID_FLOAT; \ + if (float1 != 0) \ + { \ + auto const mantissa = get_mantissa(float1); \ + auto const exponent = get_exponent(float1); \ + if (!mantissa || !exponent) \ + return INVALID_FLOAT; \ + if (mantissa.value() < minMantissa || \ + mantissa.value() > maxMantissa || \ + exponent.value() > maxExponent || \ + exponent.value() < minExponent) \ + return INVALID_FLOAT; \ + } \ } DEFINE_HOOK_FUNCTION( @@ -3542,7 +3404,7 @@ DEFINE_HOOK_FUNCTION( return OUT_OF_BOUNDS; if (!j.trace()) - return 0; + return 0ULL; if (read_len > 128) read_len = 128; @@ -3552,38 +3414,33 @@ DEFINE_HOOK_FUNCTION( *((const char*)memory + read_ptr + read_len - 1) == '\0') read_len--; + auto const messageKey = (read_len == 0) + ? "" + : std::string_view((const char*)memory + read_ptr, read_len); + if (float1 == 0) { - j.trace() << "HookTrace[" << HC_ACC() << "]: " - << (read_len == 0 - ? "" - : std::string_view( - (const char*)memory + read_ptr, read_len)) + j.trace() << "HookTrace[" << HC_ACC() << "]: " << messageKey << ": Float 0*10^(0) "; - return 0; + return 0ULL; } - uint64_t man = get_mantissa(float1); - int32_t exp = get_exponent(float1); + auto const man = get_mantissa(float1); + auto const exp = get_exponent(float1); bool neg = is_negative(float1); - if (man < minMantissa || man > maxMantissa || exp < minExponent || - exp > maxExponent) + if (!man || !exp || man.value() < minMantissa || + man.value() > maxMantissa || exp.value() < minExponent || + exp.value() > maxExponent) { - j.trace() << "HookTrace[" << HC_ACC() << "]:" - << (read_len == 0 - ? "" - : std::string_view( - (const char*)memory + read_ptr, read_len)) + j.trace() << "HookTrace[" << HC_ACC() << "]: " << messageKey << ": Float "; - return 0; + return 0ULL; } - j.trace() << "HookTrace[" << HC_ACC() << "]:" - << (read_len == 0 ? "" - : std::string_view( - (const char*)memory + read_ptr, read_len)) - << ": Float " << (neg ? "-" : "") << man << "*10^(" << exp << ")"; - return 0; + j.trace() << "HookTrace[" << HC_ACC() << "]:" << messageKey << ": Float " + << (neg ? "-" : "") << man.value() << "*10^(" << exp.value() + << ")"; + return 0ULL; HOOK_TEARDOWN(); } diff --git a/src/xrpld/app/ledger/Ledger.cpp b/src/xrpld/app/ledger/Ledger.cpp index 10d1c4436..145298dac 100644 --- a/src/xrpld/app/ledger/Ledger.cpp +++ b/src/xrpld/app/ledger/Ledger.cpp @@ -1044,7 +1044,8 @@ pendSaveValidated( Application& app, std::shared_ptr const& ledger, bool isSynchronous, - bool isCurrent) + bool isCurrent, + std::function callback) { if (!app.getHashRouter().setFlags(ledger->info().hash, SF_SAVED)) { @@ -1056,6 +1057,8 @@ pendSaveValidated( { // Either we don't need it to be finished // or it is finished + if (callback) + callback(true); return true; } } @@ -1069,6 +1072,8 @@ pendSaveValidated( JLOG(stream) << "Pend save with seq in pending saves " << ledger->info().seq; + if (callback) + callback(true); return true; } @@ -1077,15 +1082,20 @@ pendSaveValidated( app.getJobQueue().addJob( isCurrent ? jtPUBLEDGER : jtPUBOLDLEDGER, std::to_string(ledger->seq()), - [&app, ledger, isCurrent]() { - saveValidatedLedger(app, ledger, isCurrent); + [&app, ledger, isCurrent, callback]() { + bool success = saveValidatedLedger(app, ledger, isCurrent); + if (callback) + callback(success); })) { return true; } // The JobQueue won't do the Job. Do the save synchronously. - return saveValidatedLedger(app, ledger, isCurrent); + bool success = saveValidatedLedger(app, ledger, isCurrent); + if (callback) + callback(success); + return success; } void diff --git a/src/xrpld/app/ledger/Ledger.h b/src/xrpld/app/ledger/Ledger.h index 0476859c8..3c7c2c4e6 100644 --- a/src/xrpld/app/ledger/Ledger.h +++ b/src/xrpld/app/ledger/Ledger.h @@ -452,7 +452,8 @@ pendSaveValidated( Application& app, std::shared_ptr const& ledger, bool isSynchronous, - bool isCurrent); + bool isCurrent, + std::function callback = {}); std::shared_ptr loadLedgerHelper(LedgerInfo const& sinfo, Application& app, bool acquire); diff --git a/src/xrpld/app/ledger/LedgerMaster.h b/src/xrpld/app/ledger/LedgerMaster.h index e59b1b78f..c4ac1e510 100644 --- a/src/xrpld/app/ledger/LedgerMaster.h +++ b/src/xrpld/app/ledger/LedgerMaster.h @@ -139,6 +139,9 @@ public: RangeSet getPinnedLedgersRangeSet(); + void + setPinnedLedgersRangeSet(const RangeSet& range_set); + /** Apply held transactions to the open ledger This is normally called as we close the ledger. The open ledger remains open to handle new transactions @@ -205,6 +208,10 @@ public: void clearLedger(std::uint32_t seq); bool + isPinned(std::uint32_t seq); + void + unpinLedger(std::uint32_t seq); + bool isValidated(ReadView const& ledger); bool getValidatedRange(std::uint32_t& minVal, std::uint32_t& maxVal); @@ -359,7 +366,12 @@ private: std::recursive_mutex mCompleteLock; RangeSet mCompleteLedgers; + + // When both locks are needed, acquire them together with std::scoped_lock. + std::mutex mPinnedLock; RangeSet mPinnedLedgers; // Track pinned ledger ranges + bool mPinnedMergedToComplete{ + false}; // One-shot: pinned merged into mCompleteLedgers // Publish thread is running. bool mAdvanceThread{false}; diff --git a/src/xrpld/app/ledger/detail/LedgerMaster.cpp b/src/xrpld/app/ledger/detail/LedgerMaster.cpp index 506dfec33..126a4c996 100644 --- a/src/xrpld/app/ledger/detail/LedgerMaster.cpp +++ b/src/xrpld/app/ledger/detail/LedgerMaster.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -446,14 +447,25 @@ bool LedgerMaster::storeLedger(std::shared_ptr ledger, bool pin) { bool validated = ledger->info().validated; - // Returns true if we already had the ledger - if (!mLedgerHistory.insert(std::move(ledger), validated)) + // Returns true if we already had the ledger. + // + // When pin=true, we deliberately skip mLedgerHistory insertion. + // A single catalogue pack can contain tens of thousands of ledgers, + // each holding a full SHAMap of account/transaction nodes. Keeping + // them all in the in-memory history cache would consume hundreds of + // GB of RAM. Instead, pinned ledgers are persisted to the nodestore + // and SQLite, and are retrievable via RPC (which reads from SQLite), + // but are NOT kept in memory. + if (!pin && !mLedgerHistory.insert(std::move(ledger), validated)) return false; if (pin) { uint32_t seq = ledger->info().seq; - mPinnedLedgers.insert(range(seq, seq)); + { + std::lock_guard sl(mPinnedLock); + mPinnedLedgers.insert(range(seq, seq)); + } JLOG(m_journal.info()) << "Pinned ledger : " << seq; } return true; @@ -512,7 +524,7 @@ LedgerMaster::haveLedger(std::uint32_t seq) void LedgerMaster::clearLedger(std::uint32_t seq) { - std::lock_guard sl(mCompleteLock); + std::scoped_lock lock(mCompleteLock, mPinnedLock); // Don't clear pinned ledgers if (boost::icl::contains(mPinnedLedgers, seq)) @@ -525,6 +537,20 @@ LedgerMaster::clearLedger(std::uint32_t seq) mCompleteLedgers.erase(seq); } +bool +LedgerMaster::isPinned(std::uint32_t seq) +{ + std::lock_guard sl(mPinnedLock); + return boost::icl::contains(mPinnedLedgers, seq); +} + +void +LedgerMaster::unpinLedger(std::uint32_t seq) +{ + std::lock_guard sl(mPinnedLock); + mPinnedLedgers.erase(range(seq, seq)); +} + bool LedgerMaster::isValidated(ReadView const& ledger) { @@ -861,7 +887,19 @@ LedgerMaster::setFullLedger( pendSaveValidated(app_, ledger, isSynchronous, isCurrent); { - std::lock_guard ml(mCompleteLock); + std::scoped_lock lock(mCompleteLock, mPinnedLock); + // One-time merge of pinned ranges into mCompleteLedgers. + // For NORMAL/NETWORK startup, this fires on the first validated + // ledger (network quorum). For LOAD/standalone, the merge + // already happened in setPinnedLedgersRangeSet() and this + // is a no-op (mPinnedMergedToComplete is already true). + if (!mPinnedMergedToComplete && !mPinnedLedgers.empty()) + { + for (auto const& interval : mPinnedLedgers) + mCompleteLedgers.insert(interval); + mPinnedMergedToComplete = true; + } + mCompleteLedgers.insert(ledger->info().seq); } @@ -1286,53 +1324,134 @@ LedgerMaster::findNewLedgersToPublish( auto valLedger = mValidLedger.get(); std::uint32_t valSeq = valLedger->info().seq; + // Create a range of ledgers we need to publish + RangeSet toPublish; + toPublish.insert(range(pubSeq, valSeq)); + + // CRITICAL: Skip publishing pinned ledgers (except the most recent). + // + // When loading catalogues, we deliberately avoid storing pinned ledgers in + // the AcceptedLedger cache to prevent memory bloat with millions of + // ledgers. However, NetworkOPs::pubLedger() expects every published ledger + // to either: + // 1. Already exist in the AcceptedLedger cache, OR + // 2. Be able to create and cache a new AcceptedLedger + // + // For pinned ledgers, we skip the cache entirely during + // saveValidatedLedger. If we try to publish them, pubLedger creates a new + // AcceptedLedger, but canonicalize_replace_client might return a different + // instance, causing: "Assertion failed: alpAccepted->getLedger().get() == + // lpAccepted.get()" + // + // This primarily affects STANDALONE MODE where catalogue_load calls + // switchLCL when loading historical ledgers newer than the current closed + // ledger. In live network mode, historical ledgers are typically older than + // the current ledger, so switchLCL -> tryAdvance -> publish isn't + // triggered. + // + // Solution: Only publish the most recent validated ledger (which needs + // publishing) and skip all intermediate pinned ledgers (already on disk). + { + std::lock_guard sll(mPinnedLock); + RangeSet pinnedExceptLast = mPinnedLedgers; + // Remove the most recent from the pinned set so we always publish it + if (boost::icl::contains(pinnedExceptLast, valSeq)) + pinnedExceptLast.erase(range(valSeq, valSeq)); + // Subtract pinned ledgers from the set to publish + toPublish -= pinnedExceptLast; + } + scope_unlock sul{sl}; try { - for (std::uint32_t seq = pubSeq; seq <= valSeq; ++seq) + for (auto const& interval : toPublish) { - JLOG(m_journal.trace()) - << "Trying to fetch/publish valid ledger " << seq; + // Some sequences may be absent from toPublish because they are + // pinned ledgers already persisted to disk and intentionally not + // republished. The decision logic — "can pubSeq safely jump + // across this gap?" — lives in detail::canSkipPinnedGap so it + // is unit-testable without spinning up a full LedgerMaster. + // + // Core invariant (enforced by canSkipPinnedGap): never publish + // across a gap in non-pinned ledgers. Publication must remain + // contiguous for every non-pinned sequence from + // mPubLedgerSeq + 1 forward. If pubSeq is lagging because an + // earlier non-pinned ledger could not be fetched/published, we + // stop here rather than publish newer ledgers out of order. + // + // In production this branch is essentially dead code: pinned + // ranges are old historical catalogue data and pubSeq tracks + // the recent published tip, so toPublish won't have pinned- + // induced gaps. It only matters in test/standalone catalogue- + // load scenarios where pinned ranges can sit near pubSeq. + if (pubSeq < interval.first()) + { + bool canSkip; + { + std::lock_guard sll(mPinnedLock); + canSkip = detail::canSkipPinnedGap( + pubSeq, interval.first(), mPinnedLedgers); + } - std::shared_ptr ledger; - // This can throw - auto hash = hashOfSeq(*valLedger, seq, m_journal); - // VFALCO TODO Restructure this code so that zero is not - // used. - if (!hash) - hash = beast::zero; // kludge - if (seq == valSeq) - { - // We need to publish the ledger we just fully validated - ledger = valLedger; - } - else if (hash->isZero()) - { - JLOG(m_journal.fatal()) << "Ledger: " << valSeq - << " does not have hash for " << seq; - UNREACHABLE( - "ripple::LedgerMaster::findNewLedgersToPublish : ledger " - "not found"); - } - else - { - ledger = mLedgerHistory.getLedgerByHash(*hash); + if (!canSkip) + { + JLOG(m_journal.trace()) + << "Stopping publish at seq " << pubSeq + << " — skipped span [" << pubSeq << ", " + << interval.first() << ") contains non-pinned ledgers"; + break; + } + + pubSeq = interval.first(); } - if (!app_.config().LEDGER_REPLAY) + for (std::uint32_t seq = interval.first(); seq <= interval.last(); + ++seq) { - // Can we try to acquire the ledger we need? - if (!ledger && (++acqCount < ledger_fetch_size_)) - ledger = app_.getInboundLedgers().acquire( - *hash, seq, InboundLedger::Reason::GENERIC); - } + JLOG(m_journal.trace()) + << "Trying to fetch/publish valid ledger " << seq; - // Did we acquire the next ledger we need to publish? - if (ledger && (ledger->info().seq == pubSeq)) - { - ledger->setValidated(); - ret.push_back(ledger); - ++pubSeq; + std::shared_ptr ledger; + // This can throw + auto hash = hashOfSeq(*valLedger, seq, m_journal); + // VFALCO TODO Restructure this code so that zero is not + // used. + if (!hash) + hash = beast::zero; // kludge + if (seq == valSeq) + { + // We need to publish the ledger we just fully validated + ledger = valLedger; + } + else if (hash->isZero()) + { + JLOG(m_journal.fatal()) + << "Ledger: " << valSeq << " does not have hash for " + << seq; + UNREACHABLE( + "ripple::LedgerMaster::findNewLedgersToPublish : " + "ledger not found"); + } + else + { + ledger = mLedgerHistory.getLedgerByHash(*hash); + } + + if (!app_.config().LEDGER_REPLAY) + { + // Can we try to acquire the ledger we need? + if (!ledger && (++acqCount < ledger_fetch_size_)) + ledger = app_.getInboundLedgers().acquire( + *hash, seq, InboundLedger::Reason::GENERIC); + } + + // Did we acquire the next ledger we need to publish? + if (ledger && (ledger->info().seq == pubSeq)) + { + ledger->setValidated(); + ret.push_back(ledger); + ++pubSeq; + } } } @@ -1632,7 +1751,7 @@ LedgerMaster::getCompleteLedgers() std::string LedgerMaster::getPinnedLedgers() { - std::lock_guard sl(mCompleteLock); + std::lock_guard sl(mPinnedLock); return to_string(mPinnedLedgers); } @@ -1646,10 +1765,46 @@ LedgerMaster::getCompleteLedgersRangeSet() RangeSet LedgerMaster::getPinnedLedgersRangeSet() { - std::lock_guard sl(mCompleteLock); + std::lock_guard sl(mPinnedLock); return mPinnedLedgers; } +void +LedgerMaster::setPinnedLedgersRangeSet(const RangeSet& range_set) +{ + std::scoped_lock lock(mCompleteLock, mPinnedLock); + if (!mPinnedLedgers.empty()) + { + Throw( + "Expected mPinnedLedgers to be empty on startup"); + } + mPinnedLedgers.assign(range_set); + + // Normally, we defer merging pinned ranges into mCompleteLedgers until + // the first validated ledger (in setFullLedger), so that pinned history + // only becomes queryable after the node has network quorum. But if + // mCompleteLedgers is already non-empty (LOAD/standalone startup already + // called setFullLedger before we got here), the one-shot in + // setFullLedger won't fire again. Merge immediately in that case. + if (!mCompleteLedgers.empty()) + { + for (auto const& interval : mPinnedLedgers) + mCompleteLedgers.insert(interval); + mPinnedMergedToComplete = true; + JLOG(m_journal.info()) + << "Merged pinned ranges into complete ledgers immediately " + << "(startup already past setFullLedger): " << to_string(range_set); + } + else + { + // mCompleteLedgers is empty — defer merge to setFullLedger() + // when the first validated ledger arrives (network quorum). + JLOG(m_journal.info()) + << "Loaded pinned ranges, will merge on first validation: " + << to_string(range_set); + } +} + std::optional LedgerMaster::getCloseTimeBySeq(LedgerIndex ledgerIndex) { @@ -1813,15 +1968,18 @@ LedgerMaster::setLedgerRangePresent( std::uint32_t maxV, bool pin) { - std::lock_guard sl(mCompleteLock); - mCompleteLedgers.insert(range(minV, maxV)); - if (pin) { + std::scoped_lock lock(mCompleteLock, mPinnedLock); + mCompleteLedgers.insert(range(minV, maxV)); mPinnedLedgers.insert(range(minV, maxV)); JLOG(m_journal.info()) << "Pinned ledger range: " << minV << " - " << maxV; + return; } + + std::lock_guard cl(mCompleteLock); + mCompleteLedgers.insert(range(minV, maxV)); } void @@ -1841,7 +1999,7 @@ LedgerMaster::getCacheHitRate() void LedgerMaster::clearPriorLedgers(LedgerIndex seq) { - std::lock_guard sl(mCompleteLock); + std::scoped_lock lock(mCompleteLock, mPinnedLock); if (seq <= 0) return; diff --git a/src/xrpld/app/ledger/detail/PublishGap.h b/src/xrpld/app/ledger/detail/PublishGap.h new file mode 100644 index 000000000..60af2ec84 --- /dev/null +++ b/src/xrpld/app/ledger/detail/PublishGap.h @@ -0,0 +1,87 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2026 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_APP_LEDGER_DETAIL_PUBLISH_GAP_H_INCLUDED +#define RIPPLE_APP_LEDGER_DETAIL_PUBLISH_GAP_H_INCLUDED + +#include + +#include + +namespace ripple { +namespace detail { + +/** Decide whether pubSeq can safely advance across a gap in the + publication set, given the pinned-ledger ranges. + + Used by LedgerMaster::findNewLedgersToPublish: when the + publish-target set has been split into intervals (because pinned + ranges were subtracted from it), pubSeq may be behind the start of + the next interval. We may only "jump" pubSeq forward across that + gap if every skipped sequence is pinned — pinned ledgers are + persisted on disk and are intentionally never republished, so + skipping them does not violate the contiguous-publication + invariant. If any sequence in the gap is *not* pinned, then pubSeq + is lagging because an earlier non-pinned ledger could not be + fetched/published; we must halt rather than publish newer ledgers + out of order. + + Practical note: in production this guard is essentially never + exercised. Pinned ranges are old historical catalogue ranges and + pubSeq tracks the recent published tip — they don't overlap. The + guard matters only in test/standalone catalogue-load scenarios + where a catalogue can be loaded into an environment whose ledger + history places pinned seqs near the publish horizon. We keep the + guard rather than rely on the assumption holding because (a) the + cost is one RangeSet subtraction per gap and (b) silently + publishing across a non-pinned gap would corrupt the stream + contract. + + @param pubSeq The next sequence the publisher would like + to publish. + @param intervalStart The first sequence in the next non-pinned + interval to publish. + @param pinned The current pinned-ledger ranges. + + @return true pubSeq can safely jump to intervalStart + (pubSeq >= intervalStart, or every skipped seq is + pinned). + @return false the gap [pubSeq, intervalStart) contains at least + one non-pinned sequence; the publisher must stop. +*/ +inline bool +canSkipPinnedGap( + std::uint32_t pubSeq, + std::uint32_t intervalStart, + RangeSet const& pinned) +{ + // No gap. + if (pubSeq >= intervalStart) + return true; + + RangeSet skipped; + skipped.insert(range(pubSeq, intervalStart - 1)); + skipped -= pinned; + return skipped.empty(); +} + +} // namespace detail +} // namespace ripple + +#endif diff --git a/src/xrpld/app/misc/README.md b/src/xrpld/app/misc/README.md index 52e6e1493..769958d73 100644 --- a/src/xrpld/app/misc/README.md +++ b/src/xrpld/app/misc/README.md @@ -152,4 +152,4 @@ deletion routine. * [fetch_depth] will be silently set to equal the online_delete setting if online_delete is greater than fetch_depth. * In the [node_db] section, there is a performance tuning option, delete_batch, -which sets the maximum size in ledgers for each SQL DELETE query. +which sets the maximum number of SQL rows deleted per batch. diff --git a/src/xrpld/app/misc/SHAMapStore.h b/src/xrpld/app/misc/SHAMapStore.h index d8415713a..c78e653b2 100644 --- a/src/xrpld/app/misc/SHAMapStore.h +++ b/src/xrpld/app/misc/SHAMapStore.h @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -61,6 +62,9 @@ public: virtual LedgerIndex setCanDelete(LedgerIndex canDelete) = 0; + virtual void + setPinnedRanges(RangeSet const& ranges) = 0; + /** Whether advisory delete is enabled. */ virtual bool advisoryDelete() const = 0; diff --git a/src/xrpld/app/misc/SHAMapStoreImp.cpp b/src/xrpld/app/misc/SHAMapStoreImp.cpp index aff3569d3..31c0d80c6 100644 --- a/src/xrpld/app/misc/SHAMapStoreImp.cpp +++ b/src/xrpld/app/misc/SHAMapStoreImp.cpp @@ -21,15 +21,20 @@ #include #include +#include #include #include #include #include +#include #include #include #include #include +#include + +#include namespace ripple { void @@ -79,6 +84,20 @@ SHAMapStoreImp::SavedStateDB::setLastRotated(LedgerIndex seq) ripple::setLastRotated(sqlDb_, seq); } +std::string +SHAMapStoreImp::SavedStateDB::getPinnedRanges() +{ + std::lock_guard lock(mutex_); + return ripple::getPinnedRanges(sqlDb_); +} + +void +SHAMapStoreImp::SavedStateDB::setPinnedRanges(std::string const& ranges) +{ + std::lock_guard lock(mutex_); + ripple::setPinnedRanges(sqlDb_, ranges); +} + //------------------------------------------------------------------------------ SHAMapStoreImp::SHAMapStoreImp( @@ -117,6 +136,9 @@ SHAMapStoreImp::SHAMapStoreImp( get_if_exists(section, "online_delete", deleteInterval_); + // Always initialize state database for pinned ranges persistence + state_db_.init(config, dbName_); + if (deleteInterval_) { // Configuration that affects the behavior of online delete @@ -153,7 +175,6 @@ SHAMapStoreImp::SHAMapStoreImp( std::to_string(config.LEDGER_HISTORY) + ")"); } - state_db_.init(config, dbName_); if (!config.mem_backend()) dbPaths(); } @@ -183,6 +204,8 @@ SHAMapStoreImp::makeNodeStore(int readThreads) { SavedState state = state_db_.getState(); + // Create the rotation backends - needed for both DatabaseRotating and + // DatabasePinned auto writableBackend = makeBackendRotating(state.writableDb); auto archiveBackend = makeBackendRotating(state.archiveDb); if (!state.writableDb.size()) @@ -192,19 +215,58 @@ SHAMapStoreImp::makeNodeStore(int readThreads) state_db_.setState(state); } - // Create NodeStore with two backends to allow online deletion of - // data - auto dbr = std::make_unique( - app_, - scheduler_, - readThreads, - std::move(writableBackend), - std::move(archiveBackend), - nscfg, - app_.logs().journal(nodeStoreName_)); - fdRequired_ += dbr->fdRequired(); - dbRotating_ = dbr.get(); - db.reset(dynamic_cast(dbr.release())); + // Check if DatabasePinned should be created + if (nscfg.exists("pinned_type")) + { + // DatabasePinned uses the same rotation backends as + // DatabaseRotating but adds a persistent backend for pinned nodes + + // Create persistent backend for pinned data + Section pinnedConfig = nscfg; + pinnedConfig.set("type", *nscfg.get("pinned_type")); + if (auto pinnedPath = nscfg.get("pinned_path")) + pinnedConfig.set("path", *pinnedPath); + auto pinnedBackend = NodeStore::Manager::instance().make_Backend( + pinnedConfig, + megabytes(app_.config().getValueFor( + SizedItem::burstSize, std::nullopt)), + scheduler_, + app_.logs().journal(nodeStoreName_)); + pinnedBackend->open(); + + // Create DatabasePinned with rotation backends + persistent + + auto dbp = std::make_unique( + app_, + scheduler_, + readThreads, + std::move(writableBackend), + std::move(archiveBackend), + std::move(pinnedBackend), + nscfg, + app_.logs().journal(NodeStore::DatabasePinnedImp::JournalName)); + + fdRequired_ += dbp->fdRequired(); + dbRotating_ = + dbp.get(); // DatabasePinned inherits from DatabaseRotating + db.reset(dynamic_cast(dbp.release())); + } + else + { + // Create NodeStore with two backends to allow online deletion of + // data + auto dbr = std::make_unique( + app_, + scheduler_, + readThreads, + std::move(writableBackend), + std::move(archiveBackend), + nscfg, + app_.logs().journal(nodeStoreName_)); + fdRequired_ += dbr->fdRequired(); + dbRotating_ = dbr.get(); + db.reset(dynamic_cast(dbr.release())); + } } else { @@ -265,6 +327,58 @@ SHAMapStoreImp::copyNode(std::uint64_t& nodeCount, SHAMapTreeNode const& node) return true; } +void +SHAMapStoreImp::loadPinnedRanges() +{ + auto rangesStr = state_db_.getPinnedRanges(); + if (rangesStr.empty()) + return; + + auto const& nscfg = app_.config().section(ConfigSection::nodeDatabase()); + if (!nscfg.exists("pinned_type")) + { + JLOG(journal_.warn()) + << "Ignoring persisted pinned ranges because [node_db] " + "pinned_type is not configured: " + << rangesStr; + return; + } + + RangeSet persistedRanges; + if (!from_string(persistedRanges, rangesStr)) + { + Throw( + "Failed to parse persisted pinned ranges in state.db: " + + rangesStr); + } + + // state.db alone is not sufficient evidence that pinned history is + // available in the currently configured backend. Validate each interval + // boundary against the actual node store before advertising it as pinned. + auto const hasLedgerData = [this](std::uint32_t seq) { + auto const hash = app_.getRelationalDatabase().getHashByIndex(seq); + return hash.isNonZero() && + static_cast(app_.getNodeStore().fetchNodeObject(hash, seq)); + }; + + for (auto const& interval : persistedRanges) + { + if (!hasLedgerData(interval.lower()) || + !hasLedgerData(interval.upper())) + { + Throw( + "Persisted pinned interval " + + std::to_string(interval.lower()) + "-" + + std::to_string(interval.upper()) + + " is not present in the currently configured pinned store"); + } + } + + JLOG(journal_.info()) << "Loaded pinned ranges from database: " + << rangesStr; + app_.getLedgerMaster().setPinnedLedgersRangeSet(persistedRanges); +} + void SHAMapStoreImp::run() { @@ -486,17 +600,19 @@ SHAMapStoreImp::makeBackendRotating(std::string path) Section section{app_.config().section(ConfigSection::nodeDatabase())}; boost::filesystem::path newPath; - if (path.size()) + if (!path.empty()) { newPath = path; } else { + // Create new empty backend with unique path boost::filesystem::path p = get(section, "path"); p /= dbPrefix_; p += ".%%%%"; newPath = boost::filesystem::unique_path(p); } + section.set("path", newPath.string()); auto backend{NodeStore::Manager::instance().make_Backend( @@ -510,56 +626,43 @@ SHAMapStoreImp::makeBackendRotating(std::string path) } void -SHAMapStoreImp::clearSql( +SHAMapStoreImp::clearSqlRanges( LedgerIndex lastRotated, - std::string const& TableName, + RangeSet const& pinned, + std::string const& tableName, std::function()> const& getMinSeq, - std::function const& deleteBeforeSeq) + std::function const&)> const& deleteInRanges) { XRPL_ASSERT( deleteInterval_, "ripple::SHAMapStoreImp::clearSql : nonzero delete interval"); - LedgerIndex min = std::numeric_limits::max(); - - { - JLOG(journal_.trace()) - << "Begin: Look up lowest value of: " << TableName; - auto m = getMinSeq(); - JLOG(journal_.trace()) << "End: Look up lowest value of: " << TableName; - if (!m) - return; - min = *m; - } - - if (min > lastRotated || healthWait() == stopping) + auto m = getMinSeq(); + if (!m) return; - if (min == lastRotated) + + if (healthWait() == stopping) + return; + + // The pure interval-math is in detail::computeOnlineDeleteTargets so + // it can be unit-tested without spinning up SHAMapStoreImp. It + // returns the disjoint deletable intervals after subtracting pinned + // ranges from the base window [minSeq, lastRotated - 1]. + auto const target = + detail::computeOnlineDeleteTargets(*m, lastRotated, pinned); + if (target.empty()) { - // Micro-optimization mainly to clarify logs - JLOG(journal_.trace()) << "Nothing to delete from " << TableName; + JLOG(journal_.trace()) << "Nothing to delete from " << tableName + << " after considering pins."; return; } - JLOG(journal_.debug()) << "start deleting in: " << TableName << " from " - << min << " to " << lastRotated; - while (min < lastRotated) - { - min = std::min(lastRotated, min + deleteBatch_); - JLOG(journal_.trace()) - << "Begin: Delete up to " << deleteBatch_ - << " rows with LedgerSeq < " << min << " from: " << TableName; - deleteBeforeSeq(min); - JLOG(journal_.trace()) - << "End: Delete up to " << deleteBatch_ << " rows with LedgerSeq < " - << min << " from: " << TableName; - if (healthWait() == stopping) - return; - if (min < lastRotated) - std::this_thread::sleep_for(backOff_); - if (healthWait() == stopping) - return; - } - JLOG(journal_.debug()) << "finished deleting from: " << TableName; + JLOG(journal_.debug()) << "Pruning " << tableName + << ". Target ranges: " << to_string(target); + + // The deleteInRanges lambda will handle the actual database operations + deleteInRanges(target); + + JLOG(journal_.debug()) << "finished deleting from: " << tableName; } void @@ -581,6 +684,14 @@ SHAMapStoreImp::freshenCaches() void SHAMapStoreImp::clearPrior(LedgerIndex lastRotated) { + // Get pinned ranges to exclude from deletion + auto pinnedRanges = app_.getLedgerMaster().getPinnedLedgersRangeSet(); + if (!pinnedRanges.empty()) + { + JLOG(journal_.info()) + << "Online delete with pinned ranges: " << to_string(pinnedRanges); + } + // Do not allow ledgers to be acquired from the network // that are about to be deleted. minimumOnline_ = lastRotated + 1; @@ -600,36 +711,96 @@ SHAMapStoreImp::clearPrior(LedgerIndex lastRotated) if (app_.config().useTxTables()) { - clearSql( + clearSqlRanges( lastRotated, + pinnedRanges, "Transactions", [&db]() -> std::optional { return db->getTransactionsMinLedgerSeq(); }, - [&db](LedgerIndex min) -> void { - db->deleteTransactionsBeforeLedgerSeq(min); + [this, &db](RangeSet const& ranges) -> void { + for (auto const& interval : ranges) + { + // Simple delete loop with LIMIT + while (true) + { + if (healthWait() == stopping) + return; + + auto deleted = db->deleteTransactionsInRange( + interval.lower(), interval.upper(), deleteBatch_); + + if (deleted == 0) + break; + + JLOG(journal_.trace()) << "Deleted " << deleted + << " rows from Transactions"; + std::this_thread::sleep_for(backOff_); + } + } }); if (healthWait() == stopping) return; - clearSql( + clearSqlRanges( lastRotated, + pinnedRanges, "AccountTransactions", [&db]() -> std::optional { return db->getAccountTransactionsMinLedgerSeq(); }, - [&db](LedgerIndex min) -> void { - db->deleteAccountTransactionsBeforeLedgerSeq(min); + [this, &db](RangeSet const& ranges) -> void { + for (auto const& interval : ranges) + { + // Simple delete loop with LIMIT + while (true) + { + if (healthWait() == stopping) + return; + + auto deleted = db->deleteAccountTransactionsInRange( + interval.lower(), interval.upper(), deleteBatch_); + + if (deleted == 0) + break; + + JLOG(journal_.trace()) + << "Deleted " << deleted + << " rows from AccountTransactions"; + std::this_thread::sleep_for(backOff_); + } + } }); if (healthWait() == stopping) return; } - clearSql( + clearSqlRanges( lastRotated, + pinnedRanges, "Ledgers", [db]() -> std::optional { return db->getMinLedgerSeq(); }, - [db](LedgerIndex min) -> void { db->deleteBeforeLedgerSeq(min); }); + [this, db](RangeSet const& ranges) -> void { + for (auto const& interval : ranges) + { + // Simple delete loop with LIMIT + while (true) + { + if (healthWait() == stopping) + return; + + auto deleted = db->deleteLedgersInRange( + interval.lower(), interval.upper(), deleteBatch_); + + if (deleted == 0) + break; + + JLOG(journal_.trace()) + << "Deleted " << deleted << " rows from Ledgers"; + std::this_thread::sleep_for(backOff_); + } + } + }); if (healthWait() == stopping) return; } diff --git a/src/xrpld/app/misc/SHAMapStoreImp.h b/src/xrpld/app/misc/SHAMapStoreImp.h index 7d36f092b..1d4ac2bf1 100644 --- a/src/xrpld/app/misc/SHAMapStoreImp.h +++ b/src/xrpld/app/misc/SHAMapStoreImp.h @@ -24,8 +24,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -67,6 +69,10 @@ private: setState(SavedState const& state); void setLastRotated(LedgerIndex seq); + std::string + getPinnedRanges(); + void + setPinnedRanges(std::string const& ranges); }; Application& app_; @@ -132,6 +138,12 @@ public: : fetch_depth; } + bool + isOnlineDeleteEnabled() const + { + return deleteInterval_ > 0; + } + std::unique_ptr makeNodeStore(int readThreads) override; @@ -168,6 +180,14 @@ public: void onLedgerClosed(std::shared_ptr const& ledger) override; + void + setPinnedRanges(RangeSet const& ranges) override + { + // Save pinned ranges to database for persistence + std::string rangesStr = to_string(ranges); + state_db_.setPinnedRanges(rangesStr); + } + void rendezvous() const override; int @@ -210,11 +230,14 @@ private: * Call with mutex object unlocked. */ void - clearSql( + clearSqlRanges( LedgerIndex lastRotated, - std::string const& TableName, + RangeSet const& pinned, + std::string const& tableName, std::function()> const& getMinSeq, - std::function const& deleteBeforeSeq); + std::function const&)> const& + deleteInRanges); + void clearCaches(LedgerIndex validatedSeq); void @@ -233,12 +256,24 @@ private: [[nodiscard]] HealthResult healthWait(); + void + loadPinnedRanges(); + public: void start() override { + // Always load pinned ranges on startup + loadPinnedRanges(); + + // NOTE: Startup cleanup of unpinned ledgers was removed - it caused + // SQLite lock contention and confusing pauses. Database will shrink + // incrementally during rotation. Custom tools will provide pruning + // management for specialized needs. if (deleteInterval_) + { thread_ = std::thread(&SHAMapStoreImp::run, this); + } } void diff --git a/src/xrpld/app/misc/detail/AMMUtils.cpp b/src/xrpld/app/misc/detail/AMMUtils.cpp index 8a6917b8c..c376039f7 100644 --- a/src/xrpld/app/misc/detail/AMMUtils.cpp +++ b/src/xrpld/app/misc/detail/AMMUtils.cpp @@ -148,6 +148,31 @@ ammLPHolds( // Put balance in account terms. amount.negate(); } + + // If tokens can be escrowed then they can be locked in the trustline + // which means we must never spend them until the escrow is released. + if (view.rules().enabled(featurePaychanAndEscrowForTokens) && + sle->isFieldPresent(sfLockedBalance)) + { + STAmount const lockedBalance = sle->getFieldAmount(sfLockedBalance); + STAmount const spendableBalance = amount - + (lpAccount > ammAccount ? -lockedBalance : lockedBalance); + + // RH NOTE: this is defensively programmed, it should never fire + // if something bad does happen the trustline acts as a frozen line. + if (spendableBalance < beast::zero || spendableBalance > amount) + { + // LCOV_EXCL_START + JLOG(j.error()) + << "SpendableBalance has illegal value in accountHolds " + << spendableBalance; + amount.clear(Issue{currency, ammAccount}); + // LCOV_EXCL_STOP + } + else + amount = spendableBalance; + } + amount.setIssuer(ammAccount); JLOG(j.trace()) << "ammLPHolds:" diff --git a/src/xrpld/app/misc/detail/OnlineDeleteRanges.h b/src/xrpld/app/misc/detail/OnlineDeleteRanges.h new file mode 100644 index 000000000..3935b0edf --- /dev/null +++ b/src/xrpld/app/misc/detail/OnlineDeleteRanges.h @@ -0,0 +1,77 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2026 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_APP_MISC_DETAIL_ONLINE_DELETE_RANGES_H_INCLUDED +#define RIPPLE_APP_MISC_DETAIL_ONLINE_DELETE_RANGES_H_INCLUDED + +#include + +#include + +namespace ripple { +namespace detail { + +/** Compute the actual SQL-deletion target windows for an online-delete + pass, given a base [minSeq, lastRotated - 1] window and the pinned + ranges that must be preserved. + + The base window is the half-open span the caller would normally + delete (everything older than `lastRotated`, but no older than the + minimum sequence currently stored in the table being cleared). + Pinned ranges (catalogue history that must survive rotation) are + subtracted from the base — the result may be empty (everything is + pinned), one interval (no overlap, or pinned only trims an edge), + or multiple disjoint intervals (pins cut a hole in the base). + + Pure function so the math is unit-testable without standing up + SHAMapStoreImp. The actual side-effecting deletions (driven by + a per-table lambda in clearSqlRanges) operate on whatever interval + set this helper returns. + + @param minSeq The smallest sequence currently stored in the + table being cleared. If >= lastRotated, the + base window is empty and we return {}. + @param lastRotated The exclusive upper bound — sequences from + this seq forward are kept. + @param pinned Pinned-ledger ranges to preserve. + + @return The set of intervals to actually delete. Empty if either + the base window is empty or pinned ranges fully cover it. +*/ +inline RangeSet +computeOnlineDeleteTargets( + std::uint32_t minSeq, + std::uint32_t lastRotated, + RangeSet const& pinned) +{ + // Empty base window: nothing newer than minSeq qualifies for + // deletion. + if (minSeq >= lastRotated) + return {}; + + RangeSet target; + target.insert(range(minSeq, lastRotated - 1)); + target -= pinned; + return target; +} + +} // namespace detail +} // namespace ripple + +#endif diff --git a/src/xrpld/app/rdb/State.h b/src/xrpld/app/rdb/State.h index e65e9d4d5..6590cdef0 100644 --- a/src/xrpld/app/rdb/State.h +++ b/src/xrpld/app/rdb/State.h @@ -92,6 +92,23 @@ setSavedState(soci::session& session, SavedState const& state); void setLastRotated(soci::session& session, LedgerIndex seq); +/** + * @brief getPinnedRanges Returns the serialized pinned ledger ranges. + * @param session Session with the database. + * @return Serialized range set string (e.g., + * "1000000-2000000,3000000-3500000"). + */ +std::string +getPinnedRanges(soci::session& session); + +/** + * @brief setPinnedRanges Updates the pinned ledger ranges. + * @param session Session with the database. + * @param ranges Serialized range set string to save. + */ +void +setPinnedRanges(soci::session& session, std::string const& ranges); + } // namespace ripple #endif diff --git a/src/xrpld/app/rdb/backend/RWDBDatabase.h b/src/xrpld/app/rdb/backend/RWDBDatabase.h index 67f9ab455..edbcbed92 100644 --- a/src/xrpld/app/rdb/backend/RWDBDatabase.h +++ b/src/xrpld/app/rdb/backend/RWDBDatabase.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -566,6 +567,120 @@ public: // No-op for in-memory database } + std::size_t + deleteLedgersInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit = std::nullopt) override + { + XRPL_ASSERT( + minSeq <= maxSeq, + "RWDBDatabase::deleteLedgersInRange : minSeq <= maxSeq"); + XRPL_ASSERT( + !rowLimit || *rowLimit > 0, + "RWDBDatabase::deleteLedgersInRange : rowLimit must be positive"); + std::unique_lock lock(mutex_); + auto it = ledgers_.lower_bound(minSeq); + auto end = ledgers_.upper_bound(maxSeq); + + std::size_t count = 0; + while (it != end) + { + if (rowLimit && count >= *rowLimit) + break; + + ledgerHashToSeq_.erase(it->second.info.hash); + it = ledgers_.erase(it); + ++count; + } + return count; + } + + std::size_t + deleteTransactionsInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit = std::nullopt) override + { + XRPL_ASSERT( + minSeq <= maxSeq, + "RWDBDatabase::deleteTransactionsInRange : minSeq <= maxSeq"); + XRPL_ASSERT( + !rowLimit || *rowLimit > 0, + "RWDBDatabase::deleteTransactionsInRange : rowLimit must be " + "positive"); + if (!useTxTables_) + return 0; + + std::unique_lock lock(mutex_); + auto it = ledgers_.lower_bound(minSeq); + auto end = ledgers_.upper_bound(maxSeq); + + // Erase from both the per-ledger map and the global index. + // Important: erase from per-ledger map as we go so that partial + // deletes (hitting limit) make forward progress on the next call. + std::size_t count = 0; + while (it != end) + { + auto txIt = it->second.transactions.begin(); + while (txIt != it->second.transactions.end()) + { + if (rowLimit && count >= *rowLimit) + return count; + + transactionMap_.erase(txIt->first); + txIt = it->second.transactions.erase(txIt); + ++count; + } + ++it; + } + return count; + } + + std::size_t + deleteAccountTransactionsInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit = std::nullopt) override + { + XRPL_ASSERT( + minSeq <= maxSeq, + "RWDBDatabase::deleteAccountTransactionsInRange : minSeq <= " + "maxSeq"); + XRPL_ASSERT( + !rowLimit || *rowLimit > 0, + "RWDBDatabase::deleteAccountTransactionsInRange : rowLimit must be " + "positive"); + if (!useTxTables_) + return 0; + + std::unique_lock lock(mutex_); + std::size_t count = 0; + + for (auto& [_, accountData] : accountTxMap_) + { + auto txIt = accountData.ledgerTxMap.lower_bound(minSeq); + auto txEnd = accountData.ledgerTxMap.upper_bound(maxSeq); + + while (txIt != txEnd) + { + std::size_t toDelete = txIt->second.size(); + if (rowLimit && count + toDelete > *rowLimit) + { + // Partial deletion from this ledger + toDelete = *rowLimit - count; + txIt->second.resize(txIt->second.size() - toDelete); + count += toDelete; + return count; + } + + count += toDelete; + txIt = accountData.ledgerTxMap.erase(txIt); + } + } + return count; + } + ~RWDBDatabase() { // Regular maps can use standard clear diff --git a/src/xrpld/app/rdb/backend/SQLiteDatabase.h b/src/xrpld/app/rdb/backend/SQLiteDatabase.h index 27c01c1b8..c8579afa1 100644 --- a/src/xrpld/app/rdb/backend/SQLiteDatabase.h +++ b/src/xrpld/app/rdb/backend/SQLiteDatabase.h @@ -21,6 +21,7 @@ #define RIPPLE_APP_RDB_BACKEND_SQLITEDATABASE_H_INCLUDED #include +#include namespace ripple { @@ -295,6 +296,47 @@ public: virtual uint32_t getKBUsedTransaction() = 0; + /** + * @brief deleteLedgersInRange Deletes ledgers within the specified range. + * @param minSeq Minimum ledger sequence (inclusive). + * @param maxSeq Maximum ledger sequence (inclusive). + * @param rowLimit Optional limit on number of rows to delete. + * @return Number of rows deleted. + */ + virtual std::size_t + deleteLedgersInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit = std::nullopt) = 0; + + /** + * @brief deleteTransactionsInRange Deletes transactions within the + * specified ledger sequence range. + * @param minSeq Minimum ledger sequence (inclusive). + * @param maxSeq Maximum ledger sequence (inclusive). + * @param rowLimit Optional limit on number of rows to delete. + * @return Number of rows deleted. + */ + virtual std::size_t + deleteTransactionsInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit = std::nullopt) = 0; + + /** + * @brief deleteAccountTransactionsInRange Deletes account transactions + * within the specified ledger sequence range. + * @param minSeq Minimum ledger sequence (inclusive). + * @param maxSeq Maximum ledger sequence (inclusive). + * @param rowLimit Optional limit on number of rows to delete. + * @return Number of rows deleted. + */ + virtual std::size_t + deleteAccountTransactionsInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit = std::nullopt) = 0; + /** * @brief Closes the ledger database */ diff --git a/src/xrpld/app/rdb/backend/detail/Node.cpp b/src/xrpld/app/rdb/backend/detail/Node.cpp index 15b2560b2..a8c3c03e9 100644 --- a/src/xrpld/app/rdb/backend/detail/Node.cpp +++ b/src/xrpld/app/rdb/backend/detail/Node.cpp @@ -157,6 +157,45 @@ deleteBeforeLedgerSeq( << ledgerSeq << ";"; } +std::size_t +deleteRange( + soci::session& session, + TableType type, + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit) +{ + XRPL_ASSERT( + minSeq <= maxSeq, "ripple::detail::deleteRange : minSeq <= maxSeq"); + XRPL_ASSERT( + !rowLimit || *rowLimit > 0, + "ripple::detail::deleteRange : rowLimit must be positive"); + std::string sql; + if (rowLimit) + { + // SQLite doesn't support DELETE...LIMIT unless compiled with + // SQLITE_ENABLE_UPDATE_DELETE_LIMIT. Use subquery workaround. + sql = "DELETE FROM " + to_string(type) + + " WHERE rowid IN (SELECT rowid FROM " + to_string(type) + + " WHERE LedgerSeq >= " + std::to_string(minSeq) + + " AND LedgerSeq <= " + std::to_string(maxSeq) + + " ORDER BY LedgerSeq ASC LIMIT " + std::to_string(*rowLimit) + ");"; + } + else + { + sql = "DELETE FROM " + to_string(type) + + " WHERE LedgerSeq >= " + std::to_string(minSeq) + + " AND LedgerSeq <= " + std::to_string(maxSeq) + ";"; + } + + session << sql; + + // Get the number of rows deleted + long changes = 0; + session << "SELECT changes();", soci::into(changes); + return static_cast(changes); +} + std::size_t getRows(soci::session& session, TableType type) { @@ -224,8 +263,11 @@ saveValidatedLedger( Serializer s(128); s.add32(HashPrefix::ledgerMaster); addRaw(ledger->info(), s); + // Use pinnedLEDGER type for pinned ledgers, hotLEDGER for others + auto ledgerType = + app.getLedgerMaster().isPinned(seq) ? pinnedLEDGER : hotLEDGER; app.getNodeStore().store( - hotLEDGER, std::move(s.modData()), ledger->info().hash, seq); + ledgerType, std::move(s.modData()), ledger->info().hash, seq); } std::shared_ptr aLedger; @@ -235,8 +277,13 @@ saveValidatedLedger( if (!aLedger) { aLedger = std::make_shared(ledger, app); - app.getAcceptedLedgerCache().canonicalize_replace_client( - ledger->info().hash, aLedger); + + // Only cache if the ledger is NOT in the pinned range + if (!app.getLedgerMaster().isPinned(ledger->info().seq)) + { + app.getAcceptedLedgerCache().canonicalize_replace_client( + ledger->info().hash, aLedger); + } } } catch (std::exception const&) diff --git a/src/xrpld/app/rdb/backend/detail/Node.h b/src/xrpld/app/rdb/backend/detail/Node.h index 59c484d20..b12e61389 100644 --- a/src/xrpld/app/rdb/backend/detail/Node.h +++ b/src/xrpld/app/rdb/backend/detail/Node.h @@ -102,6 +102,25 @@ deleteBeforeLedgerSeq( TableType type, LedgerIndex ledgerSeq); +/** + * @brief deleteRange Deletes entries in given table + * for the ledgers within the specified range (inclusive) + * with an optional limit on the number of rows to delete. + * @param session Session with database. + * @param type Table ID from which entries will be deleted. + * @param minSeq Minimum ledger sequence (inclusive). + * @param maxSeq Maximum ledger sequence (inclusive). + * @param rowLimit Optional limit on number of rows to delete. + * @return Number of rows actually deleted. + */ +std::size_t +deleteRange( + soci::session& session, + TableType type, + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit); + /** * @brief getRows Returns number of rows in given table. * @param session Session with database. diff --git a/src/xrpld/app/rdb/backend/detail/SQLiteDatabase.cpp b/src/xrpld/app/rdb/backend/detail/SQLiteDatabase.cpp index 0dd0d9926..3e61b26cd 100644 --- a/src/xrpld/app/rdb/backend/detail/SQLiteDatabase.cpp +++ b/src/xrpld/app/rdb/backend/detail/SQLiteDatabase.cpp @@ -83,6 +83,24 @@ public: void deleteAccountTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) override; + std::size_t + deleteLedgersInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit = std::nullopt) override; + + std::size_t + deleteTransactionsInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit = std::nullopt) override; + + std::size_t + deleteAccountTransactionsInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit = std::nullopt) override; + std::size_t getTransactionCount() override; @@ -949,6 +967,48 @@ SQLiteDatabaseImp::closeLedgerDB() lgrdb_.reset(); } +std::size_t +SQLiteDatabaseImp::deleteLedgersInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit) +{ + if (!existsLedger()) + return 0; + + auto db = checkoutLedger(); + return detail::deleteRange( + *db, detail::TableType::Ledgers, minSeq, maxSeq, rowLimit); +} + +std::size_t +SQLiteDatabaseImp::deleteTransactionsInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit) +{ + if (!existsTransaction()) + return 0; + + auto db = checkoutTransaction(); + return detail::deleteRange( + *db, detail::TableType::Transactions, minSeq, maxSeq, rowLimit); +} + +std::size_t +SQLiteDatabaseImp::deleteAccountTransactionsInRange( + LedgerIndex minSeq, + LedgerIndex maxSeq, + std::optional rowLimit) +{ + if (!existsTransaction()) + return 0; + + auto db = checkoutTransaction(); + return detail::deleteRange( + *db, detail::TableType::AccountTransactions, minSeq, maxSeq, rowLimit); +} + void SQLiteDatabaseImp::closeTransactionDB() { diff --git a/src/xrpld/app/rdb/detail/State.cpp b/src/xrpld/app/rdb/detail/State.cpp index c3de86068..edad02cc6 100644 --- a/src/xrpld/app/rdb/detail/State.cpp +++ b/src/xrpld/app/rdb/detail/State.cpp @@ -43,6 +43,11 @@ initStateDB( " CanDeleteSeq INTEGER" ");"; + session << "CREATE TABLE IF NOT EXISTS PinnedLedgers (" + " Key INTEGER PRIMARY KEY," + " RangeSet TEXT" + ");"; + std::int64_t count = 0; { // SOCI requires boost::optional (not std::optional) as the parameter. @@ -75,6 +80,17 @@ initStateDB( { session << "INSERT INTO CanDelete VALUES (1, 0);"; } + + // Initialize PinnedLedgers table with empty range if not exists + { + boost::optional countO; + session << "SELECT COUNT(Key) FROM PinnedLedgers WHERE Key = 1;", + soci::into(countO); + if (!countO || *countO == 0) + { + session << "INSERT INTO PinnedLedgers VALUES (1, '');"; + } + } } LedgerIndex @@ -127,4 +143,23 @@ setLastRotated(soci::session& session, LedgerIndex seq) soci::use(seq); } +std::string +getPinnedRanges(soci::session& session) +{ + boost::optional rangeStr; + session << "SELECT RangeSet FROM PinnedLedgers WHERE Key = 1;", + soci::into(rangeStr); + + if (rangeStr) + return *rangeStr; + return ""; +} + +void +setPinnedRanges(soci::session& session, std::string const& ranges) +{ + session << "UPDATE PinnedLedgers SET RangeSet = :ranges WHERE Key = 1;", + soci::use(ranges); +} + } // namespace ripple diff --git a/src/xrpld/app/tx/detail/AMMClawback.cpp b/src/xrpld/app/tx/detail/AMMClawback.cpp index 4287067cf..21adc4a08 100644 --- a/src/xrpld/app/tx/detail/AMMClawback.cpp +++ b/src/xrpld/app/tx/detail/AMMClawback.cpp @@ -259,7 +259,6 @@ AMMClawback::equalWithdrawMatchingOneAmount( STAmount const& amount) { auto frac = Number{amount} / amountBalance; - auto amount2Withdraw = amount2Balance * frac; auto const lpTokensWithdraw = toSTAmount(lptAMMBalance.issue(), lptAMMBalance * frac); diff --git a/src/xrpld/app/tx/detail/ClaimReward.cpp b/src/xrpld/app/tx/detail/ClaimReward.cpp index 6f99c4884..e298a4015 100644 --- a/src/xrpld/app/tx/detail/ClaimReward.cpp +++ b/src/xrpld/app/tx/detail/ClaimReward.cpp @@ -17,6 +17,7 @@ */ //============================================================================== +#include #include #include #include @@ -60,6 +61,57 @@ ClaimReward::preflight(PreflightContext const& ctx) return temMALFORMED; } + if (ctx.tx.isFieldPresent(sfClaimCurrency)) + { + // IOU RewardClaim + if (!ctx.rules.enabled(featureIOURewardClaim)) + return temDISABLED; + + auto const claimAsset = ctx.tx[sfClaimCurrency]; + bool const isMPT = claimAsset.holds(); + + if (isMPT) + { + JLOG(ctx.j.debug()) << "ClaimReward: MPT is not supported yet."; + return temMALFORMED; + } + + auto const claimIssue = claimAsset.get(); + if (claimIssue.account == beast::zero || isXRP(claimIssue.currency)) + return temMALFORMED; + + if (claimIssue.account == ctx.tx.getAccountID(sfAccount)) + return temMALFORMED; + } + + if (ctx.rules.enabled(featureXahauGenesis) && + ctx.rules.enabled(featureIOURewardClaim) && + ctx.tx.isFieldPresent(sfIssuer)) + { + static auto const genesisAccountId = calcAccountID( + generateKeyPair( + KeyType::secp256k1, generateSeed("masterpassphrase")) + .first); + auto const issuer = ctx.tx.getAccountID(sfIssuer); + if (ctx.tx.isFieldPresent(sfClaimCurrency)) + { + if (issuer == genesisAccountId) + { + JLOG(ctx.j.debug()) << "ClaimReward (IOU): Issuer cannot " + "be the Genesis account"; + return temBAD_ISSUER; + } + } + else + { + if (issuer != genesisAccountId) + { + JLOG(ctx.j.debug()) << "ClaimReward (XAH): Issuer must be " + "the Genesis account"; + return temBAD_ISSUER; + } + } + } return preflight2(ctx); } @@ -90,8 +142,53 @@ ClaimReward::preclaim(PreclaimContext const& ctx) if (sleIssuer->isFieldPresent(sfAMMID)) return tecNO_PERMISSION; + + if (ctx.view.rules().enabled(featureIOURewardClaim)) + { + auto const& sleHook = ctx.view.read(keylet::hook(*issuer)); + if (!sleHook || !sleHook->isFieldPresent(sfHooks) || + sleHook->getFieldArray(sfHooks).empty()) + return tecNO_TARGET; + + bool hasClaimRewardHook = false; + auto const& hooks = sleHook->getFieldArray(sfHooks); + for (auto const& hook : hooks) + { + if (!hook.isFieldPresent(sfHookHash)) + return tefINTERNAL; // LCOV_EXCL_LINE + auto const& hash = hook.getFieldH256(sfHookHash); + auto const& sleDef = + ctx.view.read(keylet::hookDefinition(hash)); + if (!sleDef) + return tefINTERNAL; // LCOV_EXCL_LINE + + auto const& hookOn = + hook::getHookOn(hook, sleDef, sfHookOnIncoming); + if (hook::canHook(ttCLAIM_REWARD, hookOn)) + { + hasClaimRewardHook = true; + break; + } + } + if (!hasClaimRewardHook) + return tecNO_TARGET; + } } + if (ctx.tx.isFieldPresent(sfClaimCurrency)) + { + auto const claimCurrency = ctx.tx[sfClaimCurrency]; + bool const isMPT = claimCurrency.holds(); + + if (isMPT) + return tefINTERNAL; + + auto const claimIssue = claimCurrency.get(); + + if (!ctx.view.exists( + keylet::line(id, claimIssue.account, claimIssue.currency))) + return tecNO_LINE; + } return tesSUCCESS; } @@ -105,6 +202,61 @@ ClaimReward::doApply() std::optional flags = ctx_.tx[~sfFlags]; bool isOptOut = flags && *flags == tfOptOut; + + uint32_t lgrCur = view().seq(); + uint32_t lgrFirst = lgrCur; + uint32_t lgrLast = lgrCur; + uint64_t accumulator = 0ULL; + uint32_t rewardTime = std::chrono::duration_cast( + ctx_.app.getLedgerMaster() + .getValidatedLedger() + ->info() + .parentCloseTime.time_since_epoch()) + .count(); + + if (ctx_.tx.isFieldPresent(sfClaimCurrency)) + { + auto const claimCurrency = ctx_.tx[sfClaimCurrency]; + bool const isMPT = claimCurrency.holds(); + + if (isMPT) + return tefINTERNAL; + + auto const claimIssue = claimCurrency.get(); + + auto lineSle = view().peek( + keylet::line(account_, claimIssue.account, claimIssue.currency)); + if (!lineSle) + return tefINTERNAL; + + bool const isHigh = account_ > claimIssue.account; + auto const& rewardField = isHigh ? sfHighReward : sfLowReward; + + if (isOptOut) + { + if (lineSle->isFieldPresent(rewardField)) + lineSle->makeFieldAbsent(rewardField); + } + else + { + auto const& rewardAccumulatorField = + lineSle->getFieldAmount(isHigh ? sfHighLimit : sfLowLimit) + .zeroed(); + + // all actual rewards are handled by the hook on the sfIssuer + // the tt just resets the counters + auto& reward = lineSle->peekFieldObject(rewardField); + reward.setFieldU32(sfRewardLgrFirst, lgrFirst); + reward.setFieldU32(sfRewardLgrLast, lgrLast); + reward.setFieldAmount( + sfTrustLineRewardAccumulator, rewardAccumulatorField); + reward.setFieldU32(sfRewardTime, rewardTime); + } + + view().update(lineSle); + return tesSUCCESS; + } + if (isOptOut) { if (sle->isFieldPresent(sfRewardLgrFirst)) @@ -120,18 +272,10 @@ ClaimReward::doApply() { // all actual rewards are handled by the hook on the sfIssuer // the tt just resets the counters - uint32_t lgrCur = view().seq(); - sle->setFieldU32(sfRewardLgrFirst, lgrCur); - sle->setFieldU32(sfRewardLgrLast, lgrCur); - sle->setFieldU64(sfRewardAccumulator, 0ULL); - sle->setFieldU32( - sfRewardTime, - std::chrono::duration_cast( - ctx_.app.getLedgerMaster() - .getValidatedLedger() - ->info() - .parentCloseTime.time_since_epoch()) - .count()); + sle->setFieldU32(sfRewardLgrFirst, lgrFirst); + sle->setFieldU32(sfRewardLgrLast, lgrLast); + sle->setFieldU64(sfRewardAccumulator, accumulator); + sle->setFieldU32(sfRewardTime, rewardTime); } view().update(sle); diff --git a/src/xrpld/app/tx/detail/SetHook.cpp b/src/xrpld/app/tx/detail/SetHook.cpp index c7f2ffd6a..53b3b6b7e 100644 --- a/src/xrpld/app/tx/detail/SetHook.cpp +++ b/src/xrpld/app/tx/detail/SetHook.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -230,6 +231,7 @@ SetHook::inferOperation(STObject const& hookSetObj) hookSetObj.isFieldPresent(sfHookOnIncoming))) && !hookSetObj.isFieldPresent(sfHookCanEmit) && !hookSetObj.isFieldPresent(sfHookApiVersion) && + !hookSetObj.isFieldPresent(sfHookName) && !hookSetObj.isFieldPresent(sfFlags)) return hsoNOOP; @@ -267,6 +269,7 @@ SetHook::validateHookSetEntry(SetHookCtx& ctx, STObject const& hookSetObj) hookSetObj.isFieldPresent(sfHookOnIncoming) || hookSetObj.isFieldPresent(sfHookCanEmit) || hookSetObj.isFieldPresent(sfHookApiVersion) || + hookSetObj.isFieldPresent(sfHookName) || !hookSetObj.isFieldPresent(sfFlags) || !hookSetObj.isFieldPresent(sfHookNamespace)) { @@ -300,6 +303,7 @@ SetHook::validateHookSetEntry(SetHookCtx& ctx, STObject const& hookSetObj) hookSetObj.isFieldPresent(sfHookCanEmit) || hookSetObj.isFieldPresent(sfHookApiVersion) || hookSetObj.isFieldPresent(sfHookNamespace) || + hookSetObj.isFieldPresent(sfHookName) || !hookSetObj.isFieldPresent(sfFlags)) { JLOG(ctx.j.trace()) @@ -510,6 +514,14 @@ SetHook::validateHookSetEntry(SetHookCtx& ctx, STObject const& hookSetObj) // pass } + // validate sfHookName + if (hookSetObj.isFieldPresent(sfHookName)) + { + auto name = hookSetObj.getFieldVL(sfHookName); + if (!validateHookName(name, ctx.j)) + return false; + } + // finally validate web assembly byte code { if (!hookSetObj.isFieldPresent(sfCreateCode)) @@ -609,6 +621,23 @@ SetHook::validateHookSetEntry(SetHookCtx& ctx, STObject const& hookSetObj) } } +bool +SetHook::validateHookName(Blob const& name, beast::Journal const& j) +{ + if (name.size() != 0 && (name.size() < 4 || 16 < name.size())) + { + JLOG(j.trace()) + << "sfHookName must be between 8 and 32 hex characters."; + return false; + } + if (!URIToken::validateUTF8(name)) + { + JLOG(j.trace()) << "sfHookName must be a valid UTF-8 string."; + return false; + } + return true; +} + // Note that if fee calculation causes an overflow then INITIAL_XRP is returned // as a way of ensuring that the txn cannot possibly meet the fee requirement. XRPAmount @@ -783,6 +812,10 @@ SetHook::preflight(PreflightContext const& ctx) hookSetObj.isFieldPresent(sfHookCanEmit)) return temDISABLED; + if (!ctx.rules.enabled(featureNamedHooks) && + hookSetObj.isFieldPresent(sfHookName)) + return temDISABLED; + for (auto const& hookSetElement : hookSetObj) { auto const& name = hookSetElement.getFName(); @@ -792,7 +825,7 @@ SetHook::preflight(PreflightContext const& ctx) name != sfHookOn && name != sfHookOnOutgoing && name != sfHookOnIncoming && name != sfHookGrants && name != sfHookApiVersion && name != sfFlags && - name != sfHookCanEmit) + name != sfHookCanEmit && name != sfHookName) { JLOG(ctx.j.trace()) << "HookSet(" << hook::log::HOOK_INVALID_FIELD << ")[" @@ -1348,6 +1381,8 @@ SetHook::setHook() std::optional newHookCanEmit; std::optional defHookCanEmit; + std::optional newHookName; + // when hsoCREATE is invoked it populates this variable in case the hook // definition already exists and the operation falls through into a // hsoINSTALL operation instead @@ -1416,7 +1451,6 @@ SetHook::setHook() if (oldDefSLE && oldDefSLE->isFieldPresent(sfHookCanEmit)) defHookCanEmit = oldDefSLE->getFieldH256(sfHookCanEmit); - if (oldHook && oldHook->get().isFieldPresent(sfHookCanEmit)) oldHookCanEmit = oldHook->get().getFieldH256(sfHookCanEmit); else if (defHookCanEmit) @@ -1453,6 +1487,9 @@ SetHook::setHook() newNamespace = hookSetObj->get().getFieldH256(sfHookNamespace); newDirKeylet = keylet::hookStateDir(account_, *newNamespace); } + + if (hookSetObj->get().isFieldPresent(sfHookName)) + newHookName = hookSetObj->get().getFieldVL(sfHookName); } // users may destroy a namespace in any operation except NOOP and @@ -1574,6 +1611,9 @@ SetHook::setHook() newHook.setFieldH256( sfHookNamespace, oldHook->get().getFieldH256(sfHookNamespace)); + if (oldHook->get().isFieldPresent(sfHookName)) + newHook.setFieldVL( + sfHookName, oldHook->get().getFieldVL(sfHookName)); // set the namespace if it differs from the definition namespace if (newNamespace) @@ -1638,6 +1678,20 @@ SetHook::setHook() newHook.setFieldH256(sfHookCanEmit, *newHookCanEmit); } + // set the hookname field on ltHook when it is explicitly + // provided + if (newHookName) + { + if (newHookName->size() == 0) + { + newHook.makeFieldAbsent(sfHookName); + } + else + { + newHook.setFieldVL(sfHookName, *newHookName); + } + } + // parameters if (hookSetObj->get().isFieldPresent(sfHookParameters) && hookSetObj->get().getFieldArray(sfHookParameters).empty()) @@ -1839,6 +1893,12 @@ SetHook::setHook() newHook.setFieldArray(sfHookGrants, grants); } + if (hookSetObj->get().isFieldPresent(sfHookName) && + hookSetObj->get().getFieldVL(sfHookName).size() > 0) + newHook.setFieldVL( + sfHookName, + hookSetObj->get().getFieldVL(sfHookName)); + slesToInsert.emplace(keylet, newHookDef); newHook.setFieldH256(sfHookHash, *createHookHash); newHooks.push_back(std::move(newHook)); @@ -1950,6 +2010,11 @@ SetHook::setHook() *defHookCanEmit == *newHookCanEmit)) newHook.setFieldH256(sfHookCanEmit, *newHookCanEmit); + // set the hookname field on ltHook when it is explicitly + // provided + if (newHookName && newHookName->size() > 0) + newHook.setFieldVL(sfHookName, *newHookName); + // parameters TER result = updateHookParameters( ctx, @@ -1999,7 +2064,8 @@ SetHook::setHook() // sfParameters: 1 reserve PER entry // sfGrants are: 1 reserve PER entry // sfHookHash, sfHookNamespace, sfHookOn, sfHookOnOutgoing, - // sfHookOnIncoming, sfHookCanEmit sfHookApiVersion, sfFlags: free + // sfHookOnIncoming, sfHookCanEmit sfHookApiVersion, sfFlags, + // sfHookName: free // ltHookDefinition is not reserved because it is an unowned object, // rather the uploader is billed via fee according to the following: diff --git a/src/xrpld/app/tx/detail/SetHook.h b/src/xrpld/app/tx/detail/SetHook.h index 8aa53b54c..db67715e4 100644 --- a/src/xrpld/app/tx/detail/SetHook.h +++ b/src/xrpld/app/tx/detail/SetHook.h @@ -91,6 +91,9 @@ public: static HookSetValidation validateHookSetEntry(SetHookCtx& ctx, STObject const& hookSetObj); + static bool + validateHookName(Blob const& name, beast::Journal const& j); + static uint32_t computeHookReserve(STObject const& hookObj); diff --git a/src/xrpld/app/tx/detail/SetOracle.cpp b/src/xrpld/app/tx/detail/SetOracle.cpp index 055143cc6..2f5a72d14 100644 --- a/src/xrpld/app/tx/detail/SetOracle.cpp +++ b/src/xrpld/app/tx/detail/SetOracle.cpp @@ -209,6 +209,17 @@ SetOracle::doApply() { auto const oracleID = keylet::oracle(account_, ctx_.tx[sfOracleDocumentID]); + auto populatePriceData = [](STObject& priceData, STObject const& entry) { + setPriceDataInnerObjTemplate(priceData); + priceData.setFieldCurrency( + sfBaseAsset, entry.getFieldCurrency(sfBaseAsset)); + priceData.setFieldCurrency( + sfQuoteAsset, entry.getFieldCurrency(sfQuoteAsset)); + priceData.setFieldU64(sfAssetPrice, entry.getFieldU64(sfAssetPrice)); + if (entry.isFieldPresent(sfScale)) + priceData.setFieldU8(sfScale, entry.getFieldU8(sfScale)); + }; + if (auto sle = ctx_.view().peek(oracleID)) { // update @@ -249,15 +260,7 @@ SetOracle::doApply() { // add a token pair with the price STObject priceData{sfPriceData}; - setPriceDataInnerObjTemplate(priceData); - priceData.setFieldCurrency( - sfBaseAsset, entry.getFieldCurrency(sfBaseAsset)); - priceData.setFieldCurrency( - sfQuoteAsset, entry.getFieldCurrency(sfQuoteAsset)); - priceData.setFieldU64( - sfAssetPrice, entry.getFieldU64(sfAssetPrice)); - if (entry.isFieldPresent(sfScale)) - priceData.setFieldU8(sfScale, entry.getFieldU8(sfScale)); + populatePriceData(priceData, entry); pairs.emplace(key, std::move(priceData)); } } @@ -285,7 +288,19 @@ SetOracle::doApply() sle->setFieldVL(sfProvider, ctx_.tx[sfProvider]); if (ctx_.tx.isFieldPresent(sfURI)) sle->setFieldVL(sfURI, ctx_.tx[sfURI]); - auto const& series = ctx_.tx.getFieldArray(sfPriceDataSeries); + + STArray series; + std::map, STObject> pairs; + for (auto const& entry : ctx_.tx.getFieldArray(sfPriceDataSeries)) + { + auto const key = tokenPairKey(entry); + STObject priceData{sfPriceData}; + populatePriceData(priceData, entry); + pairs.emplace(key, std::move(priceData)); + } + for (auto const& iter : pairs) + series.push_back(std::move(iter.second)); + sle->setFieldArray(sfPriceDataSeries, series); sle->setFieldVL(sfAssetClass, ctx_.tx[sfAssetClass]); sle->setFieldU32(sfLastUpdateTime, ctx_.tx[sfLastUpdateTime]); diff --git a/src/xrpld/app/tx/detail/Transactor.cpp b/src/xrpld/app/tx/detail/Transactor.cpp index 24d999bfc..5ffe29407 100644 --- a/src/xrpld/app/tx/detail/Transactor.cpp +++ b/src/xrpld/app/tx/detail/Transactor.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -148,6 +149,16 @@ preflight1(PreflightContext const& ctx) } } + if (ctx.tx.isFieldPresent(sfHookName)) + { + if (!ctx.rules.enabled(featureHooks) || + !ctx.rules.enabled(featureNamedHooks)) + return temMALFORMED; + + if (!SetHook::validateHookName(ctx.tx.getFieldVL(sfHookName), ctx.j)) + return temMALFORMED; + } + auto const spk = ctx.tx.getSigningPubKey(); if (!spk.empty() && !publicKeyType(makeSlice(spk))) @@ -266,8 +277,24 @@ Transactor::calculateHookChainFee( // at the same ledger the fee calculation for it can no longer occur if (!hookDef) { + // LCOV_EXCL_START printf("calculateHookChainFee edge case\n"); continue; + // LCOV_EXCL_STOP + } + + std::optional requiredHookName; + if (hookObj.isFieldPresent(sfHookName) && + hookObj.getFieldVL(sfHookName).size() > 0) + requiredHookName = hookObj.getFieldVL(sfHookName); + + if (requiredHookName) + { + // need to specify same hook name in the transaction + if (!tx.isFieldPresent(sfHookName)) + continue; + if (*requiredHookName != tx.getFieldVL(sfHookName)) + continue; } uint32_t flags = 0; @@ -1311,16 +1338,34 @@ Transactor::executeHookChain( if (hookSkips.find(hookHash) != hookSkips.end()) { + // LCOV_EXCL_START JLOG(j_.trace()) << "HookInfo: Skipping " << hookHash; continue; + // LCOV_EXCL_STOP } auto const& hookDef = ctx_.view().peek(keylet::hookDefinition(hookHash)); if (!hookDef) { + // LCOV_EXCL_START JLOG(j_.warn()) << "HookError[]: Failure: hook def missing (send)"; continue; + // LCOV_EXCL_STOP + } + + std::optional requiredHookName; + if (hookObj.isFieldPresent(sfHookName) && + hookObj.getFieldVL(sfHookName).size() > 0) + requiredHookName = hookObj.getFieldVL(sfHookName); + + if (requiredHookName) + { + // need to specify same hook name in the transaction + if (!ctx_.tx.isFieldPresent(sfHookName)) + continue; + if (*requiredHookName != ctx_.tx.getFieldVL(sfHookName)) + continue; } // check if the hook can fire @@ -2151,7 +2196,8 @@ Transactor::operator()() bool const has240819 = view().rules().enabled(fix240819); bool const has240911 = view().rules().enabled(fix240911); - + bool const hasIOURewardClaim = + view().rules().enabled(featureIOURewardClaim); auto const& sfRewardFields = *(ripple::SField::knownCodeToField.at(917511 - has240819)); @@ -2161,11 +2207,92 @@ Transactor::operator()() SField const& metaType = node.getFName(); uint16_t nodeType = node.getFieldU16(sfLedgerEntryType); - // we only care about ltACCOUNT_ROOT objects being modified or - // created - if (nodeType != ltACCOUNT_ROOT || metaType == sfDeletedNode) + // we only care about ltACCOUNT_ROOT and ltRIPPLE_STATE objects + // being modified or created + if ((nodeType != ltACCOUNT_ROOT && nodeType != ltRIPPLE_STATE) || + metaType == sfDeletedNode) continue; + // ltRippleState + if (nodeType == ltRIPPLE_STATE) + { + if (!hasIOURewardClaim) + continue; + + if (!node.isFieldPresent(sfPreviousFields) || + !node.isFieldPresent(sfLedgerIndex)) + continue; + auto sle = view().peek( + Keylet{ltRIPPLE_STATE, node.getFieldH256(sfLedgerIndex)}); + if (!sle) + continue; + STObject& previousFields = (const_cast(node)) + .getField(sfPreviousFields) + .downcast(); + if (!previousFields.isFieldPresent(sfBalance)) + continue; + + auto balance = previousFields.getFieldAmount(sfBalance); + + if (balance.native()) + continue; + + SField const* sfRewardFields[] = {&sfLowReward, &sfHighReward}; + for (auto const* sfRewardFieldPtr : sfRewardFields) + { + auto const& sfRewardField = *sfRewardFieldPtr; + + if (!sle->isFieldPresent(sfRewardField)) + continue; + + auto balance_ = balance; + if (sfRewardField == sfHighReward) + balance_.negate(); + + if (balance_.negative()) + balance_.clear(); + + auto& reward = sle->peekFieldObject(sfRewardField); + uint32_t lgrLast = reward.getFieldU32(sfRewardLgrLast); + uint32_t lgrElapsed = lgrCur - lgrLast; + + // update even in cases such as overflow or underflow. + reward.setFieldU32(sfRewardLgrLast, lgrCur); + + // overflow safety + if (lgrElapsed > lgrCur || lgrElapsed == 0) + continue; + + auto accum = + reward.getFieldAmount(sfTrustLineRewardAccumulator); + + STAmount accumNew; + try + { + accumNew = accum + + multiply(balance_, + STAmount(((uint64_t)lgrElapsed)), + balance_.issue()); + } + catch (std::exception const&) + { + // Overflow detected, skip this reward calculation + continue; + } + + // check for overflow(<) and underflow(=) + if (accumNew <= accum) + continue; + + reward.setFieldAmount( + sfTrustLineRewardAccumulator, accumNew); + } + + view().update(sle); + continue; + } + + // ltAccountRoot if (!node.isFieldPresent(sfRewardFields) || !node.isFieldPresent(sfLedgerIndex)) continue; diff --git a/src/xrpld/core/detail/Config.cpp b/src/xrpld/core/detail/Config.cpp index 34ae49c35..46f15a664 100644 --- a/src/xrpld/core/detail/Config.cpp +++ b/src/xrpld/core/detail/Config.cpp @@ -1077,6 +1077,57 @@ Config::loadFromString(std::string const& fileContents) } } } + + // DatabasePinned validation (applies to all modes including standalone) + auto db_section = section(ConfigSection::nodeDatabase()); + if (db_section.exists("pinned_type")) + { + // Ensure base type is specified + if (!db_section.exists("type")) + { + Throw( + "type must be specified when using pinned_type"); + } + + // Ensure pinned_path is specified for filesystem-backed types. + // In-memory backends (rwdb) don't need a path. + auto pinnedTypeForPath = get(db_section, "pinned_type", ""); + boost::algorithm::to_lower(pinnedTypeForPath); + bool const needsPath = + pinnedTypeForPath != "rwdb" && pinnedTypeForPath != "memory"; + if (needsPath && !db_section.exists("pinned_path")) + { + Throw( + "pinned_path is required when pinned_type is set"); + } + + // Ensure online_delete is set (pinning without deletion is + // redundant) + if (auto delete_interval = get(db_section, "online_delete", 0); + delete_interval == 0) + { + Throw( + "pinned_type requires online_delete (pinning without " + "deletion is redundant)"); + } + + // In non-standalone mode, require a durable backend. + // Standalone mode is unrestricted (tests use rwdb, etc). + if (!RUN_STANDALONE) + { + auto pinnedType = get(db_section, "pinned_type", ""); + boost::algorithm::to_lower(pinnedType); + if (pinnedType != "nudb" && pinnedType != "rocksdb") + { + Throw( + "pinned_type must be a durable backend (NuDB or " + "RocksDB), got '" + + pinnedType + + "'. Non-durable backends lose pinned data on " + "restart."); + } + } + } } boost::filesystem::path diff --git a/src/xrpld/ledger/detail/View.cpp b/src/xrpld/ledger/detail/View.cpp index 615830376..8a7c65141 100644 --- a/src/xrpld/ledger/detail/View.cpp +++ b/src/xrpld/ledger/detail/View.cpp @@ -375,10 +375,12 @@ accountHolds( // if something bad does happen the trustline acts as a frozen line. if (spendableBalance < beast::zero || spendableBalance > amount) { + // LCOV_EXCL_START JLOG(j.error()) << "SpendableBalance has illegal value in accountHolds " << spendableBalance; amount.clear(Issue{currency, issuer}); + // LCOV_EXCL_STOP } else amount = spendableBalance; diff --git a/src/xrpld/nodestore/NodeObject.h b/src/xrpld/nodestore/NodeObject.h index f5a1fdb07..c88e99d9a 100644 --- a/src/xrpld/nodestore/NodeObject.h +++ b/src/xrpld/nodestore/NodeObject.h @@ -34,9 +34,43 @@ enum NodeObjectType : std::uint32_t { hotLEDGER = 1, hotACCOUNT_NODE = 3, hotTRANSACTION_NODE = 4, - hotDUMMY = 512 // an invalid or missing object + hotDUMMY = 512, // an invalid or missing object + + // Uncached variants - these bypass the cache when stored. + // These are routing-only values that MUST be reduced to their hot + // equivalents before serialization (on-disk format is a single byte). + pinnedACCOUNT_NODE = 1003, + pinnedTRANSACTION_NODE = 1004, + pinnedLEDGER = 1005 }; +/** Returns true if the type is a pinned (routing-only) variant. */ +inline bool +isPinnedType(NodeObjectType type) +{ + return type == pinnedACCOUNT_NODE || type == pinnedTRANSACTION_NODE || + type == pinnedLEDGER; +} + +/** Map pinned types back to their serializable hot equivalents. + Returns the type unchanged if it is not a pinned variant. +*/ +inline NodeObjectType +toHotType(NodeObjectType type) +{ + switch (type) + { + case pinnedACCOUNT_NODE: + return hotACCOUNT_NODE; + case pinnedTRANSACTION_NODE: + return hotTRANSACTION_NODE; + case pinnedLEDGER: + return hotLEDGER; + default: + return type; + } +} + /** A simple object that the Ledger uses to store entries. NodeObjects are comprised of a type, a hash, and a blob. They can be uniquely identified by the hash, which is a half-SHA512 of diff --git a/src/xrpld/nodestore/detail/DatabaseNodeImp.cpp b/src/xrpld/nodestore/detail/DatabaseNodeImp.cpp index 85e5d3c0d..0264fc17c 100644 --- a/src/xrpld/nodestore/detail/DatabaseNodeImp.cpp +++ b/src/xrpld/nodestore/detail/DatabaseNodeImp.cpp @@ -32,9 +32,16 @@ DatabaseNodeImp::store( { storeStats(1, data.size()); + // Pinned types bypass the cache and get reduced to hot equivalents + bool skipCache = isPinnedType(type); + if (skipCache) + type = toHotType(type); + auto obj = NodeObject::createObject(type, std::move(data), hash); backend_->store(obj); - if (cache_) + + // Only add to cache if it's not an uncached type + if (cache_ && !skipCache) { // After the store, replace a negative cache entry if there is one cache_->canonicalize( diff --git a/src/xrpld/nodestore/detail/DatabaseNodeImp.h b/src/xrpld/nodestore/detail/DatabaseNodeImp.h index 881d8a67c..49074110d 100644 --- a/src/xrpld/nodestore/detail/DatabaseNodeImp.h +++ b/src/xrpld/nodestore/detail/DatabaseNodeImp.h @@ -26,7 +26,6 @@ namespace ripple { namespace NodeStore { - class DatabaseNodeImp : public Database { public: diff --git a/src/xrpld/nodestore/detail/DatabasePinnedImp.cpp b/src/xrpld/nodestore/detail/DatabasePinnedImp.cpp new file mode 100644 index 000000000..040554b1b --- /dev/null +++ b/src/xrpld/nodestore/detail/DatabasePinnedImp.cpp @@ -0,0 +1,272 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2025 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include +#include +#include + +namespace ripple { +namespace NodeStore { + +DatabasePinnedImp::DatabasePinnedImp( + Application& app, + Scheduler& scheduler, + int readThreads, + std::shared_ptr writableBackend, + std::shared_ptr archiveBackend, + std::shared_ptr persistent, + Section const& config, + beast::Journal j) + : DatabaseRotating(scheduler, readThreads, config, j) + , app_(app) + , rotating_( + app, + scheduler, + readThreads, + std::move(writableBackend), + std::move(archiveBackend), + config, + j) + , persistent_(std::move(persistent)) +{ + // Update fdRequired to include all backends + fdRequired_ = rotating_.fdRequired(); + if (persistent_) + fdRequired_ += persistent_->fdRequired(); +} + +void +DatabasePinnedImp::store( + NodeObjectType type, + Blob&& data, + uint256 const& hash, + std::uint32_t ledgerSeq) +{ + // Route based on type + if (isPinnedType(type)) + { + // Reduce to serializable hot type before storage + type = toHotType(type); + + // Pinned types go to persistent storage + auto count = ++pinnedStoreCount_; + if (count % 1000 == 0) + { + JLOG(j_.trace()) + << "Pinned stores: " << count << " (type=" << type << ")"; + } + + auto nObj = NodeObject::createObject(type, std::move(data), hash); + persistent_->store(nObj); + storeStats(1, nObj->getData().size()); + } + else + { + // Hot types go through rotating storage + auto count = ++hotStoreCount_; + if (count % 10000 == 0) + { + JLOG(j_.trace()) + << "Hot stores: " << count << " (type=" << type << ")"; + } + rotating_.store(type, std::move(data), hash, ledgerSeq); + } +} + +std::shared_ptr +DatabasePinnedImp::fetchNodeObject( + uint256 const& hash, + std::uint32_t ledgerSeq, + FetchReport& fetchReport, + bool duplicate) +{ + // Optimization: Use ledgerSeq to predict backend order + // Most callers (13/18) provide reliable ledgerSeq: SHAMap node fetches, + // ledger headers, tx metadata, ledger copying operations. For these cases, + // checking the likely backend first reduces unnecessary lookups. + // When ledgerSeq=0 (unknown), falls back to default order. + // IMPORTANT: Always checks both backends for correctness. + if (ledgerSeq != 0) + { + refreshPinnedRangesCache(); + + if (likelyPinned(ledgerSeq)) + { + // Try persistent backend first (pinned data) + if (auto obj = tryPersistent(hash, fetchReport)) + return obj; + + // Fallback: try rotating backends + return rotating_.fetchNodeObject( + hash, ledgerSeq, fetchReport, duplicate); + } + else + { + // Not pinned: fall through to default path below + } + } + + // Default path: try rotating backends first (hot data) + if (auto obj = + rotating_.fetchNodeObject(hash, ledgerSeq, fetchReport, duplicate)) + return obj; + + // Fallback: try persistent backend (pinned data) + return tryPersistent(hash, fetchReport); +} + +void +DatabasePinnedImp::rotate( + std::unique_ptr&& newBackend, + std::function const& f) +{ + // Delegate to the inner rotating database. Pinned data lives in + // persistent_ and is never touched by rotation — only hot data + // in rotating_ participates in the rotate/archive/delete cycle. + rotating_.rotate(std::move(newBackend), f); +} + +std::string +DatabasePinnedImp::getName() const +{ + return "Pinned:" + rotating_.getName() + "+" + persistent_->getName(); +} + +std::int32_t +DatabasePinnedImp::getWriteLoad() const +{ + return rotating_.getWriteLoad() + persistent_->getWriteLoad(); +} + +void +DatabasePinnedImp::importDatabase(Database& source) +{ + // Import is not supported with DatabasePinned. The dual-backend + // routing relies on knowing which ledger ranges are pinned, which + // is only possible with catalogue packs where the ranges are known + // quantities. A generic import would need to scan for ledger + // headers and walk SHAMap trees to determine routing — not + // impossible, but not currently implemented. + Throw( + "--import is not supported when [node_db] pinned_type is configured. " + "DatabasePinned uses separate rotating and persistent backends, and a " + "generic node import cannot determine pinned ledger ranges or update " + "state.db pinned range metadata. Use catalogue_load for pinned " + "history import, or run --import with a non-pinned node store."); +} + +bool +DatabasePinnedImp::isSameDB(std::uint32_t s1, std::uint32_t s2) +{ + // Used by async read threads to determine if a fetched object can + // be reused for requests at different sequences. Since rotating and + // persistent are both part of the same logical database (same hash + // space), objects are always reusable regardless of which backend + // they came from. + return true; +} + +void +DatabasePinnedImp::sync() +{ + rotating_.sync(); + persistent_->sync(); +} + +bool +DatabasePinnedImp::storeLedger(std::shared_ptr const& srcLedger) +{ + // Store to persistent since ledger headers should be pinned + return Database::storeLedger(*srcLedger, persistent_); +} + +void +DatabasePinnedImp::sweep() +{ + // Delegate to rotating - persistent has no cache + rotating_.sweep(); +} + +void +DatabasePinnedImp::for_each(std::function)> f) +{ + // Visit both rotating and persistent backends + rotating_.for_each(f); + persistent_->for_each(f); +} + +void +DatabasePinnedImp::refreshPinnedRangesCache() const +{ + using namespace std::chrono; + + auto now = steady_clock::now(); + auto lastRefreshTime = + steady_clock::time_point(steady_clock::duration(lastRefresh_.load())); + + // Only refresh if cache is stale (>1 second old) + if (now - lastRefreshTime < seconds(1)) + return; + + // Create new RangeSet snapshot from LedgerMaster + auto newRanges = std::make_shared>( + app_.getLedgerMaster().getPinnedLedgersRangeSet()); + + // Atomic swap - all concurrent readers see old or new, never torn state + std::atomic_store(&cachedPinnedRanges_, newRanges); + lastRefresh_.store(now.time_since_epoch().count()); +} + +bool +DatabasePinnedImp::likelyPinned(std::uint32_t ledgerSeq) const +{ + auto ranges = std::atomic_load(&cachedPinnedRanges_); + return ranges && boost::icl::contains(*ranges, ledgerSeq); +} + +std::shared_ptr +DatabasePinnedImp::tryPersistent(uint256 const& hash, FetchReport& fetchReport) +{ + std::shared_ptr nodeObject; + Status status; + try + { + status = persistent_->fetch(hash.data(), &nodeObject); + } + catch (std::exception const& e) + { + JLOG(j_.fatal()) << "Exception fetching from persistent: " << e.what(); + Rethrow(); + } + + if (status == ok && nodeObject) + { + fetchReport.wasFound = true; + // Note: We do NOT copy pinned data to rotating storage even if + // duplicate=true. Pinned data stays in persistent storage. + } + return nodeObject; +} + +} // namespace NodeStore +} // namespace ripple diff --git a/src/xrpld/nodestore/detail/DatabasePinnedImp.h b/src/xrpld/nodestore/detail/DatabasePinnedImp.h new file mode 100644 index 000000000..107b20843 --- /dev/null +++ b/src/xrpld/nodestore/detail/DatabasePinnedImp.h @@ -0,0 +1,154 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2025 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_DATABASEPINNEDIMP_H_INCLUDED +#define RIPPLE_NODESTORE_DATABASEPINNEDIMP_H_INCLUDED + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ripple { +namespace NodeStore { + +/** + * DatabasePinned routes nodes to either rotating memory storage (via + * DatabaseRotatingImp) or persistent storage (NuDB) based on NodeObjectType + * values. + * + * Uses composition: DatabaseRotatingImp handles all rotation logic for hot + * nodes, while this class adds a persistent layer for pinned nodes. + * + * Pinned types (pinnedACCOUNT_NODE, pinnedTRANSACTION_NODE, pinnedLEDGER) go to + * persistent storage and stay forever. Hot types go through the normal + * rotation. + */ +class DatabasePinnedImp : public DatabaseRotating +{ +private: + Application& app_; // For accessing LedgerMaster's pinned ranges + DatabaseRotatingImp rotating_; // Handles rotation for hot nodes + std::shared_ptr persistent_; // NuDB for pinned nodes + + // Debug counters for store routing + std::atomic pinnedStoreCount_{0}; + std::atomic hotStoreCount_{0}; + + // Lock-free cached pinned ranges for backend selection hint + // Use std::atomic_load/store free functions for thread-safe access + mutable std::shared_ptr> cachedPinnedRanges_; + mutable std::atomic + lastRefresh_{0}; + +public: + static constexpr auto JournalName = "DatabasePinned"; + + DatabasePinnedImp( + Application& app, + Scheduler& scheduler, + int readThreads, + std::shared_ptr writableBackend, + std::shared_ptr archiveBackend, + std::shared_ptr persistent, + Section const& config, + beast::Journal j); + + ~DatabasePinnedImp() override + { + stop(); + } + + // Total count of store() calls routed to the persistent backend + // (pinned types). Exposed for tests and diagnostics. + uint64_t + pinnedStoreCount() const + { + return pinnedStoreCount_.load(std::memory_order_relaxed); + } + + // Total count of store() calls routed to the rotating backend + // (hot types). Exposed for tests and diagnostics. + uint64_t + hotStoreCount() const + { + return hotStoreCount_.load(std::memory_order_relaxed); + } + + // DatabaseRotating interface - delegates to rotating_ + void + rotate( + std::unique_ptr&& newBackend, + std::function const& f) override; + + // Database interface implementation + std::string + getName() const override; + std::int32_t + getWriteLoad() const override; + void + importDatabase(Database& source) override; + bool + isSameDB(std::uint32_t s1, std::uint32_t s2) override; + void + store( + NodeObjectType type, + Blob&& data, + uint256 const& hash, + std::uint32_t ledgerSeq) override; + void + sync() override; + bool + storeLedger(std::shared_ptr const& srcLedger) override; + void + sweep() override; + +private: + std::shared_ptr + fetchNodeObject( + uint256 const& hash, + std::uint32_t ledgerSeq, + FetchReport& fetchReport, + bool duplicate) override; + + // Helper methods for backend selection optimization + void + refreshPinnedRangesCache() const; + + bool + likelyPinned(std::uint32_t ledgerSeq) const; + + std::shared_ptr + tryPersistent(uint256 const& hash, FetchReport& fetchReport); + + void + for_each(std::function)> f) override; +}; + +} // namespace NodeStore +} // namespace ripple + +#endif \ No newline at end of file diff --git a/src/xrpld/nodestore/detail/DatabaseRotatingImp.cpp b/src/xrpld/nodestore/detail/DatabaseRotatingImp.cpp index 1499976d0..e194e29a9 100644 --- a/src/xrpld/nodestore/detail/DatabaseRotatingImp.cpp +++ b/src/xrpld/nodestore/detail/DatabaseRotatingImp.cpp @@ -61,57 +61,7 @@ DatabaseRotatingImp::rotate( { std::lock_guard lock(mutex_); - // Before rotating, ensure all pinned ledgers are in the writable - // backend - JLOG(j_.info()) - << "Ensuring pinned ledgers are preserved before backend rotation"; - - // Use a lambda to handle the preservation of pinned ledgers - auto ensurePinnedLedgersInWritable = [this]() { - // Get list of pinned ledgers - auto pinnedLedgers = - app_.getLedgerMaster().getPinnedLedgersRangeSet(); - - for (auto const& range : pinnedLedgers) - { - for (auto seq = range.lower(); seq <= range.upper(); ++seq) - { - uint256 hash = app_.getLedgerMaster().getHashBySeq(seq); - if (hash.isZero()) - continue; - - // Try to load the ledger - auto ledger = app_.getLedgerMaster().getLedgerByHash(hash); - if (ledger && ledger->isImmutable()) - { - // If we have the ledger, store it in the writable - // backend - JLOG(j_.debug()) << "Ensuring pinned ledger " << seq - << " is in writable backend"; - // TQ: TODO: check this - Database::storeLedger(*ledger, writableBackend_); - } - else - { - // If we don't have the ledger in memory, try to fetch - // its objects directly - JLOG(j_.debug()) - << "Attempting to copy pinned ledger " << seq - << " header to writable backend"; - std::shared_ptr headerObj; - Status status = - archiveBackend_->fetch(hash.data(), &headerObj); - if (status == ok && headerObj) - writableBackend_->store(headerObj); - } - } - } - }; - - // Execute the lambda - ensurePinnedLedgersInWritable(); - - // Now it's safe to mark the archive backend for deletion + // Mark the archive backend for deletion archiveBackend_->setDeletePath(); oldArchiveBackend = std::move(archiveBackend_); diff --git a/src/xrpld/nodestore/detail/DatabaseRotatingImp.h b/src/xrpld/nodestore/detail/DatabaseRotatingImp.h index 9052de07e..ba8b6754a 100644 --- a/src/xrpld/nodestore/detail/DatabaseRotatingImp.h +++ b/src/xrpld/nodestore/detail/DatabaseRotatingImp.h @@ -29,6 +29,9 @@ namespace NodeStore { class DatabaseRotatingImp : public DatabaseRotating { + friend class DatabasePinnedImp; // Allow DatabasePinnedImp to access + // private members + public: DatabaseRotatingImp() = delete; DatabaseRotatingImp(DatabaseRotatingImp const&) = delete; diff --git a/src/xrpld/rpc/detail/CatalogueStream.cpp b/src/xrpld/rpc/detail/CatalogueStream.cpp new file mode 100644 index 000000000..45cd1018a --- /dev/null +++ b/src/xrpld/rpc/detail/CatalogueStream.cpp @@ -0,0 +1,363 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012-2014 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include + +#include + +#include +#include +#include +#include +#include + +namespace ripple { +namespace RPC { + +namespace { + +std::size_t +serializeSHAMapToStream( + SHAMap const& shaMap, + CatalogueOutputStream& stream, + SHAMapNodeType nodeType, + std::optional> baseSHAMap = + std::nullopt) +{ + using StreamState = + std::pair; + + static std::mutex streamStateMutex; + static std::unordered_map streamBytesWritten; + + constexpr std::uint64_t flushThreshold = 256 * 1024 * 1024; + std::uint64_t localBytesWritten = 0; + + auto tryFlush = [](auto& s) { + if constexpr (requires(decltype(s) str) { str.flush(); }) + { + s.flush(); + } + }; + + { + std::lock_guard lock(streamStateMutex); + auto const now = std::chrono::steady_clock::now(); + + if (auto const it = + streamBytesWritten.find(static_cast(&stream)); + it != streamBytesWritten.end()) + { + localBytesWritten = it->second.first; + } + + for (auto it = streamBytesWritten.begin(); + it != streamBytesWritten.end();) + { + if (now - it->second.second > std::chrono::minutes(5)) + it = streamBytesWritten.erase(it); + else + ++it; + } + } + + auto serializeLeaf = [&stream, &localBytesWritten, &tryFlush]( + SHAMapItem const& item, + SHAMapNodeType type) -> bool { + stream.write(reinterpret_cast(&type), 1); + localBytesWritten += 1; + + auto const key = item.key(); + stream.write(reinterpret_cast(key.data()), 32); + localBytesWritten += 32; + + auto const data = item.slice(); + std::uint32_t size = data.size(); + stream.write(reinterpret_cast(&size), 4); + localBytesWritten += 4; + + stream.write(reinterpret_cast(data.data()), size); + localBytesWritten += size; + + if (localBytesWritten >= flushThreshold) + { + tryFlush(stream); + localBytesWritten = 0; + } + + return !stream.fail(); + }; + + auto serializeRemovedLeaf = + [&stream, &localBytesWritten, &tryFlush](uint256 const& key) -> bool { + auto t = SHAMapNodeType::tnREMOVE; + stream.write(reinterpret_cast(&t), 1); + localBytesWritten += 1; + + stream.write(reinterpret_cast(key.data()), 32); + localBytesWritten += 32; + + if (localBytesWritten >= flushThreshold) + { + tryFlush(stream); + localBytesWritten = 0; + } + + return !stream.fail(); + }; + + std::size_t nodeCount = 0; + + if (baseSHAMap) + { + auto const& baseMap = baseSHAMap->get(); + + if (shaMap.getHash() != baseMap.getHash()) + { + SHAMap::Delta differences; + + if (shaMap.compare( + baseMap, differences, std::numeric_limits::max())) + { + for (auto const& [key, deltaItem] : differences) + { + auto const& newItem = deltaItem.first; + auto const& oldItem = deltaItem.second; + + if (!oldItem && newItem) + { + if (serializeLeaf(*newItem, nodeType)) + ++nodeCount; + } + else if (oldItem && !newItem) + { + if (serializeRemovedLeaf(key)) + ++nodeCount; + } + else if ( + oldItem && newItem && + oldItem->slice() != newItem->slice()) + { + if (serializeLeaf(*newItem, nodeType)) + ++nodeCount; + } + } + + auto t = SHAMapNodeType::tnTERMINAL; + stream.write(reinterpret_cast(&t), 1); + localBytesWritten += 1; + + if (localBytesWritten >= flushThreshold) + { + tryFlush(stream); + localBytesWritten = 0; + } + + std::lock_guard lock(streamStateMutex); + streamBytesWritten[static_cast(&stream)] = { + localBytesWritten, std::chrono::steady_clock::now()}; + + return nodeCount; + } + } + else + { + return 0; + } + } + + for (auto const& item : shaMap) + { + if (serializeLeaf(item, nodeType)) + ++nodeCount; + } + + auto t = SHAMapNodeType::tnTERMINAL; + stream.write(reinterpret_cast(&t), 1); + localBytesWritten += 1; + + if (localBytesWritten >= flushThreshold) + { + tryFlush(stream); + localBytesWritten = 0; + } + + { + std::lock_guard lock(streamStateMutex); + streamBytesWritten[static_cast(&stream)] = { + localBytesWritten, std::chrono::steady_clock::now()}; + } + + return nodeCount; +} + +bool +deserializeSHAMapFromStream( + SHAMap& shaMap, + CatalogueInputStream& stream, + SHAMapNodeType nodeType, + NodeObjectType flushType, + bool allowRemoval, + beast::Journal const& j) +{ + try + { + auto deserializeLeaf = [&shaMap, &stream, &j, nodeType, allowRemoval]( + SHAMapNodeType& parsedType) -> bool { + stream.read(reinterpret_cast(&parsedType), 1); + + if (parsedType == SHAMapNodeType::tnTERMINAL) + return false; + + uint256 key; + std::uint32_t size{0}; + + stream.read(reinterpret_cast(key.data()), 32); + + if (stream.fail()) + { + JLOG(j.error()) + << "Deserialization: stream stopped unexpectedly " + << "while trying to read key of next entry"; + return false; + } + + if (parsedType == SHAMapNodeType::tnREMOVE) + { + if (!allowRemoval) + { + JLOG(j.error()) + << "Deserialization: unexpected removal in this map " + "type"; + return false; + } + + if (!shaMap.hasItem(key)) + { + JLOG(j.error()) + << "Deserialization: removal of key " << to_string(key) + << " but key is already absent."; + return false; + } + + shaMap.delItem(key); + return true; + } + + stream.read(reinterpret_cast(&size), 4); + + if (stream.fail()) + { + JLOG(j.error()) + << "Deserialization: stream stopped unexpectedly while " + << "trying to read size of data for key " << to_string(key); + return false; + } + + if (size > 1024 * 1024 * 1024) + { + JLOG(j.error()) + << "Deserialization: size of " << to_string(key) + << " is suspiciously large (" << size << " bytes), " + << "bailing."; + return false; + } + + std::vector data(size); + stream.read(reinterpret_cast(data.data()), size); + + if (stream.fail()) + { + JLOG(j.error()) + << "Deserialization: Unexpected EOF while reading data " + << "for " << to_string(key); + return false; + } + + auto item = make_shamapitem(key, makeSlice(data)); + if (shaMap.hasItem(key)) + return shaMap.updateGiveItem(nodeType, std::move(item)); + + return shaMap.addGiveItem(nodeType, std::move(item)); + }; + + auto lastParsed = SHAMapNodeType::tnREMOVE; + while (!stream.eof() && deserializeLeaf(lastParsed)) + ; + + if (lastParsed != SHAMapNodeType::tnTERMINAL) + { + JLOG(j.error()) + << "Deserialization: Unexpected EOF, terminal node not found."; + return false; + } + + shaMap.flushDirty(flushType); + return true; + } + catch (std::exception const& e) + { + JLOG(j.error()) << "Exception during deserialization: " << e.what(); + return false; + } +} + +} // namespace + +std::size_t +serializeStateMapToStream( + SHAMap const& stateMap, + CatalogueOutputStream& stream, + std::optional> prevStateMap) +{ + return serializeSHAMapToStream( + stateMap, stream, SHAMapNodeType::tnACCOUNT_STATE, prevStateMap); +} + +std::size_t +serializeTxMapToStream(SHAMap const& txMap, CatalogueOutputStream& stream) +{ + return serializeSHAMapToStream( + txMap, stream, SHAMapNodeType::tnTRANSACTION_MD); +} + +bool +deserializeStateMapFromStream( + SHAMap& stateMap, + CatalogueInputStream& stream, + NodeObjectType flushType, + beast::Journal const& j) +{ + return deserializeSHAMapFromStream( + stateMap, stream, SHAMapNodeType::tnACCOUNT_STATE, flushType, true, j); +} + +bool +deserializeTxMapFromStream( + SHAMap& txMap, + CatalogueInputStream& stream, + NodeObjectType flushType, + beast::Journal const& j) +{ + return deserializeSHAMapFromStream( + txMap, stream, SHAMapNodeType::tnTRANSACTION_MD, flushType, false, j); +} + +} // namespace RPC +} // namespace ripple diff --git a/src/xrpld/rpc/detail/CatalogueStream.h b/src/xrpld/rpc/detail/CatalogueStream.h new file mode 100644 index 000000000..7b19f6485 --- /dev/null +++ b/src/xrpld/rpc/detail/CatalogueStream.h @@ -0,0 +1,65 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012-2014 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_RPC_CATALOGUESTREAM_H_INCLUDED +#define RIPPLE_RPC_CATALOGUESTREAM_H_INCLUDED + +#include +#include +#include + +#include + +#include +#include + +namespace ripple { +namespace RPC { + +using CatalogueInputStream = boost::iostreams::filtering_istream; +using CatalogueOutputStream = boost::iostreams::filtering_ostream; + +std::size_t +serializeStateMapToStream( + SHAMap const& stateMap, + CatalogueOutputStream& stream, + std::optional> prevStateMap = + std::nullopt); + +std::size_t +serializeTxMapToStream(SHAMap const& txMap, CatalogueOutputStream& stream); + +bool +deserializeStateMapFromStream( + SHAMap& stateMap, + CatalogueInputStream& stream, + NodeObjectType flushType, + beast::Journal const& j); + +bool +deserializeTxMapFromStream( + SHAMap& txMap, + CatalogueInputStream& stream, + NodeObjectType flushType, + beast::Journal const& j); + +} // namespace RPC +} // namespace ripple + +#endif diff --git a/src/xrpld/rpc/handlers/Catalogue.cpp b/src/xrpld/rpc/handlers/Catalogue.cpp index 57a59550d..2b4d34b72 100644 --- a/src/xrpld/rpc/handlers/Catalogue.cpp +++ b/src/xrpld/rpc/handlers/Catalogue.cpp @@ -21,10 +21,13 @@ #include #include #include +#include +#include #include #include #include #include +#include #include #include #include @@ -678,10 +681,10 @@ doCatalogueCreate(RPC::JsonContext& context) return false; } - size_t stateNodesWritten = - ledger->stateMap().serializeToStream(*compStream, prevStateMap); + size_t stateNodesWritten = RPC::serializeStateMapToStream( + ledger->stateMap(), *compStream, prevStateMap); size_t txNodesWritten = - ledger->txMap().serializeToStream(*compStream); + RPC::serializeTxMapToStream(ledger->txMap(), *compStream); predictor.addLedger(info.seq, byteCounter.getBytesWritten()); @@ -910,6 +913,21 @@ doCatalogueLoad(RPC::JsonContext& context) } } opCleanup; + // Reject if DatabasePinned is not configured. Without it, loaded + // data lands in rotating storage and will be rotated away. + { + auto const& nscfg = + context.app.config().section(ConfigSection::nodeDatabase()); + if (!nscfg.exists("pinned_type")) + { + return rpcError( + rpcINVALID_PARAMS, + "catalogue_load requires [node_db] pinned_type to be " + "configured. Without it, loaded data will be lost on " + "the next database rotation."); + } + } + if (!context.params.isMember(jss::input_file)) return rpcError(rpcINVALID_PARAMS, "expected input_file"); @@ -971,6 +989,10 @@ doCatalogueLoad(RPC::JsonContext& context) uint8_t version = getCatalogueVersion(header.version); uint8_t compressionLevel = getCompressionLevel(header.version); + if (header.min_ledger > header.max_ledger) + return rpcError( + rpcINVALID_PARAMS, "catalogue min_ledger must be <= max_ledger"); + // Initialize status tracking { std::unique_lock writeLock(catalogueStatusMutex); @@ -1191,7 +1213,11 @@ doCatalogueLoad(RPC::JsonContext& context) ledger->setLedgerInfo(info); // Deserialize the complete state map from leaf nodes - if (!ledger->stateMap().deserializeFromStream(*decompStream)) + if (!RPC::deserializeStateMapFromStream( + ledger->stateMap(), + *decompStream, + pinnedACCOUNT_NODE, + context.j)) { JLOG(context.j.error()) << "Failed to deserialize base ledger state"; @@ -1217,7 +1243,11 @@ doCatalogueLoad(RPC::JsonContext& context) *snapshot); // Apply delta (only leaf-node changes) - if (!ledger->stateMap().deserializeFromStream(*decompStream)) + if (!RPC::deserializeStateMapFromStream( + ledger->stateMap(), + *decompStream, + pinnedACCOUNT_NODE, + context.j)) { JLOG(context.j.error()) << "Failed to apply delta to ledger " << info.seq; @@ -1226,17 +1256,17 @@ doCatalogueLoad(RPC::JsonContext& context) } // pull in the tx map - if (!ledger->txMap().deserializeFromStream(*decompStream)) + if (!RPC::deserializeTxMapFromStream( + ledger->txMap(), + *decompStream, + pinnedTRANSACTION_NODE, + context.j)) { JLOG(context.j.error()) << "Failed to apply delta to ledger " << info.seq; return rpcError(rpcINTERNAL, "Failed to apply ledger delta"); } - // Finalize the ledger - ledger->stateMap().flushDirty(hotACCOUNT_NODE); - ledger->txMap().flushDirty(hotTRANSACTION_NODE); - ledger->setAccepted( info.closeTime, info.closeTimeResolution, @@ -1260,12 +1290,84 @@ doCatalogueLoad(RPC::JsonContext& context) rpcINTERNAL, "Catalogue file contains a corrupted ledger."); } - // Save in database - pendSaveValidated(context.app, ledger, false, false); - - // Store in ledger master + // IMPORTANT: Mark as pinned BEFORE saving to database. + // This ensures isPinned() returns true when saveValidatedLedger + // checks, which: (a) routes to persistent backend via pinnedLEDGER + // type, and (b) skips AcceptedLedgerCache to avoid memory bloat. context.app.getLedgerMaster().storeLedger(ledger, true); + // Scope guard: un-pin if we exit without a successful save. + // Dismissed on success below. + bool saveDone = false; + auto unpinGuard = [&]() { + if (!saveDone) + context.app.getLedgerMaster().unpinLedger(ledger->info().seq); + }; + // Use a simple RAII wrapper to guarantee the guard runs + struct OnExit + { + std::function fn; + ~OnExit() + { + fn(); + } + } unpinOnExit{unpinGuard}; + + // Save in database - wait for completion to avoid memory bloat. + // Uses pendSaveValidated to respect job queue tuning on live + // servers (jtPUBOLDLEDGER), runs synchronously in standalone. + { + auto savePromise = std::make_shared>(); + auto saveFuture = savePromise->get_future(); + + bool queued = pendSaveValidated( + context.app, + ledger, + context.app.config().standalone(), + false, + [savePromise](bool success) { + savePromise->set_value(success); + }); + + if (!queued) + return rpcError(rpcINTERNAL, "Failed to save ledger"); + + // Wait for the async save to complete. Note: if the save + // job throws, the JobQueue has no exception handling — the + // process will std::terminate before we ever see a + // broken_promise here. The catch is defensive in case the + // job queue gains exception handling in the future. + bool saved = false; + try + { + saved = saveFuture.get(); + } + catch (std::future_error const& e) + { + JLOG(context.j.error()) + << "Save job for ledger " << ledger->info().seq + << " failed with exception (promise broken): " << e.what(); + return rpcError( + rpcINTERNAL, + "Save job crashed for ledger " + + std::to_string(ledger->info().seq)); + } + + if (!saved) + { + JLOG(context.j.error()) << "Failed to save ledger " + << ledger->info().seq << " to SQLite"; + return rpcError( + rpcINTERNAL, + "Failed to save ledger " + + std::to_string(ledger->info().seq) + + " to SQLite database"); + } + + // Save succeeded — dismiss the guard + saveDone = true; + } + if (info.seq == header.max_ledger && context.app.getLedgerMaster().getClosedLedger()->info().seq < info.seq) @@ -1277,6 +1379,20 @@ doCatalogueLoad(RPC::JsonContext& context) context.app.getLedgerMaster().setLedgerRangePresent( header.min_ledger, info.seq, true); + // Persist pinned ranges to state.db after every ledger. + // This is a single row UPDATE, so it's cheap. + // + // DURABILITY NOTE: There is a small crash window between the + // ledger save above and this setPinnedRanges call. If the + // process crashes in that window, the ledger data is safely + // in the persistent backend (always opened based on config, + // independent of state.db), and fetchNodeObject will still + // find it via the tryPersistent fallback. However, state.db + // won't record the pinned range, so mCompleteLedgers won't + // include those seqs until catalogue_load is re-run. + context.app.getSHAMapStore().setPinnedRanges( + context.app.getLedgerMaster().getPinnedLedgersRangeSet()); + // Store the ledger prevLedger = ledger; ledgersLoaded++; diff --git a/src/xrpld/rpc/handlers/ServerDefinitions.cpp b/src/xrpld/rpc/handlers/ServerDefinitions.cpp index 628311332..8c99a8de2 100644 --- a/src/xrpld/rpc/handlers/ServerDefinitions.cpp +++ b/src/xrpld/rpc/handlers/ServerDefinitions.cpp @@ -267,32 +267,6 @@ private: ret[jss::FIELDS][i++] = a; } - { - Json::Value a = Json::arrayValue; - a[0U] = "hash"; - Json::Value v = Json::objectValue; - v[jss::nth] = 257; - v[jss::isVLEncoded] = false; - v[jss::isSerialized] = false; - v[jss::isSigningField] = false; - v[jss::type] = "Hash256"; - a[1U] = v; - ret[jss::FIELDS][i++] = a; - } - - { - Json::Value a = Json::arrayValue; - a[0U] = "index"; - Json::Value v = Json::objectValue; - v[jss::nth] = 258; - v[jss::isVLEncoded] = false; - v[jss::isSerialized] = false; - v[jss::isSigningField] = false; - v[jss::type] = "Hash256"; - a[1U] = v; - ret[jss::FIELDS][i++] = a; - } - { Json::Value a = Json::arrayValue; a[0U] = "taker_gets_funded"; @@ -326,22 +300,23 @@ private: Json::Value innerObj = Json::objectValue; - uint32_t fc = code & 0xFFU; - uint32_t tc = code >> 16U; + uint32_t type = f->fieldType; - innerObj[jss::nth] = fc; + innerObj[jss::nth] = f->fieldValue; innerObj[jss::isVLEncoded] = - (tc == 7U /* Blob */ || tc == 8U /* AccountID */ || - tc == 19U /* Vector256 */); + (type == 7U /* Blob */ || type == 8U /* AccountID */ || + type == 19U /* Vector256 */); innerObj[jss::isSerialized] = - (tc < - 10000); /* TRANSACTION, LEDGER_ENTRY, VALIDATION, METADATA */ + (type < 10000 && f->fieldName != "hash" && + f->fieldName != + "index"); /* hash, index, TRANSACTION, LEDGER_ENTRY, + VALIDATION, METADATA */ innerObj[jss::isSigningField] = f->shouldInclude(false); - innerObj[jss::type] = type_map[tc]; + innerObj[jss::type] = type_map[type]; Json::Value innerArray = Json::arrayValue; innerArray[0U] = f->fieldName; diff --git a/src/xrpld/shamap/SHAMap.h b/src/xrpld/shamap/SHAMap.h index 2174181bf..e2c9bf4c0 100644 --- a/src/xrpld/shamap/SHAMap.h +++ b/src/xrpld/shamap/SHAMap.h @@ -367,35 +367,6 @@ public: void invariants() const; -public: - /** - * Serialize a SHAMap to a stream, optionally as a delta from another map - * Only leaf nodes are serialized since inner nodes can be reconstructed. - * - * @param stream The output stream to write to - * @param writtenNodes Set to track written node hashes to avoid duplicates - * @param baseSHAMap Optional base map to compute delta against - * @return Number of nodes written - */ - template - std::size_t - serializeToStream( - StreamType& stream, - std::optional> baseSHAMap = - std::nullopt) const; - - /** - * Deserialize a SHAMap from a stream - * Reconstructs the full tree from leaf nodes. - * - * @param stream The input stream to read from - * @param baseSHAMap Optional base map to apply deltas to - * @return True if deserialization succeeded - */ - template - bool - deserializeFromStream(StreamType& stream); - private: using SharedPtrNodeStack = std::stack, SHAMapNodeID>>; diff --git a/src/xrpld/shamap/detail/SHAMap.cpp b/src/xrpld/shamap/detail/SHAMap.cpp index a6b7563ca..2f50b5a8d 100644 --- a/src/xrpld/shamap/detail/SHAMap.cpp +++ b/src/xrpld/shamap/detail/SHAMap.cpp @@ -1297,395 +1297,4 @@ SHAMap::invariants() const node->invariants(true); } -template -std::size_t -SHAMap::serializeToStream( - StreamType& stream, - std::optional> baseSHAMap) const -{ - // Static map to track bytes written to streams - static std::mutex streamMapMutex; - static std::unordered_map< - void*, - std::pair> - streamBytesWritten; - - // Flush threshold: 256 MiB - constexpr uint64_t flushThreshold = 256 * 1024 * 1024; - - // Local byte counter for this stream - uint64_t localBytesWritten = 0; - - // Single lambda that uses compile-time check for flush method existence - auto tryFlush = [](auto& s) { - if constexpr (requires(decltype(s) str) { str.flush(); }) - { - s.flush(); - } - // No-op if flush doesn't exist - compiler will optimize this branch out - }; - - // Get the current bytes written from the global map (with lock) - { - std::lock_guard lock(streamMapMutex); - auto it = streamBytesWritten.find(static_cast(&stream)); - if (it != streamBytesWritten.end()) - { - localBytesWritten = it->second.first; - } - - // Random cleanup of old entries (while we have the lock) - if (!streamBytesWritten.empty()) - { - auto now = std::chrono::steady_clock::now(); - size_t randomIndex = std::rand() % streamBytesWritten.size(); - auto cleanupIt = std::next(streamBytesWritten.begin(), randomIndex); - - // If entry is older than 5 minutes, remove it - if (now - cleanupIt->second.second > std::chrono::minutes(5)) - { - streamBytesWritten.erase(cleanupIt); - } - } - } - - std::unordered_set> writtenNodes; - - if (!root_) - return 0; - - std::size_t nodeCount = 0; - - auto serializeLeaf = [&stream, &localBytesWritten, &tryFlush]( - SHAMapLeafNode const& node) -> bool { - // write the node type - auto t = node.getType(); - stream.write(reinterpret_cast(&t), 1); - localBytesWritten += 1; - - // write the key - auto const key = node.peekItem()->key(); - stream.write(reinterpret_cast(key.data()), 32); - localBytesWritten += 32; - - // write the data size - auto data = node.peekItem()->slice(); - uint32_t size = data.size(); - stream.write(reinterpret_cast(&size), 4); - localBytesWritten += 4; - - // write the data - stream.write(reinterpret_cast(data.data()), size); - localBytesWritten += size; - - // Check if we should flush without locking - if (localBytesWritten >= flushThreshold) - { - tryFlush(stream); - localBytesWritten = 0; - } - - return !stream.fail(); - }; - - auto serializeRemovedLeaf = - [&stream, &localBytesWritten, &tryFlush](uint256 const& key) -> bool { - // to indicate a node is removed it is written with a removal type - auto t = SHAMapNodeType::tnREMOVE; - stream.write(reinterpret_cast(&t), 1); - localBytesWritten += 1; - - // write the key - stream.write(reinterpret_cast(key.data()), 32); - localBytesWritten += 32; - - // Check if we should flush without locking - if (localBytesWritten >= flushThreshold) - { - tryFlush(stream); - localBytesWritten = 0; - } - - return !stream.fail(); - }; - - // If we're creating a delta, first compute the differences - if (baseSHAMap && baseSHAMap->get().root_) - { - const SHAMap& baseMap = baseSHAMap->get(); - - // Only compute delta if the maps are different - if (getHash() != baseMap.getHash()) - { - Delta differences; - - if (compare(baseMap, differences, std::numeric_limits::max())) - { - // Process each difference - for (auto const& [key, deltaItem] : differences) - { - auto const& newItem = deltaItem.first; - auto const& oldItem = deltaItem.second; - - if (!oldItem && newItem) - { - // Added item - SHAMapLeafNode* leaf = findKey(key); - if (leaf && serializeLeaf(*leaf)) - ++nodeCount; - } - else if (oldItem && !newItem) - { - // Removed item - if (serializeRemovedLeaf(key)) - ++nodeCount; - } - else if ( - oldItem && newItem && - oldItem->slice() != newItem->slice()) - { - // Modified item - SHAMapLeafNode* leaf = findKey(key); - if (leaf && serializeLeaf(*leaf)) - ++nodeCount; - } - } - - // write a terminal symbol to indicate the map stream has ended - auto t = SHAMapNodeType::tnTERMINAL; - stream.write(reinterpret_cast(&t), 1); - localBytesWritten += 1; - - // Check if we should flush without locking - if (localBytesWritten >= flushThreshold) - { - tryFlush(stream); - localBytesWritten = 0; - } - - // Update the global counter at the end (with lock) - { - std::lock_guard lock(streamMapMutex); - auto& streamData = - streamBytesWritten[static_cast(&stream)]; - streamData.first = localBytesWritten; - streamData.second = std::chrono::steady_clock::now(); - } - - return nodeCount; - } - } - else - { - // Maps are identical, nothing to write - return 0; - } - } - - // Otherwise walk the entire tree and serialize all leaf nodes - std::function walkTree = - [&](SHAMapTreeNode const& node, SHAMapNodeID const& nodeID) { - if (node.isLeaf()) - { - auto const& leaf = static_cast(node); - auto const& hash = leaf.getHash(); - - // Avoid duplicates - if (writtenNodes.insert(hash).second) - { - if (serializeLeaf(leaf)) - ++nodeCount; - } - return; - } - - // It's an inner node, process its children - auto const& inner = static_cast(node); - for (int i = 0; i < branchFactor; ++i) - { - if (!inner.isEmptyBranch(i)) - { - auto const& childHash = inner.getChildHash(i); - - // Skip already written nodes - if (writtenNodes.find(childHash) != writtenNodes.end()) - continue; - - auto childNode = - descendThrow(const_cast(&inner), i); - if (childNode) - { - SHAMapNodeID childID = nodeID.getChildNodeID(i); - walkTree(*childNode, childID); - } - } - } - }; - - // Start walking from root - walkTree(*root_, SHAMapNodeID()); - - // write a terminal symbol to indicate the map stream has ended - auto t = SHAMapNodeType::tnTERMINAL; - stream.write(reinterpret_cast(&t), 1); - localBytesWritten += 1; - - // Check if we should flush one last time without locking - if (localBytesWritten >= flushThreshold) - { - tryFlush(stream); - localBytesWritten = 0; - } - - // Update the global counter at the end (with lock) - { - std::lock_guard lock(streamMapMutex); - auto& streamData = streamBytesWritten[static_cast(&stream)]; - streamData.first = localBytesWritten; - streamData.second = std::chrono::steady_clock::now(); - } - - return nodeCount; -} - -template -bool -SHAMap::deserializeFromStream(StreamType& stream) -{ - try - { - JLOG(journal_.info()) << "Deserialization: Starting to deserialize " - "from stream"; - - if (state_ != SHAMapState::Modifying && state_ != SHAMapState::Synching) - return false; - - if (!root_) - root_ = std::make_shared(cowid_); - - // Define a lambda to deserialize a leaf node - auto deserializeLeaf = - [this, &stream](SHAMapNodeType& nodeType /* out */) -> bool { - stream.read(reinterpret_cast(&nodeType), 1); - - if (nodeType == SHAMapNodeType::tnTERMINAL) - { - // end of map - return false; - } - - uint256 key; - uint32_t size{0}; - - stream.read(reinterpret_cast(key.data()), 32); - - if (stream.fail()) - { - JLOG(journal_.error()) - << "Deserialization: stream stopped unexpectedly " - << "while trying to read key of next entry"; - return false; - } - - if (nodeType == SHAMapNodeType::tnREMOVE) - { - // deletion - if (!hasItem(key)) - { - JLOG(journal_.error()) - << "Deserialization: removal of key " << to_string(key) - << " but key is already absent."; - return false; - } - delItem(key); - return true; - } - - stream.read(reinterpret_cast(&size), 4); - - if (stream.fail()) - { - JLOG(journal_.error()) - << "Deserialization: stream stopped unexpectedly" - << " while trying to read size of data for key " - << to_string(key); - return false; - } - - if (size > 1024 * 1024 * 1024) - { - JLOG(journal_.error()) - << "Deserialization: size of " << to_string(key) - << " is suspiciously large (" << size - << " bytes), bailing."; - return false; - } - - std::vector data; - data.resize(size); - - stream.read(reinterpret_cast(data.data()), size); - if (stream.fail()) - { - JLOG(journal_.error()) - << "Deserialization: Unexpected EOF while reading data for " - << to_string(key); - return false; - } - - auto item = make_shamapitem(key, makeSlice(data)); - if (hasItem(key)) - return updateGiveItem(nodeType, std::move(item)); - - return addGiveItem(nodeType, std::move(item)); - }; - - SHAMapNodeType lastParsed; - while (!stream.eof() && deserializeLeaf(lastParsed)) - ; - - if (lastParsed != SHAMapNodeType::tnTERMINAL) - { - JLOG(journal_.error()) - << "Deserialization: Unexpected EOF, terminal node not found."; - return false; - } - - // Flush any dirty nodes and update hashes - flushDirty(hotUNKNOWN); - - return true; - } - catch (std::exception const& e) - { - JLOG(journal_.error()) - << "Exception during deserialization: " << e.what(); - return false; - } -} - -// explicit instantiation of templates for rpc::Catalogue - -using FilteringInputStream = boost::iostreams::filtering_stream< - boost::iostreams::input, - char, - std::char_traits, - std::allocator, - boost::iostreams::public_>; - -template bool -SHAMap::deserializeFromStream(FilteringInputStream&); - -using FilteringOutputStream = boost::iostreams::filtering_stream< - boost::iostreams::output, - char, - std::char_traits, - std::allocator, - boost::iostreams::public_>; - -template std::size_t -SHAMap::serializeToStream( - FilteringOutputStream&, - std::optional> baseSHAMap) const; - } // namespace ripple