Compare commits

..
Author SHA1 Message Date
tequ 86ac135c63 refactor: include transactor headers from transactions.macro
Partial cherry-pick of XRPLF/rippled#5590. Each TRANSACTION entry now
carries its transactor header behind `#if TRANSACTION_INCLUDE`, and
applySteps.cpp pulls them in by expanding the macro with
TRANSACTION_INCLUDE defined. Adding a transaction type no longer
requires touching the include list in applySteps.cpp.

ApplyContext.h (reached via Transactor.h) and XahauGenesis.h (unused
there) are dropped from applySteps.cpp.
2026-10-22 15:44:30 +09:00
Richard Holland 3dc41b2a59 ignore amendment cfg option (#900)
Use like so, on testnet:
[ignore_amendments]
HookOnV2
2026-10-17 09:19:39 +10:00
Richard Holland b9f2914c0e catalogue speed and memory fixes (#883) 2026-10-14 14:13:37 +10:00
tequ 023f55e283 fix: keep ledger_history in standalone mode for rwdb null node store (#893) 2026-10-14 12:30:51 +10:00
tequandRichard Holland d20e8ffb56 fix(hook): crash on the transaction after removing a HookName (#890)
* fix(hook): Read sfHookName via isFieldPresent instead of operator[]

SetHook removes a hook's name with makeFieldAbsent on an untemplated
sfHooks entry, which leaves an STI_NOTPRESENT placeholder for
sfHookName. STObject::at(~sfHookName) then fails the dynamic_cast,
dereferences the null template and asserts (null deref in release).

Every other optional field of an sfHooks entry is already read with
isFieldPresent/getField*, which treats the placeholder as absent.
Align the three sfHookName readers with that convention:

- Transactor::calculateHookChainFee
- Transactor::executeHookChain
- ClaimReward::preclaim (fix20260929 path)

hook::canHook now takes std::optional<Blob> so callers can pass the
copied value without a dangling Slice.

Add regression tests that remove a hook name with a HookName-only
SetHook update and then submit the next transaction (Invoke and
ClaimReward). Both crashed before this change.

* claude fixes

---------

Co-authored-by: Richard Holland <[email protected]>
2026-10-13 10:57:14 +10:00
Richard Holland c63ee8cc5f fix mkdir in release builder (#889) 2026-10-12 13:19:11 +10:00
49 changed files with 2733 additions and 469 deletions

No files matched your search

+1
View File
@@ -127,6 +127,7 @@ test.rpc > xrpl.basics
test.rpc > xrpld.app
test.rpc > xrpld.core
test.rpc > xrpld.net
test.rpc > xrpld.nodestore
test.rpc > xrpld.overlay
test.rpc > xrpld.rpc
test.rpc > xrpld.shamap
+167 -1
View File
@@ -24,11 +24,21 @@
/**
* TRANSACTION(tag, value, name, fields)
*
* To ease maintenance, you may replace any unneeded values with "..."
* e.g. #define TRANSACTION(tag, value, name, ...)
*
* You must define a transactor class in the `ripple` namespace named `name`,
* and include its header in `src/xrpld/app/tx/detail/applySteps.cpp`.
* and include its header alongside the TRANSACTOR definition using this
* format:
* #if TRANSACTION_INCLUDE
* # include <xrpld/app/tx/detail/HEADER.h>
* #endif
*/
/** This transaction type executes a payment. */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/Payment.h>
#endif
TRANSACTION(ttPAYMENT, 0, Payment, ({
{sfDestination, soeREQUIRED},
{sfAmount, soeREQUIRED, soeMPTSupported},
@@ -41,6 +51,9 @@ TRANSACTION(ttPAYMENT, 0, Payment, ({
}))
/** This transaction type creates an escrow object. */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/Escrow.h>
#endif
TRANSACTION(ttESCROW_CREATE, 1, EscrowCreate, ({
{sfDestination, soeREQUIRED},
{sfAmount, soeREQUIRED},
@@ -62,6 +75,9 @@ TRANSACTION(ttESCROW_FINISH, 2, EscrowFinish, ({
/** This transaction type adjusts various account settings. */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/SetAccount.h>
#endif
TRANSACTION(ttACCOUNT_SET, 3, AccountSet, ({
{sfEmailHash, soeOPTIONAL},
{sfWalletLocator, soeOPTIONAL},
@@ -85,6 +101,9 @@ TRANSACTION(ttESCROW_CANCEL, 4, EscrowCancel, ({
}))
/** This transaction type sets or clears an account's "regular key". */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/SetRegularKey.h>
#endif
TRANSACTION(ttREGULAR_KEY_SET, 5, SetRegularKey, ({
{sfRegularKey, soeOPTIONAL},
}))
@@ -92,6 +111,9 @@ TRANSACTION(ttREGULAR_KEY_SET, 5, SetRegularKey, ({
// 6 deprecated
/** This transaction type creates an offer to trade one asset for another. */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/CreateOffer.h>
#endif
TRANSACTION(ttOFFER_CREATE, 7, OfferCreate, ({
{sfTakerPays, soeREQUIRED},
{sfTakerGets, soeREQUIRED},
@@ -101,6 +123,9 @@ TRANSACTION(ttOFFER_CREATE, 7, OfferCreate, ({
}))
/** This transaction type cancels existing offers to trade one asset for another. */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/CancelOffer.h>
#endif
TRANSACTION(ttOFFER_CANCEL, 8, OfferCancel, ({
{sfOfferSequence, soeOPTIONAL},
{sfOfferID, soeOPTIONAL}, // keylet as alternative to offerseq
@@ -109,6 +134,9 @@ TRANSACTION(ttOFFER_CANCEL, 8, OfferCancel, ({
// 9 deprecated
/** This transaction type creates a new set of tickets. */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/CreateTicket.h>
#endif
TRANSACTION(ttTICKET_CREATE, 10, TicketCreate, ({
{sfTicketCount, soeREQUIRED},
}))
@@ -118,12 +146,18 @@ TRANSACTION(ttTICKET_CREATE, 10, TicketCreate, ({
/** This transaction type modifies the signer list associated with an account. */
// The SignerEntries are optional because a SignerList is deleted by
// setting the SignerQuorum to zero and omitting SignerEntries.
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/SetSignerList.h>
#endif
TRANSACTION(ttSIGNER_LIST_SET, 12, SignerListSet, ({
{sfSignerQuorum, soeREQUIRED},
{sfSignerEntries, soeOPTIONAL},
}))
/** This transaction type creates a new unidirectional XRP payment channel. */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/PayChan.h>
#endif
TRANSACTION(ttPAYCHAN_CREATE, 13, PaymentChannelCreate, ({
{sfDestination, soeREQUIRED},
{sfAmount, soeREQUIRED},
@@ -151,6 +185,9 @@ TRANSACTION(ttPAYCHAN_CLAIM, 15, PaymentChannelClaim, ({
}))
/** This transaction type creates a new check. */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/CreateCheck.h>
#endif
TRANSACTION(ttCHECK_CREATE, 16, CheckCreate, ({
{sfDestination, soeREQUIRED},
{sfSendMax, soeREQUIRED},
@@ -160,6 +197,9 @@ TRANSACTION(ttCHECK_CREATE, 16, CheckCreate, ({
}))
/** This transaction type cashes an existing check. */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/CashCheck.h>
#endif
TRANSACTION(ttCHECK_CASH, 17, CheckCash, ({
{sfCheckID, soeREQUIRED},
{sfAmount, soeOPTIONAL},
@@ -167,11 +207,17 @@ TRANSACTION(ttCHECK_CASH, 17, CheckCash, ({
}))
/** This transaction type cancels an existing check. */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/CancelCheck.h>
#endif
TRANSACTION(ttCHECK_CANCEL, 18, CheckCancel, ({
{sfCheckID, soeREQUIRED},
}))
/** This transaction type grants or revokes authorization to transfer funds. */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/DepositPreauth.h>
#endif
TRANSACTION(ttDEPOSIT_PREAUTH, 19, DepositPreauth, ({
{sfAuthorize, soeOPTIONAL},
{sfUnauthorize, soeOPTIONAL},
@@ -180,6 +226,9 @@ TRANSACTION(ttDEPOSIT_PREAUTH, 19, DepositPreauth, ({
}))
/** This transaction type modifies a trustline between two accounts. */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/SetTrust.h>
#endif
TRANSACTION(ttTRUST_SET, 20, TrustSet, ({
{sfLimitAmount, soeOPTIONAL},
{sfQualityIn, soeOPTIONAL},
@@ -187,6 +236,9 @@ TRANSACTION(ttTRUST_SET, 20, TrustSet, ({
}))
/** This transaction type deletes an existing account. */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/DeleteAccount.h>
#endif
TRANSACTION(ttACCOUNT_DELETE, 21, AccountDelete, ({
{sfDestination, soeREQUIRED},
{sfDestinationTag, soeOPTIONAL},
@@ -194,11 +246,17 @@ TRANSACTION(ttACCOUNT_DELETE, 21, AccountDelete, ({
}))
/** This transaction type installs a hook. */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/SetHook.h>
#endif
TRANSACTION(ttHOOK_SET, 22, SetHook, ({
{sfHooks, soeREQUIRED},
}))
/** This transaction mints a new NFT. */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/NFTokenMint.h>
#endif
TRANSACTION(ttNFTOKEN_MINT, 25, NFTokenMint, ({
{sfNFTokenTaxon, soeREQUIRED},
{sfTransferFee, soeOPTIONAL},
@@ -210,12 +268,18 @@ TRANSACTION(ttNFTOKEN_MINT, 25, NFTokenMint, ({
}))
/** This transaction burns (i.e. destroys) an existing NFT. */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/NFTokenBurn.h>
#endif
TRANSACTION(ttNFTOKEN_BURN, 26, NFTokenBurn, ({
{sfNFTokenID, soeREQUIRED},
{sfOwner, soeOPTIONAL},
}))
/** This transaction creates a new offer to buy or sell an NFT. */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/NFTokenCreateOffer.h>
#endif
TRANSACTION(ttNFTOKEN_CREATE_OFFER, 27, NFTokenCreateOffer, ({
{sfNFTokenID, soeREQUIRED},
{sfAmount, soeREQUIRED},
@@ -225,11 +289,17 @@ TRANSACTION(ttNFTOKEN_CREATE_OFFER, 27, NFTokenCreateOffer, ({
}))
/** This transaction cancels an existing offer to buy or sell an existing NFT. */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/NFTokenCancelOffer.h>
#endif
TRANSACTION(ttNFTOKEN_CANCEL_OFFER, 28, NFTokenCancelOffer, ({
{sfNFTokenOffers, soeREQUIRED},
}))
/** This transaction accepts an existing offer to buy or sell an existing NFT. */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/NFTokenAcceptOffer.h>
#endif
TRANSACTION(ttNFTOKEN_ACCEPT_OFFER, 29, NFTokenAcceptOffer, ({
{sfNFTokenBuyOffer, soeOPTIONAL},
{sfNFTokenSellOffer, soeOPTIONAL},
@@ -237,12 +307,18 @@ TRANSACTION(ttNFTOKEN_ACCEPT_OFFER, 29, NFTokenAcceptOffer, ({
}))
/** This transaction claws back issued tokens. */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/Clawback.h>
#endif
TRANSACTION(ttCLAWBACK, 30, Clawback, ({
{sfAmount, soeREQUIRED, soeMPTSupported},
{sfHolder, soeOPTIONAL},
}))
/** This transaction claws back tokens from an AMM pool. */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/AMMClawback.h>
#endif
TRANSACTION(ttAMM_CLAWBACK, 31, AMMClawback, ({
{sfHolder, soeREQUIRED},
{sfAsset, soeREQUIRED},
@@ -251,6 +327,9 @@ TRANSACTION(ttAMM_CLAWBACK, 31, AMMClawback, ({
}))
/** This transaction type creates an AMM instance */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/AMMCreate.h>
#endif
TRANSACTION(ttAMM_CREATE, 35, AMMCreate, ({
{sfAmount, soeREQUIRED},
{sfAmount2, soeREQUIRED},
@@ -258,6 +337,9 @@ TRANSACTION(ttAMM_CREATE, 35, AMMCreate, ({
}))
/** This transaction type deposits into an AMM instance */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/AMMDeposit.h>
#endif
TRANSACTION(ttAMM_DEPOSIT, 36, AMMDeposit, ({
{sfAsset, soeREQUIRED},
{sfAsset2, soeREQUIRED},
@@ -269,6 +351,9 @@ TRANSACTION(ttAMM_DEPOSIT, 36, AMMDeposit, ({
}))
/** This transaction type withdraws from an AMM instance */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/AMMWithdraw.h>
#endif
TRANSACTION(ttAMM_WITHDRAW, 37, AMMWithdraw, ({
{sfAsset, soeREQUIRED},
{sfAsset2, soeREQUIRED},
@@ -279,6 +364,9 @@ TRANSACTION(ttAMM_WITHDRAW, 37, AMMWithdraw, ({
}))
/** This transaction type votes for the trading fee */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/AMMVote.h>
#endif
TRANSACTION(ttAMM_VOTE, 38, AMMVote, ({
{sfAsset, soeREQUIRED},
{sfAsset2, soeREQUIRED},
@@ -286,6 +374,9 @@ TRANSACTION(ttAMM_VOTE, 38, AMMVote, ({
}))
/** This transaction type bids for the auction slot */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/AMMBid.h>
#endif
TRANSACTION(ttAMM_BID, 39, AMMBid, ({
{sfAsset, soeREQUIRED},
{sfAsset2, soeREQUIRED},
@@ -295,12 +386,18 @@ TRANSACTION(ttAMM_BID, 39, AMMBid, ({
}))
/** This transaction type deletes AMM in the empty state */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/AMMDelete.h>
#endif
TRANSACTION(ttAMM_DELETE, 40, AMMDelete, ({
{sfAsset, soeREQUIRED},
{sfAsset2, soeREQUIRED},
}))
/** This transaction mints/burns/buys/sells a URI TOKEN */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/URIToken.h>
#endif
TRANSACTION(ttURITOKEN_MINT, 45, URITokenMint, ({
{sfURI, soeREQUIRED},
{sfDigest, soeOPTIONAL},
@@ -328,6 +425,9 @@ TRANSACTION(ttURITOKEN_CANCEL_SELL_OFFER, 49, URITokenCancelSellOffer, ({
}))
/** This transactions creates a crosschain sequence number */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/XChainBridge.h>
#endif
TRANSACTION(ttXCHAIN_CREATE_CLAIM_ID, 50, XChainCreateClaimID, ({
{sfXChainBridge, soeREQUIRED},
{sfSignatureReward, soeREQUIRED},
@@ -407,6 +507,9 @@ TRANSACTION(ttXCHAIN_CREATE_BRIDGE, 57, XChainCreateBridge, ({
}))
/** This transaction type creates or updates a DID */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/DID.h>
#endif
TRANSACTION(ttDID_SET, 58, DIDSet, ({
{sfDIDDocument, soeOPTIONAL},
{sfURI, soeOPTIONAL},
@@ -417,6 +520,9 @@ TRANSACTION(ttDID_SET, 58, DIDSet, ({
TRANSACTION(ttDID_DELETE, 59, DIDDelete, ({}))
/** This transaction type creates an Oracle instance */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/SetOracle.h>
#endif
TRANSACTION(ttORACLE_SET, 60, OracleSet, ({
{sfOracleDocumentID, soeREQUIRED},
{sfProvider, soeOPTIONAL},
@@ -427,17 +533,26 @@ TRANSACTION(ttORACLE_SET, 60, OracleSet, ({
}))
/** This transaction type deletes an Oracle instance */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/DeleteOracle.h>
#endif
TRANSACTION(ttORACLE_DELETE, 61, OracleDelete, ({
{sfOracleDocumentID, soeREQUIRED},
}))
/** This transaction type fixes a problem in the ledger state */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/LedgerStateFix.h>
#endif
TRANSACTION(ttLEDGER_STATE_FIX, 62, LedgerStateFix, ({
{sfLedgerFixType, soeREQUIRED},
{sfOwner, soeOPTIONAL},
}))
/** This transaction type creates a MPTokensIssuance instance */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/MPTokenIssuanceCreate.h>
#endif
TRANSACTION(ttMPTOKEN_ISSUANCE_CREATE, 63, MPTokenIssuanceCreate, ({
{sfAssetScale, soeOPTIONAL},
{sfTransferFee, soeOPTIONAL},
@@ -446,23 +561,35 @@ TRANSACTION(ttMPTOKEN_ISSUANCE_CREATE, 63, MPTokenIssuanceCreate, ({
}))
/** This transaction type destroys a MPTokensIssuance instance */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/MPTokenIssuanceDestroy.h>
#endif
TRANSACTION(ttMPTOKEN_ISSUANCE_DESTROY, 64, MPTokenIssuanceDestroy, ({
{sfMPTokenIssuanceID, soeREQUIRED},
}))
/** This transaction type sets flags on a MPTokensIssuance or MPToken instance */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/MPTokenIssuanceSet.h>
#endif
TRANSACTION(ttMPTOKEN_ISSUANCE_SET, 65, MPTokenIssuanceSet, ({
{sfMPTokenIssuanceID, soeREQUIRED},
{sfHolder, soeOPTIONAL},
}))
/** This transaction type authorizes a MPToken instance */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/MPTokenAuthorize.h>
#endif
TRANSACTION(ttMPTOKEN_AUTHORIZE, 66, MPTokenAuthorize, ({
{sfMPTokenIssuanceID, soeREQUIRED},
{sfHolder, soeOPTIONAL},
}))
/** This transaction type create an Credential instance */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/Credentials.h>
#endif
TRANSACTION(ttCREDENTIAL_CREATE, 67, CredentialCreate, ({
{sfSubject, soeREQUIRED},
{sfCredentialType, soeREQUIRED},
@@ -484,6 +611,9 @@ TRANSACTION(ttCREDENTIAL_DELETE, 69, CredentialDelete, ({
}))
/** This transaction type modify a NFToken */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/NFTokenModify.h>
#endif
TRANSACTION(ttNFTOKEN_MODIFY, 70, NFTokenModify, ({
{sfNFTokenID, soeREQUIRED},
{sfOwner, soeOPTIONAL},
@@ -491,12 +621,18 @@ TRANSACTION(ttNFTOKEN_MODIFY, 70, NFTokenModify, ({
}))
/** This transaction type creates or modifies a Permissioned Domain */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/PermissionedDomainSet.h>
#endif
TRANSACTION(ttPERMISSIONED_DOMAIN_SET, 71, PermissionedDomainSet, ({
{sfDomainID, soeOPTIONAL},
{sfAcceptedCredentials, soeREQUIRED},
}))
/** This transaction type deletes a Permissioned Domain */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/PermissionedDomainDelete.h>
#endif
TRANSACTION(ttPERMISSIONED_DOMAIN_DELETE, 72, PermissionedDomainDelete, ({
{sfDomainID, soeREQUIRED},
}))
@@ -510,6 +646,9 @@ TRANSACTION(ttPERMISSIONED_DOMAIN_DELETE, 72, PermissionedDomainDelete, ({
open ledger, so a reveal only becomes public after the revealing
validator's transaction set is locked.
*/
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/Change.h>
#endif
TRANSACTION(ttENTROPY, 89, Entropy, ({
{sfLedgerSequence, soeREQUIRED},
{sfPublicKey, soeREQUIRED},
@@ -519,17 +658,26 @@ TRANSACTION(ttENTROPY, 89, Entropy, ({
/* sfAccount is supplied by TxFormats::commonFields; listing it here as well
makes the SOTemplate reject the format at construction. */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/SetManifest.h>
#endif
TRANSACTION(ttMANIFEST_SET, 91, SetManifest, ({
{sfManifest, soeREQUIRED},
}))
/* A pseudo-txn alarm signal for invoking a hook, emitted by validators after alarm set conditions are met */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/Cron.h>
#endif
TRANSACTION(ttCRON, 92, Cron, ({
{sfOwner, soeREQUIRED},
{sfLedgerSequence, soeREQUIRED},
}))
/* Sechedule an alarm for later */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/CronSet.h>
#endif
TRANSACTION(ttCRON_SET, 93, CronSet, ({
{sfDelaySeconds, soeOPTIONAL},
{sfRepeatCount, soeOPTIONAL},
@@ -537,6 +685,9 @@ TRANSACTION(ttCRON_SET, 93, CronSet, ({
}))
/* A note attaching transactor that allows the owner or issuer (on a object by object basis) to attach remarks */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/SetRemarks.h>
#endif
TRANSACTION(ttREMARKS_SET, 94, SetRemarks, ({
{sfObjectID, soeREQUIRED},
{sfRemarks, soeREQUIRED},
@@ -544,6 +695,9 @@ TRANSACTION(ttREMARKS_SET, 94, SetRemarks, ({
/* A payment transactor that delivers only the exact amounts specified, creating accounts and TLs as needed
* that the sender pays for. */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/Remit.h>
#endif
TRANSACTION(ttREMIT, 95, Remit, ({
{sfDestination, soeREQUIRED},
{sfAmounts, soeOPTIONAL},
@@ -558,12 +712,18 @@ TRANSACTION(ttREMIT, 95, Remit, ({
/** This transaction can only be used by the genesis account, which is controlled exclusively by
* rewards/governance hooks, to print new XRP to be delivered directly to an array of destinations,
* according to reward schedule */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/GenesisMint.h>
#endif
TRANSACTION(ttGENESIS_MINT, 96, GenesisMint, ({
{sfGenesisMints, soeREQUIRED},
}))
/** This transaction accepts a proof of burn from an external network as a basis
* for minting according to featureImport */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/Import.h>
#endif
TRANSACTION(ttIMPORT, 97, Import, ({
{sfBlob, soeREQUIRED},
{sfIssuer, soeOPTIONAL},
@@ -571,12 +731,18 @@ TRANSACTION(ttIMPORT, 97, Import, ({
/** This transaction resets accumulator/counters and claims a reward for holding an average balance
* from a specified hook */
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/ClaimReward.h>
#endif
TRANSACTION(ttCLAIM_REWARD, 98, ClaimReward, ({
{sfIssuer, soeOPTIONAL},
{sfClaimCurrency, soeOPTIONAL},
}))
/** This transaction invokes a hook, providing arbitrary data. Essentially as a 0 drop payment. **/
#if TRANSACTION_INCLUDE
# include <xrpld/app/tx/detail/Invoke.h>
#endif
TRANSACTION(ttINVOKE, 99, Invoke, ({
{sfBlob, soeOPTIONAL},
{sfDestination, soeOPTIONAL},
+2
View File
@@ -171,6 +171,7 @@ JSS(amendment_blocked); // out: NetworkOPs
JSS(amm_account); // in: amm_info
JSS(amount); // out: AccountChannels, amm_info
JSS(amount2); // out: amm_info
JSS(anchored_through); // out: CatalogueLoad
JSS(api_version); // in: many, out: Version
JSS(api_version_low); // out: Version
JSS(applied); // out: SubmitTransaction
@@ -744,6 +745,7 @@ JSS(TRANSACTION_FLAGS_INDICES); // out: RPC server_definitions
JSS(type_hex); // out: STPathSet
JSS(unl); // out: UnlList
JSS(unlimited); // out: Connection.h
JSS(unpinned); // out: CatalogueUnpin
JSS(uptime); // out: GetCounts
JSS(uri); // out: ValidatorSites
JSS(url); // in/out: Subscribe, Unsubscribe
+74
View File
@@ -1271,6 +1271,79 @@ public:
BEAST_EXPECT(table->needValidatedLedger(257));
}
void
testIgnoredUnsupported()
{
testcase("ignore_amendments");
using namespace std::chrono_literals;
weeks constexpr w(1);
test::jtx::Env env{*this, makeConfig()};
static std::vector<AmendmentTable::FeatureInfo> const supported =
makeDefaultYes(yes_);
// Ignore every unsupported_ amendment except the first one.
BEAST_EXPECT(unsupported_.size() > 1);
std::vector<std::string> const ignoredNames(
unsupported_.begin() + 1, unsupported_.end());
auto table = make_AmendmentTable(
env.app(),
w,
supported,
emptySection_,
emptySection_,
journal_,
makeSection(ignoredNames));
for (auto const& a : ignoredNames)
BEAST_EXPECT(table->isSupported(amendmentId(a)));
BEAST_EXPECT(!table->isSupported(amendmentId(unsupported_[0])));
majorityAmendments_t const majority;
// Enabling only ignored amendments must not block.
std::set<uint256> enabled;
for (auto const& a : ignoredNames)
enabled.insert(amendmentId(a));
table->doValidatedLedger(1, enabled, majority);
BEAST_EXPECT(!table->hasUnsupportedEnabled());
BEAST_EXPECT(table->unsupportedAmendments().empty());
// An ignored amendment must never be voted for.
auto const desired = table->getDesired();
for (auto const& a : ignoredNames)
BEAST_EXPECT(
std::find(desired.begin(), desired.end(), amendmentId(a)) ==
desired.end());
// A non-ignored unsupported amendment still blocks.
enabled.insert(amendmentId(unsupported_[0]));
table->doValidatedLedger(2, enabled, majority);
BEAST_EXPECT(table->hasUnsupportedEnabled());
// Bad entries are rejected.
Section bad("ignore_amendments");
bad.append("NotARealFeatureName");
try
{
make_AmendmentTable(
env.app(),
w,
supported,
emptySection_,
emptySection_,
journal_,
bad);
fail("expected bad [ignore_amendments] entry to throw");
}
catch (std::runtime_error const&)
{
pass();
}
}
void
testFeature(FeatureBitset const& feat)
{
@@ -1294,6 +1367,7 @@ public:
testBadConfig();
testEnableVeto();
testHasUnsupported();
testIgnoredUnsupported();
testFeature(all - fixMajorityCalc);
testFeature(all);
}
+19
View File
@@ -578,6 +578,25 @@ public:
ter(tecNO_TARGET));
env.close();
}
// HookName removed with a HookName-only update
// The sfHooks entry keeps an STI_NOTPRESENT placeholder for
// sfHookName, which preclaim must treat as "no name".
{
Json::Value jv;
jv[jss::Account] = issuer.human();
jv[jss::TransactionType] = jss::SetHook;
jv[jss::Flags] = 0;
jv[jss::Hooks] = Json::Value{Json::arrayValue};
jv[jss::Hooks][0U][jss::Hook][jss::HookName] = "";
env(jv, fee(XRP(1)));
env.close();
env(reward::claim(alice),
reward::issuer(issuer),
fee(XRP(1)),
ter(tesSUCCESS));
env.close();
}
// Vaild HookOn
{
auto hookObj = hso(jtx::genesis::AcceptHook, overrideFlag);
+146
View File
@@ -33,9 +33,11 @@
#include <xrpl/protocol/jss.h>
#include <boost/filesystem.hpp>
#include <chrono>
#include <exception>
#include <limits>
#include <string>
#include <thread>
namespace ripple {
namespace test {
@@ -93,6 +95,27 @@ class SHAMapStore_test : public beast::unit_test::suite
return cfg;
}
// Online delete with DatabasePinned over in-memory backends. Unlike
// pinnedPersistent, the hot store keeps nodes (type=rwdb is the null
// node store), so ledgers closed in the test can be fetched. There is
// no database_path, so every SHAMapStoreImp opens its own private
// state.db: a second instance built by a test does not see (or trip
// over) the rotation state of the Env's own.
static auto
pinnedInMemory(
std::unique_ptr<Config> cfg,
std::string const& nodePath,
std::string const& pinnedPath)
{
cfg = onlineDelete(std::move(cfg));
auto& section = cfg->section(ConfigSection::nodeDatabase());
section.set("type", "memory");
section.set("path", nodePath);
section.set("pinned_type", "memory");
section.set("pinned_path", pinnedPath);
return cfg;
}
bool
goodLedger(
jtx::Env& env,
@@ -1153,6 +1176,127 @@ public:
}
}
// catalogue_unpin of everything (or a rolled-back first load) leaves the
// pinned set empty. That must persist as "no pins", not as a string that
// fails to parse on the next start.
void
testPinnedRangeRestoreEmpty()
{
testcase("pinned range restore of an empty set");
using namespace jtx;
beast::temp_dir tempDir;
auto const tempPath = boost::filesystem::path(tempDir.path());
auto const nodePath = (tempPath / "node").string();
auto const pinnedPath = (tempPath / "pinned").string();
Env env(*this, envconfig(pinnedInMemory, nodePath, pinnedPath));
NodeStoreScheduler scheduler(env.app().getJobQueue());
auto const journal = env.app().journal("SHAMapStoreTest");
try
{
SHAMapStoreImp restoreCheck(env.app(), scheduler, journal);
RangeSet<std::uint32_t> ranges;
ranges.insert(range(100u, 200u));
restoreCheck.setPinnedRanges(ranges);
restoreCheck.setPinnedRanges(RangeSet<std::uint32_t>{});
restoreCheck.start();
restoreCheck.stop();
pass();
}
catch (std::exception const& e)
{
fail(
std::string("restart with no pinned ranges threw: ") +
e.what());
}
BEAST_EXPECT(
env.app().getLedgerMaster().getPinnedLedgersRangeSet().empty());
}
// A crash between a pinned-range checkpoint and the backend committing
// the last ledgers leaves the recorded interval's tail missing. Startup
// trims it back to the last ledger present instead of refusing to start.
void
testPinnedRangeRestoreTrimsMissingTail()
{
testcase("pinned range restore trims a missing tail");
using namespace jtx;
beast::temp_dir tempDir;
auto const tempPath = boost::filesystem::path(tempDir.path());
auto const nodePath = (tempPath / "node").string();
auto const pinnedPath = (tempPath / "pinned").string();
Env env(*this, envconfig(pinnedInMemory, nodePath, pinnedPath));
for (int i = 0; i < 4; ++i)
env.close();
auto& rdb = env.app().getRelationalDatabase();
auto const last = env.app().getLedgerMaster().getValidLedgerIndex();
auto const first = last - 2;
// Ledger saves may complete asynchronously; wait for the SQL rows.
for (int i = 0; i < 100 && rdb.getHashByIndex(last).isZero(); ++i)
std::this_thread::sleep_for(std::chrono::milliseconds(10));
if (!BEAST_EXPECT(
rdb.getHashByIndex(first).isNonZero() &&
rdb.getHashByIndex(last).isNonZero()))
return;
NodeStoreScheduler scheduler(env.app().getJobQueue());
auto const journal = env.app().journal("SHAMapStoreTest");
try
{
SHAMapStoreImp restoreCheck(env.app(), scheduler, journal);
// Recorded through last + 10, but only [first, last] exist.
RangeSet<std::uint32_t> recorded;
recorded.insert(range(first, last + 10));
restoreCheck.setPinnedRanges(recorded);
restoreCheck.start();
restoreCheck.stop();
}
catch (std::exception const& e)
{
fail(std::string("restart with a missing tail threw: ") + e.what());
return;
}
RangeSet<std::uint32_t> expected;
expected.insert(range(first, last));
BEAST_EXPECT(
env.app().getLedgerMaster().getPinnedLedgersRangeSet() == expected);
// A missing tail longer than the trim limit (65536 ledgers) is not
// an interrupted load: it means the wrong pinned store, so startup
// still refuses.
try
{
SHAMapStoreImp restoreCheck(env.app(), scheduler, journal);
RangeSet<std::uint32_t> recorded;
recorded.insert(range(first, last + 70000));
restoreCheck.setPinnedRanges(recorded);
restoreCheck.start();
restoreCheck.stop();
fail("Expected startup failure for an over-long missing tail");
}
catch (std::runtime_error const& e)
{
BEAST_EXPECT(
std::string(e.what()).find("Persisted pinned interval") !=
std::string::npos);
}
}
// Helper: compare two RangeSet<uint32_t> for set-equality. The
// test cares about which sequences are in the result, not the
// particular interval representation.
@@ -1302,6 +1446,8 @@ public:
testNullFactoryDropsWrites();
testRotate();
testPinnedRangeRestoreRequiresPinnedData();
testPinnedRangeRestoreEmpty();
testPinnedRangeRestoreTrimsMissingTail();
}
};
+112
View File
@@ -2134,6 +2134,118 @@ public:
}
}
{
/// Rename and remove the HookName with HookName-only updates
Env env{*this, features};
env.fund(XRP(10000), alice);
env.close();
// Create with hook name
auto jvh = hso(accept_wasm);
jvh[jss::HookName] = "41424344";
env(ripple::test::jtx::hook(alice, {{jvh}}, 0),
M("Create named hook"),
HSFEE);
env.close();
auto const hookNameOnlyUpdate = [&](std::string const& name) {
Json::Value jv;
jv[jss::Account] = alice.human();
jv[jss::TransactionType] = jss::SetHook;
jv[jss::Flags] = 0;
jv[jss::Hooks] = Json::Value{Json::arrayValue};
Json::Value iv;
iv[jss::HookName] = name;
jv[jss::Hooks][0U][jss::Hook] = iv;
return jv;
};
// Rename: update carrying only a non-empty sfHookName
env(hookNameOnlyUpdate("4142434445"),
M("Rename hook with a HookName-only update"),
HSFEE);
env.close();
{
auto const hooks = env.le(keylet::hook(alice));
BEAST_EXPECT(hooks && hooks->isFieldPresent(sfHooks));
auto const& hooksArray = hooks->getFieldArray(sfHooks);
BEAST_EXPECT(hooksArray.size() == 1);
BEAST_EXPECT(hooksArray[0].isFieldPresent(sfHookName));
BEAST_EXPECT(
strHex(hooksArray[0].getFieldVL(sfHookName)) ==
"4142434445");
}
// Remove: update carrying only an empty sfHookName
env(hookNameOnlyUpdate(""),
M("Remove hook name with a HookName-only update"),
HSFEE);
// In the same open ledger as the removal: the open view hands
// the next transaction the very sfHooks entry the SetHook just
// modified, so the fee calculation and the hook chain must
// tolerate the removed name. This is the path that crashed the
// node at submit, and crashed every validator replaying the
// pair while building the ledger.
{
auto jv = invoke::invoke(alice);
auto const expectedFee =
calculateBaseFee(*env.current(), *env.jt(jv).stx);
BEAST_EXPECT(expectedFee == drops(19));
env(jv,
M("Invoke in the same open ledger as the removal"),
HSFEE);
}
env.close();
{
auto const hookExecutions =
env.meta()->getFieldArray(sfHookExecutions);
BEAST_EXPECT(hookExecutions.size() == 1);
BEAST_EXPECT(
hookExecutions[0].getFieldH256(sfHookHash) == accept_hash);
}
{
auto const hooks = env.le(keylet::hook(alice));
BEAST_EXPECT(hooks && hooks->isFieldPresent(sfHooks));
auto const& hooksArray = hooks->getFieldArray(sfHooks);
BEAST_EXPECT(hooksArray.size() == 1);
BEAST_EXPECT(!hooksArray[0].isFieldPresent(sfHookName));
}
// The now non-named hook must still be charged for and
// executed by transactions in later ledgers, where the entry
// is re-read from its serialized form.
{
auto jv = invoke::invoke(alice);
auto const expectedFee =
calculateBaseFee(*env.current(), *env.jt(jv).stx);
BEAST_EXPECT(expectedFee == drops(19));
env(jv, M("Invoke after removing the hook name"), HSFEE);
env.close();
auto const hookExecutions =
env.meta()->getFieldArray(sfHookExecutions);
BEAST_EXPECT(hookExecutions.size() == 1);
BEAST_EXPECT(
hookExecutions[0].getFieldH256(sfHookHash) == accept_hash);
}
// Removing the name of a hook that has none must be a no-op,
// not a thrown FieldErr (-> tefEXCEPTION): makeFieldAbsent on
// the untemplated entry throws when the field is missing.
env(hookNameOnlyUpdate(""),
M("Remove hook name again (no-op)"),
HSFEE);
env.close();
{
auto const hooks = env.le(keylet::hook(alice));
BEAST_EXPECT(hooks && hooks->isFieldPresent(sfHooks));
auto const& hooksArray = hooks->getFieldArray(sfHooks);
BEAST_EXPECT(hooksArray.size() == 1);
BEAST_EXPECT(!hooksArray[0].isFieldPresent(sfHookName));
}
}
// Install named hook
auto jvh = hso(accept_wasm);
jvh[jss::HookName] = "41424344";
+31
View File
@@ -1122,6 +1122,36 @@ trustthesevalidators.gov
}
}
// Standalone forces ledger_history to zero, but [node_db] type=rwdb
// (the null node store) sizes its retained-ledger window from it and
// SHAMapStoreImp throws on zero. cfg/rippled.cfg ships exactly this
// configuration, and jtx never calls Config::setup, so pin it here.
void
testSetupStandaloneRwdb()
{
testcase("setup standalone rwdb keeps ledger_history");
for (std::string const type : {"rwdb", "NuDB"})
{
detail::RippledCfgGuard const cfg(
*this,
"testSetupStandaloneRwdb_" + type,
"",
"",
true,
"[node_db]\ntype=" + type + "\npath=main\n");
Config config;
config.setup(
cfg.configFile(),
/*bQuiet*/ true,
/* bSilent */ true,
/* bStandalone */ true);
BEAST_EXPECT(config.standalone());
BEAST_EXPECT(config.nullBackend() == (type == "rwdb"));
BEAST_EXPECT(config.LEDGER_HISTORY == (type == "rwdb" ? 256 : 0));
}
}
void
testPort()
{
@@ -1813,6 +1843,7 @@ r.ripple.com:51235
testValidatorsFile();
testSetup(false);
testSetup(true);
testSetupStandaloneRwdb();
testPort();
testZeroPort();
testWhitespace();
+8
View File
@@ -66,6 +66,14 @@ public:
// Write the batch
storeBatch(*backend, batch);
// Durability on demand: RocksDB drains its write queue and
// syncs its WAL; in-memory backends have nothing to wait for;
// NuDB commits on its own thread and cannot promise it.
backend->store(batch.front());
bool const durable = backend->syncDurable();
BEAST_EXPECT(durable == (type != "nudb"));
backend->sync();
{
// Read it back in
Batch copy;
+640 -1
View File
@@ -18,16 +18,22 @@
//==============================================================================
#include <test/jtx.h>
#include <xrpld/app/ledger/Ledger.h>
#include <xrpld/app/ledger/LedgerMaster.h>
#include <xrpld/core/ConfigSections.h>
#include <xrpld/nodestore/detail/DatabasePinnedImp.h>
#include <xrpl/beast/unit_test.h>
#include <xrpl/beast/utility/temp_dir.h>
#include <xrpl/protocol/jss.h>
#include <boost/filesystem.hpp>
#include <chrono>
#include <cstddef>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <grpc/impl/codegen/compression_types.h>
#include <iterator>
#include <optional>
#include <thread>
namespace ripple {
@@ -102,14 +108,28 @@ struct CatalogueEnvOpts
};
// envconfig with pinned_type set so catalogue_load passes the config guard.
// Uses rwdb for in-process speed (allowed in standalone).
//
// envconfig() selects [node_db] type=rwdb, which is the tree-only null node
// store: SHAMapStore never builds DatabasePinned on it, so catalogue_load
// would refuse to run. Use the in-memory "memory" store for the hot side
// instead. It is shared by path within the process, so each config gets its
// own path (SHAMapStore creates the directory; nothing is written to it).
// The pinned side uses rwdb, an in-memory persistent backend.
inline std::unique_ptr<Config>
catalogueEnvconfig(CatalogueEnvOpts const& opts = {})
{
auto cfg = test::jtx::envconfig();
auto& nodeDb = cfg->section(ConfigSection::nodeDatabase());
if (opts.withPinnedType)
{
nodeDb.set("type", "memory");
nodeDb.set(
"path",
(boost::filesystem::temp_directory_path() /
boost::filesystem::unique_path("catalogue-test-%%%%-%%%%-%%%%"))
.string());
nodeDb.set("pinned_type", "rwdb");
}
nodeDb.set("online_delete", opts.onlineDelete);
return cfg;
}
@@ -1067,6 +1087,621 @@ class Catalogue_test : public beast::unit_test::suite
}
public:
// ---- Editing uncompressed catalogues -----------------------------------
//
// Ledger record: seq, hash, txHash, accountHash, parentHash, drops,
// closeFlags, closeTimeResolution, closeTime, parentCloseTime; then the
// state map and the tx map, each a run of leaf records ending in
// tnTERMINAL. A leaf record is type (1), key (32), and, unless it is a
// tnREMOVE, size (4) and data.
static constexpr std::size_t catLedgerHeaderSize =
4 + 32 * 4 + 8 + sizeof(int) + 4 + 8 + 8;
struct CatLedgerRecord
{
std::size_t header; // first byte of the ledger header
std::size_t state; // first byte of the state map
std::size_t tx; // first byte of the tx map
std::size_t end; // one past the tx map's tnTERMINAL
};
// Position just past the map that starts at `pos`.
static std::size_t
skipCatalogueMap(std::vector<std::uint8_t> const& bytes, std::size_t pos)
{
while (pos < bytes.size())
{
auto const type = bytes[pos++];
if (type == 255) // tnTERMINAL
return pos;
pos += 32;
if (type == 254) // tnREMOVE
continue;
if (pos + 4 > bytes.size())
break;
std::uint32_t size = 0;
std::memcpy(&size, &bytes[pos], 4);
pos += 4 + size;
}
return bytes.size();
}
// Position of the size field of the first leaf (not a removal) in the
// map that starts at `pos`, if the map has one.
static std::optional<std::size_t>
firstLeafSizeField(std::vector<std::uint8_t> const& bytes, std::size_t pos)
{
while (pos < bytes.size())
{
auto const type = bytes[pos];
if (type == 255)
return std::nullopt;
if (type != 254)
return pos + 1 + 32;
pos += 1 + 32;
}
return std::nullopt;
}
static std::vector<CatLedgerRecord>
indexCatalogue(std::vector<std::uint8_t> const& bytes)
{
std::vector<CatLedgerRecord> records;
std::size_t pos = sizeof(TestCATLHeader);
while (pos + catLedgerHeaderSize <= bytes.size())
{
CatLedgerRecord r;
r.header = pos;
r.state = pos + catLedgerHeaderSize;
r.tx = skipCatalogueMap(bytes, r.state);
r.end = skipCatalogueMap(bytes, r.tx);
records.push_back(r);
pos = r.end;
}
return records;
}
static std::vector<std::uint8_t>
readCatalogue(std::string const& path)
{
std::ifstream in(path, std::ios::binary);
return {
std::istreambuf_iterator<char>(in),
std::istreambuf_iterator<char>()};
}
static void
writeCatalogue(std::string const& path, std::vector<std::uint8_t> const& b)
{
std::ofstream out(path, std::ios::binary | std::ios::trunc);
out.write(reinterpret_cast<char const*>(b.data()), b.size());
}
static Json::Value
loadCatalogue(test::jtx::Env& env, std::string const& path, bool ignoreHash)
{
Json::Value params{Json::objectValue};
params[jss::input_file] = path;
if (ignoreHash)
params[jss::ignore_hash] = true;
return env.client().invoke("catalogue_load", params)[jss::result];
}
static bool
errorContains(Json::Value const& result, std::string const& what)
{
return result[jss::status] == "error" &&
result[jss::error_message].asString().find(what) !=
std::string::npos;
}
void
testCatalogueLoadFailures(FeatureBitset features)
{
testcase("catalogue_load: failures part way through a file");
using namespace test::jtx;
using beast::severities::kDisabled;
boost::filesystem::path tempDir =
boost::filesystem::temp_directory_path() /
boost::filesystem::unique_path();
boost::filesystem::create_directories(tempDir);
auto const goodPath = (tempDir / "good.catl").string();
// An uncompressed catalogue of ledgers 3-6, so records can be edited.
{
Env env{*this, catalogueEnvconfig(), features, nullptr, kDisabled};
prepareLedgerData(env, 3);
Json::Value params{Json::objectValue};
params[jss::min_ledger] = 3;
params[jss::max_ledger] = 6;
params[jss::output_file] = goodPath;
params[jss::compression_level] = 0;
auto const result =
env.client().invoke("catalogue_create", params)[jss::result];
BEAST_EXPECT(result[jss::status] == jss::success);
}
auto const good = readCatalogue(goodPath);
auto const records = indexCatalogue(good);
if (!BEAST_EXPECT(
records.size() == 4 && records.back().end == good.size()))
return;
// Write `bytes` to a new file, load it into a fresh server, and hand
// the result and the resulting pinned set to `check`.
int fileNo = 0;
auto loadEdited = [&](std::vector<std::uint8_t> const& bytes,
bool ignoreHash,
auto&& check) {
auto const path =
(tempDir / ("edited" + std::to_string(++fileNo) + ".catl"))
.string();
writeCatalogue(path, bytes);
Env env{*this, catalogueEnvconfig(), features, nullptr, kDisabled};
auto const result = loadCatalogue(env, path, ignoreHash);
check(
result, env.app().getLedgerMaster().getPinnedLedgersRangeSet());
};
auto const hashField = offsetof(TestCATLHeader, hash);
auto const maxLedgerField = offsetof(TestCATLHeader, max_ledger);
// Offsets within a ledger header record.
constexpr std::size_t accountHashAt = 4 + 32 * 2;
constexpr std::size_t dropsAt = 4 + 32 * 4;
// The stored checksum is wrong but every ledger is intact: all four
// load, then the whole-file check fails and the run is rolled back.
{
auto bytes = good;
bytes[hashField] ^= 0xff;
loadEdited(
bytes, false, [&](auto const& result, auto const& pinned) {
BEAST_EXPECT(errorContains(
result, "catalogue hash verification failed"));
BEAST_EXPECT(pinned.empty());
});
}
// A ledger part way through is corrupt. The load stops there, the
// rest of the file is still hashed, the checksum fails, and the
// ledgers loaded before it are rolled back too.
{
auto bytes = good;
bytes[records[2].header + dropsAt] ^= 0x01;
loadEdited(
bytes, false, [&](auto const& result, auto const& pinned) {
BEAST_EXPECT(errorContains(
result, "catalogue hash verification failed"));
BEAST_EXPECT(pinned.empty());
});
}
// A checksum failure only rolls back what this run added: ledgers
// that were already pinned stay pinned.
{
auto bytes = good;
bytes[hashField] ^= 0xff;
auto const badPath = (tempDir / "badsum.catl").string();
writeCatalogue(badPath, bytes);
Env env{*this, catalogueEnvconfig(), features, nullptr, kDisabled};
BEAST_EXPECT(
loadCatalogue(env, goodPath, false)[jss::status] ==
jss::success);
BEAST_EXPECT(errorContains(
loadCatalogue(env, badPath, false),
"catalogue hash verification failed"));
auto& lm = env.app().getLedgerMaster();
BEAST_EXPECT(to_string(lm.getPinnedLedgersRangeSet()) == "3-6");
// The pinned store: an empty batch is a no-op, an in-memory
// backend is durable on demand, and a fetch made for
// online_delete's copy pass (duplicate == true) finds nodes
// that exist only in the pinned store.
auto* const pinnedDb = dynamic_cast<NodeStore::DatabasePinnedImp*>(
&env.app().getNodeStore());
if (BEAST_EXPECT(pinnedDb))
{
pinnedDb->storePinnedBatch({});
BEAST_EXPECT(pinnedDb->syncPinned());
auto const stateRoot =
uint256::fromVoid(&good[records[0].header + accountHashAt]);
BEAST_EXPECT(env.app().getNodeStore().fetchNodeObject(
stateRoot, 0, NodeStore::FetchType::synchronous, true));
}
}
// Out of sequence: ledger 4's record claims to be ledger 9.
{
auto bytes = good;
std::uint32_t const nine = 9;
std::memcpy(&bytes[records[1].header], &nine, 4);
loadEdited(
bytes, true, [&](auto const& result, auto const& pinned) {
BEAST_EXPECT(errorContains(result, "out of sequence"));
BEAST_EXPECT(boost::icl::contains(pinned, 3u));
BEAST_EXPECT(!boost::icl::contains(pinned, 4u));
});
}
// The header promises a ledger 7 the file does not contain. Ledgers
// 3-6 load (hash checks are off) before the file runs out.
{
auto bytes = good;
std::uint32_t const seven = 7;
std::memcpy(&bytes[maxLedgerField], &seven, 4);
loadEdited(
bytes, true, [&](auto const& result, auto const& pinned) {
BEAST_EXPECT(errorContains(
result, "Unexpected end of catalogue file"));
BEAST_EXPECT(to_string(pinned) == "3-6");
});
}
// An absurd leaf size in each kind of map is rejected while parsing.
auto withHugeLeaf = [&](std::size_t sizeField) {
auto bytes = good;
std::uint32_t const huge = 0xFFFFFFFF;
std::memcpy(&bytes[sizeField], &huge, 4);
return bytes;
};
// ... in the base ledger's state map.
if (auto const field = firstLeafSizeField(good, records[0].state);
BEAST_EXPECT(field))
{
loadEdited(
withHugeLeaf(*field),
true,
[&](auto const& result, auto const& pinned) {
BEAST_EXPECT(errorContains(
result, "Failed to load base ledger state"));
BEAST_EXPECT(pinned.empty());
});
}
// ... in a later ledger's state delta.
{
std::optional<std::size_t> index;
std::optional<std::size_t> field;
for (std::size_t i = 1; i < records.size() && !field; ++i)
if ((field = firstLeafSizeField(good, records[i].state)))
index = i;
if (BEAST_EXPECT(field))
{
std::uint32_t const seq = 3 + *index;
loadEdited(
withHugeLeaf(*field),
true,
[&](auto const& result, auto const& pinned) {
BEAST_EXPECT(errorContains(
result, "Failed to apply ledger delta"));
BEAST_EXPECT(boost::icl::contains(pinned, seq - 1));
BEAST_EXPECT(!boost::icl::contains(pinned, seq));
});
}
}
// ... in a ledger's transaction map.
{
std::optional<std::size_t> index;
std::optional<std::size_t> field;
for (std::size_t i = 0; i < records.size() && !field; ++i)
if ((field = firstLeafSizeField(good, records[i].tx)))
index = i;
if (BEAST_EXPECT(field))
{
std::uint32_t const seq = 3 + *index;
loadEdited(
withHugeLeaf(*field),
true,
[&](auto const& result, auto const& pinned) {
BEAST_EXPECT(errorContains(
result, "Failed to apply ledger delta"));
BEAST_EXPECT(!boost::icl::contains(pinned, seq));
});
}
}
boost::filesystem::remove_all(tempDir);
}
void
testCataloguePinnedStoreRequired(FeatureBitset features)
{
testcase("catalogue_load / catalogue_unpin: pinned store required");
using namespace test::jtx;
using beast::severities::kDisabled;
// pinned_type is set, but [node_db] type=rwdb is the null node
// store, on which SHAMapStore never builds the pinned store.
{
auto cfg = test::jtx::envconfig();
auto& nodeDb = cfg->section(ConfigSection::nodeDatabase());
nodeDb.set("pinned_type", "rwdb");
nodeDb.set("online_delete", "256");
Env env{*this, std::move(cfg), features, nullptr, kDisabled};
Json::Value params{Json::objectValue};
params[jss::input_file] = "/nonexistent.catl";
auto const result =
env.client().invoke("catalogue_load", params)[jss::result];
BEAST_EXPECT(
errorContains(result, "requires the pinned node store"));
}
// catalogue_unpin needs pinned_type configured at all.
{
Env env{
*this,
catalogueEnvconfig({.withPinnedType = false}),
features,
nullptr,
kDisabled};
Json::Value params{Json::objectValue};
params[jss::ledger_index_min] = 1;
params[jss::ledger_index_max] = 2;
auto const result =
env.client().invoke("catalogue_unpin", params)[jss::result];
BEAST_EXPECT(errorContains(result, "pinned_type"));
}
}
void
testCatalogueChainLinkage(FeatureBitset features)
{
testcase("catalogue_load: Ledgers must chain");
using namespace test::jtx;
Env env{
*this,
catalogueEnvconfig(),
features,
nullptr,
beast::severities::kDisabled,
};
prepareLedgerData(env, 3);
boost::filesystem::path tempDir =
boost::filesystem::temp_directory_path() /
boost::filesystem::unique_path();
boost::filesystem::create_directories(tempDir);
auto cataloguePath = (tempDir / "test.catl").string();
// Uncompressed, so ledger records can be located and edited.
{
Json::Value params{Json::objectValue};
params[jss::min_ledger] = 3;
params[jss::max_ledger] = 6;
params[jss::output_file] = cataloguePath;
params[jss::compression_level] = 0;
auto const result =
env.client().invoke("catalogue_create", params)[jss::result];
BEAST_EXPECT(result[jss::status] == jss::success);
}
std::vector<std::uint8_t> bytes;
{
std::ifstream in(cataloguePath, std::ios::binary);
bytes.assign(
std::istreambuf_iterator<char>(in),
std::istreambuf_iterator<char>());
}
// Ledger record: seq, hash, txHash, accountHash, parentHash, drops,
// closeFlags, closeTimeResolution, closeTime, parentCloseTime; then
// the state map and tx map as leaf records ending in tnTERMINAL.
constexpr std::size_t ledgerHeaderSize =
4 + 32 * 4 + 8 + sizeof(int) + 4 + 8 + 8;
auto skipMap = [&](std::size_t pos) -> std::size_t {
while (pos < bytes.size())
{
auto const type = bytes[pos++];
if (type == 255) // tnTERMINAL
return pos;
pos += 32;
if (type == 254) // tnREMOVE
continue;
if (pos + 4 > bytes.size())
break;
std::uint32_t size = 0;
std::memcpy(&size, &bytes[pos], 4);
pos += 4 + size;
}
return bytes.size();
};
// Skip the base ledger (3) to reach ledger 4.
std::size_t pos = sizeof(TestCATLHeader) + ledgerHeaderSize;
pos = skipMap(skipMap(pos));
if (!BEAST_EXPECT(pos + ledgerHeaderSize <= bytes.size()))
return;
LedgerInfo info;
std::size_t p = pos;
auto take = [&](void* dst, std::size_t n) {
std::memcpy(dst, &bytes[p], n);
p += n;
};
std::uint64_t drops = 0, closeTime = 0, parentCloseTime = 0;
std::uint32_t resolution = 0;
take(&info.seq, 4);
std::size_t const hashOffset = p;
take(info.hash.data(), 32);
take(info.txHash.data(), 32);
take(info.accountHash.data(), 32);
std::size_t const parentOffset = p;
take(info.parentHash.data(), 32);
take(&drops, 8);
take(&info.closeFlags, sizeof(info.closeFlags));
take(&resolution, 4);
take(&closeTime, 8);
take(&parentCloseTime, 8);
info.drops = drops;
info.closeTime = NetClock::time_point{NetClock::duration{closeTime}};
info.parentCloseTime =
NetClock::time_point{NetClock::duration{parentCloseTime}};
info.closeTimeResolution = NetClock::duration{resolution};
BEAST_EXPECT(info.seq == 4);
// Sanity: the record parses to a ledger whose hash checks out.
BEAST_EXPECT(calculateLedgerHash(info) == info.hash);
// Re-parent ledger 4 and fix up its hash, so the file stays
// internally consistent ledger by ledger; only the chain check can
// catch it.
info.parentHash = ~info.parentHash;
info.hash = calculateLedgerHash(info);
std::memcpy(&bytes[parentOffset], info.parentHash.data(), 32);
std::memcpy(&bytes[hashOffset], info.hash.data(), 32);
{
std::ofstream out(
cataloguePath, std::ios::binary | std::ios::trunc);
out.write(
reinterpret_cast<char const*>(bytes.data()), bytes.size());
}
Env loadEnv{
*this,
catalogueEnvconfig(),
features,
nullptr,
beast::severities::kDisabled,
};
Json::Value params{Json::objectValue};
params[jss::input_file] = cataloguePath;
params[jss::ignore_hash] = true; // the file was edited
auto const result =
loadEnv.client().invoke("catalogue_load", params)[jss::result];
BEAST_EXPECT(result[jss::status] == "error");
BEAST_EXPECT(
result[jss::error_message].asString().find(
"does not chain to its predecessor") != std::string::npos);
// The base ledger verified and stays pinned; nothing after it does.
auto const pinned =
loadEnv.app().getLedgerMaster().getPinnedLedgersRangeSet();
BEAST_EXPECT(boost::icl::contains(pinned, 3u));
BEAST_EXPECT(!boost::icl::contains(pinned, 4u));
boost::filesystem::remove_all(tempDir);
}
void
testCatalogueUnpin(FeatureBitset features)
{
testcase("catalogue_unpin");
using namespace test::jtx;
Env env{
*this,
catalogueEnvconfig(),
features,
nullptr,
beast::severities::kDisabled,
};
prepareLedgerData(env, 5);
boost::filesystem::path tempDir =
boost::filesystem::temp_directory_path() /
boost::filesystem::unique_path();
boost::filesystem::create_directories(tempDir);
auto cataloguePath = (tempDir / "test.catl").string();
{
Json::Value params{Json::objectValue};
params[jss::min_ledger] = 3;
params[jss::max_ledger] = 8;
params[jss::output_file] = cataloguePath;
auto const result =
env.client().invoke("catalogue_create", params)[jss::result];
BEAST_EXPECT(result[jss::status] == jss::success);
}
{
Json::Value params{Json::objectValue};
params[jss::input_file] = cataloguePath;
auto const result =
env.client().invoke("catalogue_load", params)[jss::result];
BEAST_EXPECT(result[jss::status] == jss::success);
}
auto& lm = env.app().getLedgerMaster();
BEAST_EXPECT(to_string(lm.getPinnedLedgersRangeSet()) == "3-8");
auto unpin = [&](std::optional<Json::Value> min,
std::optional<Json::Value> max) {
Json::Value params{Json::objectValue};
if (min)
params[jss::ledger_index_min] = *min;
if (max)
params[jss::ledger_index_max] = *max;
return env.client().invoke("catalogue_unpin", params)[jss::result];
};
// Bad parameters
BEAST_EXPECT(unpin({}, {})[jss::status] == "error");
BEAST_EXPECT(unpin(5, {})[jss::status] == "error");
BEAST_EXPECT(unpin(6, 5)[jss::status] == "error");
BEAST_EXPECT(unpin(-1, 5)[jss::status] == "error");
BEAST_EXPECT(unpin("x", 5)[jss::status] == "error");
BEAST_EXPECT(to_string(lm.getPinnedLedgersRangeSet()) == "3-8");
// Cut a hole
{
auto const result = unpin(5, 6);
BEAST_EXPECT(result[jss::status] == jss::success);
BEAST_EXPECT(result[jss::unpinned] == "5-6");
BEAST_EXPECT(result[jss::complete_ledgers_pinned] == "3-4,7-8");
BEAST_EXPECT(lm.isPinned(4) && lm.isPinned(7));
BEAST_EXPECT(!lm.isPinned(5) && !lm.isPinned(6));
}
// A range only partly pinned unpins just the pinned part
{
auto const result = unpin(1, 4);
BEAST_EXPECT(result[jss::status] == jss::success);
BEAST_EXPECT(result[jss::unpinned] == "3-4");
BEAST_EXPECT(result[jss::complete_ledgers_pinned] == "7-8");
}
// Nothing pinned in range: success, nothing unpinned. (Unsigned
// JSON values are accepted as well as non-negative signed ones.)
{
auto const result = unpin(Json::UInt(5), Json::UInt(6));
BEAST_EXPECT(result[jss::status] == jss::success);
BEAST_EXPECT(result[jss::unpinned] == "empty");
BEAST_EXPECT(result[jss::complete_ledgers_pinned] == "7-8");
}
// Unpinning the rest leaves nothing pinned. (state.db then records
// an empty set, which must not stop the next start; see
// SHAMapStore_test.)
{
auto const result = unpin(1, 100);
BEAST_EXPECT(result[jss::status] == jss::success);
BEAST_EXPECT(result[jss::unpinned] == "7-8");
BEAST_EXPECT(result[jss::complete_ledgers_pinned] == "empty");
BEAST_EXPECT(lm.getPinnedLedgersRangeSet().empty());
}
// Re-loading the file pins it again
{
Json::Value params{Json::objectValue};
params[jss::input_file] = cataloguePath;
auto const result =
env.client().invoke("catalogue_load", params)[jss::result];
BEAST_EXPECT(result[jss::status] == jss::success);
BEAST_EXPECT(to_string(lm.getPinnedLedgersRangeSet()) == "3-8");
}
boost::filesystem::remove_all(tempDir);
}
void
run() override
{
@@ -1081,6 +1716,10 @@ public:
testCatalogueFileSize(all);
testCatalogueCompression(all);
testCatalogueStatus(all);
testCatalogueChainLinkage(all);
testCatalogueUnpin(all);
testCatalogueLoadFailures(all);
testCataloguePinnedStoreRequired(all);
}
};
+50
View File
@@ -1934,6 +1934,56 @@ static RPCCallTestData const rpcCallTestArray[] = {
RPCCallTestData::bad_cast,
R"()"},
// catalogue_unpin
// ------------------------------------------------------------
{"catalogue_unpin: minimal.",
__LINE__,
{"catalogue_unpin", "5000001", "6000000"},
RPCCallTestData::no_exception,
R"({
"method" : "catalogue_unpin",
"params" : [
{
"api_version" : %API_VER%,
"ledger_index_max" : 6000000,
"ledger_index_min" : 5000001
}
]
})"},
{"catalogue_unpin: too few arguments.",
__LINE__,
{"catalogue_unpin", "5000001"},
RPCCallTestData::no_exception,
R"({
"method" : "catalogue_unpin",
"params" : [
{
"error" : "badSyntax",
"error_code" : 1,
"error_message" : "Syntax error."
}
]
})"},
{"catalogue_unpin: too many arguments.",
__LINE__,
{"catalogue_unpin", "1", "2", "3"},
RPCCallTestData::no_exception,
R"({
"method" : "catalogue_unpin",
"params" : [
{
"error" : "badSyntax",
"error_code" : 1,
"error_message" : "Syntax error."
}
]
})"},
{"catalogue_unpin: non-integer bound.",
__LINE__,
{"catalogue_unpin", "first", "6000000"},
RPCCallTestData::bad_cast,
R"()"},
// channel_authorize
// -----------------------------------------------------------
{"channel_authorize: minimal.",
+79
View File
@@ -29,7 +29,9 @@
#include <xrpl/beast/utility/Journal.h>
#include <xrpl/protocol/Serializer.h>
#include <algorithm>
#include <optional>
#include <vector>
namespace ripple {
namespace tests {
@@ -131,6 +133,7 @@ public:
testCanonicalizeMergesInnerChildren(journal);
testCanonicalizeSharesDuplicateContent(journal);
testGetLinkedNode(journal);
testUpsertAndFlushToSink(journal);
}
void
@@ -284,6 +287,82 @@ public:
BEAST_EXPECT(liveLinked > 0);
}
void
testUpsertAndFlushToSink(beast::Journal const& journal)
{
testcase("upsertGiveItem and flushDirty to a sink");
tests::TestNodeFamily f(journal);
SHAMap a{SHAMapType::FREE, f}; // built with upsertGiveItem
SHAMap b{SHAMapType::FREE, f}; // reference: addItem/updateGiveItem
// Distinct first nibbles: a root with three leaf children.
constexpr uint256 h1(
"092891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7");
constexpr uint256 h2(
"436ccbac3347baa1f1e53baeef1f43334da88f1f6d70d963b833afd6dfa289fe");
constexpr uint256 h3(
"b92891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
auto const nm = SHAMapNodeType::tnTRANSACTION_NM;
// Absent keys are added.
BEAST_EXPECT(a.upsertGiveItem(nm, make_shamapitem(h1, IntToVUC(1))));
BEAST_EXPECT(a.upsertGiveItem(nm, make_shamapitem(h2, IntToVUC(2))));
BEAST_EXPECT(a.upsertGiveItem(nm, make_shamapitem(h3, IntToVUC(3))));
// A present key has its data replaced.
BEAST_EXPECT(a.upsertGiveItem(nm, make_shamapitem(h2, IntToVUC(20))));
// A change of leaf type is refused and leaves the item untouched.
BEAST_EXPECT(!a.upsertGiveItem(
SHAMapNodeType::tnTRANSACTION_MD,
make_shamapitem(h1, IntToVUC(10))));
BEAST_EXPECT(b.addItem(nm, make_shamapitem(h1, IntToVUC(1))));
BEAST_EXPECT(b.addItem(nm, make_shamapitem(h2, IntToVUC(2))));
BEAST_EXPECT(b.addItem(nm, make_shamapitem(h3, IntToVUC(3))));
BEAST_EXPECT(b.updateGiveItem(nm, make_shamapitem(h2, IntToVUC(20))));
{
auto const v20 = IntToVUC(20);
auto const& item = a.peekItem(h2);
BEAST_EXPECT(
item && item->slice() == Slice(v20.data(), v20.size()));
auto const v1 = IntToVUC(1);
auto const& item1 = a.peekItem(h1);
BEAST_EXPECT(
item1 && item1->slice() == Slice(v1.data(), v1.size()));
}
// Flush to a sink. Do not call getHash() on `a` first: it unshares
// the map and the flush would then have nothing to write.
auto const cacheBefore = f.getTreeNodeCache()->size();
std::vector<uint256> written;
bool sinkArgsOk = true;
SHAMap::FlushSink const sink =
[&](NodeObjectType t, Blob&& data, uint256 const& hash) {
sinkArgsOk =
sinkArgsOk && t == hotTRANSACTION_NODE && !data.empty();
written.push_back(hash);
};
a.flushDirty(hotTRANSACTION_NODE, sink);
BEAST_EXPECT(sinkArgsOk);
BEAST_EXPECT(written.size() == 4); // root + three leaves
// Unlike flushDirty(t), nothing was canonicalized into the cache.
BEAST_EXPECT(f.getTreeNodeCache()->size() == cacheBefore);
// The upserted map matches the reference, and its root was written.
BEAST_EXPECT(a.getHash() == b.getHash());
BEAST_EXPECT(
std::find(
written.begin(), written.end(), a.getHash().as_uint256()) !=
written.end());
// Everything is now shared: a second flush writes nothing.
written.clear();
a.flushDirty(hotTRANSACTION_NODE, sink);
BEAST_EXPECT(written.empty());
}
void
testGetLinkedNode(beast::Journal const& journal)
{
+1 -1
View File
@@ -86,7 +86,7 @@ bool
canHook(
STTx const& tx,
ripple::uint256 hookOn,
std::optional<ripple::Slice> hookName);
std::optional<ripple::Blob> hookName);
bool
canEmit(ripple::TxType txType, ripple::uint256 hookCanEmit);
+2 -2
View File
@@ -814,7 +814,7 @@ bool
hook::canHook(
STTx const& tx,
ripple::uint256 hookOn,
std::optional<ripple::Slice> hookName)
std::optional<ripple::Blob> hookName)
{
if (!canHookTT(tx.getTxnType(), hookOn))
return false;
@@ -828,7 +828,7 @@ hook::canHook(
// transaction, so we can't hook without the hook name
return false;
return tx[sfHookName] == *hookName;
return tx.getFieldVL(sfHookName) == *hookName;
}
ripple::uint256
+3 -2
View File
@@ -25,7 +25,8 @@ namespace ripple {
AcceptedLedger::AcceptedLedger(
std::shared_ptr<ReadView const> const& ledger,
Application& app)
Application& app,
bool withJson)
: mLedger(ledger)
{
transactions_.reserve(256);
@@ -33,7 +34,7 @@ AcceptedLedger::AcceptedLedger(
auto insertAll = [&](auto const& txns) {
for (auto const& item : txns)
transactions_.emplace_back(std::make_unique<AcceptedLedgerTx>(
ledger, item.first, item.second));
ledger, item.first, item.second, withJson));
};
transactions_.reserve(256);
+7 -1
View File
@@ -44,9 +44,15 @@ namespace ripple {
class AcceptedLedger : public CountedObject<AcceptedLedger>
{
public:
/** @param withJson build each transaction's publishable JSON. Pass
false only for ledgers that will never be published from this
object (e.g. pinned catalogue history being saved to SQL);
getJson() on such transactions returns a null value.
*/
AcceptedLedger(
std::shared_ptr<ReadView const> const& ledger,
Application& app);
Application& app,
bool withJson = true);
std::shared_ptr<ReadView const> const&
getLedger() const
+7 -1
View File
@@ -28,7 +28,8 @@ namespace ripple {
AcceptedLedgerTx::AcceptedLedgerTx(
std::shared_ptr<ReadView const> const& ledger,
std::shared_ptr<STTx const> const& txn,
std::shared_ptr<STObject const> const& met)
std::shared_ptr<STObject const> const& met,
bool withJson)
: mTxn(txn)
, mMeta(txn->getTransactionID(), ledger->seq(), *met)
, mAffected(mMeta.getAffectedAccounts())
@@ -41,6 +42,11 @@ AcceptedLedgerTx::AcceptedLedgerTx(
met->add(s);
mRawMeta = std::move(s.modData());
// The JSON (full tx + meta + hex raw meta per transaction) is only
// needed for publishing. Bulk SQL saves of pinned history skip it.
if (!withJson)
return;
mJson = Json::objectValue;
mJson[jss::transaction] = mTxn->getJson(JsonOptions::none);
+3 -1
View File
@@ -46,7 +46,8 @@ public:
AcceptedLedgerTx(
std::shared_ptr<ReadView const> const& ledger,
std::shared_ptr<STTx const> const&,
std::shared_ptr<STObject const> const&);
std::shared_ptr<STObject const> const&,
bool withJson = true);
std::shared_ptr<STTx const> const&
getTxn() const
@@ -88,6 +89,7 @@ public:
std::string
getEscMeta() const;
/** Null if constructed with withJson == false. */
Json::Value const&
getJson() const
{
+13
View File
@@ -228,6 +228,19 @@ public:
isPinned(std::uint32_t seq);
void
unpinLedger(std::uint32_t seq);
/** Remove `ranges` from the pinned set.
@param dropFromComplete of the ledgers actually unpinned, those to
also remove from complete_ledgers now (used to roll back a
load). Otherwise unpinned ledgers stay complete until online
delete releases them.
@return the ledgers that were pinned and no longer are.
*/
RangeSet<std::uint32_t>
unpinLedgers(
RangeSet<std::uint32_t> const& ranges,
RangeSet<std::uint32_t> const& dropFromComplete = {});
bool
isValidated(ReadView const& ledger);
bool
+16 -1
View File
@@ -501,7 +501,9 @@ LedgerMaster::storeLedger(std::shared_ptr<Ledger const> ledger, bool pin)
std::lock_guard sl(mPinnedLock);
mPinnedLedgers.insert(range(seq, seq));
}
JLOG(m_journal.info()) << "Pinned ledger : " << seq;
// catalogue_load pins every ledger it imports; at info this would
// emit one line per ledger for the whole file.
JLOG(m_journal.debug()) << "Pinned ledger : " << seq;
}
return true;
}
@@ -586,6 +588,19 @@ LedgerMaster::unpinLedger(std::uint32_t seq)
mPinnedLedgers.erase(range(seq, seq));
}
RangeSet<std::uint32_t>
LedgerMaster::unpinLedgers(
RangeSet<std::uint32_t> const& ranges,
RangeSet<std::uint32_t> const& dropFromComplete)
{
std::scoped_lock lock(mCompleteLock, mPinnedLock);
RangeSet<std::uint32_t> removed = mPinnedLedgers & ranges;
mPinnedLedgers -= removed;
if (!dropFromComplete.empty())
mCompleteLedgers -= (removed & dropFromComplete);
return removed;
}
bool
LedgerMaster::isValidated(ReadView const& ledger)
{
+17 -1
View File
@@ -1260,13 +1260,29 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline)
Section const& upVoted = config_->section(SECTION_AMENDMENTS);
Section const& ignored = config_->section(SECTION_IGNORE_AMENDMENTS);
// [ignore_amendments] exists to keep long-lived test networks syncable
// after an amendment they activated is removed from the codebase. It
// must never be usable on a production network.
if (!ignored.lines().empty() &&
(config_->NETWORK_ID == 0 || config_->NETWORK_ID == 21337))
{
JLOG(m_journal.fatal()) << "[" SECTION_IGNORE_AMENDMENTS
"] is not permitted on "
"network_id "
<< config_->NETWORK_ID;
return false;
}
m_amendmentTable = make_AmendmentTable(
*this,
config().AMENDMENT_MAJORITY_TIME,
supported,
upVoted,
downVoted,
logs_->journal("Amendments"));
logs_->journal("Amendments"),
ignored);
}
Pathfinder::initPathTable();
+2 -1
View File
@@ -211,7 +211,8 @@ make_AmendmentTable(
std::vector<AmendmentTable::FeatureInfo> const& supported,
Section const& enabled,
Section const& vetoed,
beast::Journal journal);
beast::Journal journal,
Section const& ignored = Section{});
} // namespace ripple
+51 -9
View File
@@ -381,7 +381,9 @@ void
SHAMapStoreImp::loadPinnedRanges()
{
auto rangesStr = state_db_.getPinnedRanges();
if (rangesStr.empty())
// "empty" is what to_string() renders for an empty set; accept it in
// case it was stored before setPinnedRanges() learned to write "".
if (rangesStr.empty() || rangesStr == "empty")
return;
auto const& nscfg = app_.config().section(ConfigSection::nodeDatabase());
@@ -411,22 +413,62 @@ SHAMapStoreImp::loadPinnedRanges()
static_cast<bool>(app_.getNodeStore().fetchNodeObject(hash, seq));
};
//
// catalogue_load records ranges at checkpoints, after handing the pinned
// backend every node of the recorded ledgers, but not every backend can
// make those writes durable on demand (NuDB commits on its own thread).
// A crash can therefore leave the tail of an interval recorded but not
// on disk. Each ledger's nodes reach the backend before its header, and
// ledgers in sequence, so what survives is a prefix: trim the upper bound
// back to the last ledger whose header is present instead of refusing to
// start. A missing lower bound still means the configured pinned store
// is not the one these ranges were loaded into.
constexpr std::uint32_t maxTailTrim = 65536;
RangeSet<std::uint32_t> verified;
for (auto const& interval : persistedRanges)
{
if (!hasLedgerData(interval.lower()) ||
!hasLedgerData(interval.upper()))
{
Throw<std::runtime_error>(
"Persisted pinned interval " +
std::to_string(interval.lower()) + "-" +
auto const lower = interval.lower();
auto upper = interval.upper();
auto const notPresent = [&]() {
return std::runtime_error(
"Persisted pinned interval " + std::to_string(lower) + "-" +
std::to_string(interval.upper()) +
" is not present in the currently configured pinned store");
};
if (!hasLedgerData(lower))
Throw<std::runtime_error>(notPresent());
std::uint32_t trimmed = 0;
while (upper > lower && !hasLedgerData(upper))
{
if (++trimmed > maxTailTrim)
Throw<std::runtime_error>(notPresent());
--upper;
}
if (trimmed)
{
JLOG(journal_.warn())
<< "Pinned interval " << lower << "-" << interval.upper()
<< " was recorded but its last " << trimmed
<< " ledger(s) are not in the pinned store (likely an "
"interrupted catalogue_load); trimmed to "
<< lower << "-" << upper
<< ". Re-run catalogue_load to restore them.";
}
verified.insert(range(lower, upper));
}
if (verified != persistedRanges)
state_db_.setPinnedRanges(to_string(verified));
JLOG(journal_.info()) << "Loaded pinned ranges from database: "
<< rangesStr;
app_.getLedgerMaster().setPinnedLedgersRangeSet(persistedRanges);
<< to_string(verified);
app_.getLedgerMaster().setPinnedLedgersRangeSet(verified);
}
void
+5 -3
View File
@@ -184,9 +184,11 @@ public:
void
setPinnedRanges(RangeSet<std::uint32_t> const& ranges) override
{
// Save pinned ranges to database for persistence
std::string rangesStr = to_string(ranges);
state_db_.setPinnedRanges(rangesStr);
// Save pinned ranges to database for persistence. to_string()
// renders an empty set as "empty", which from_string() rejects, so
// store "" instead: loadPinnedRanges() treats that as nothing pinned.
state_db_.setPinnedRanges(
ranges.empty() ? std::string{} : to_string(ranges));
}
void
+78 -6
View File
@@ -302,6 +302,15 @@ struct AmendmentState
/** Indicates an amendment that this server has code support for. */
bool supported = false;
/** Indicates an amendment listed in [ignore_amendments]. The server has
no code for it, but must not become amendment blocked if it is (or
becomes) enabled on the ledger. Such an amendment is reported as
`supported` internally so every blocking check (LedgerMaster,
NetworkOPs, Change) passes, but is never voted for and is shown as
unsupported + ignored over RPC.
*/
bool ignored = false;
/** The name of this amendment, possibly empty. */
std::string name;
@@ -476,7 +485,8 @@ public:
std::vector<FeatureInfo> const& supported,
Section const& enabled,
Section const& vetoed,
beast::Journal journal);
beast::Journal journal,
Section const& ignored);
uint256
find(std::string const& name) const override;
@@ -543,7 +553,8 @@ AmendmentTableImpl::AmendmentTableImpl(
std::vector<FeatureInfo> const& supported,
Section const& enabled,
Section const& vetoed,
beast::Journal journal)
beast::Journal journal,
Section const& ignored)
: lastUpdateSeq_(0)
, majorityTime_(majorityTime)
, unsupportedEnabled_(false)
@@ -586,6 +597,59 @@ AmendmentTableImpl::AmendmentTableImpl(
<< " voted by default if not enabled on the ledger.";
}
// Parse ignored amendments from config. Each line is either a registered
// feature name (e.g. "HookOnV2") or "<64 hex amendment ID> [name]". These
// are deliberately NOT persisted to wallet.db: removing the line from the
// config must restore normal amendment-blocking behaviour.
for (auto const& line : ignored.lines())
{
std::string const trimmed = boost::algorithm::trim_copy(line);
if (trimmed.empty())
continue;
std::vector<std::string> parts;
boost::algorithm::split(
parts,
trimmed,
boost::algorithm::is_space(),
boost::algorithm::token_compress_on);
uint256 id;
std::string name;
if (id.parseHex(parts[0]))
name = parts.size() > 1 ? parts[1] : std::string{};
else if (auto const f = getRegisteredFeature(parts[0]))
{
id = *f;
name = parts[0];
}
else
Throw<std::runtime_error>(
"Invalid entry '" + line + "' in [" + ignored.name() +
"]: expected a registered feature name or amendment ID");
AmendmentState& s = add(id, lock);
if (s.supported)
{
JLOG(j_.warn()) << "[" << ignored.name() << "] lists amendment "
<< id << " (" << s.name
<< ") which this server supports; entry has no "
"effect.";
continue;
}
if (!name.empty())
s.name = name;
s.supported = true;
s.ignored = true;
s.vote = AmendmentVote::obsolete;
JLOG(j_.warn()) << "Amendment " << id << " (" << s.name
<< ") is unsupported but listed in [" << ignored.name()
<< "]: this server will NOT become amendment blocked "
"if it is enabled on the ledger.";
}
hash_set<uint256> detect_conflict;
// Parse enabled amendments from config
for (std::pair<uint256, std::string> const& a : parseSection(enabled))
@@ -775,7 +839,12 @@ AmendmentTableImpl::enable(uint256 const& amendment)
s.enabled = true;
if (!s.supported)
if (s.ignored)
{
JLOG(j_.warn()) << "Ignored unsupported amendment " << amendment << " ("
<< s.name << ") is enabled on the ledger.";
}
else if (!s.supported)
{
JLOG(j_.error()) << "Unsupported amendment " << amendment
<< " activated.";
@@ -1044,7 +1113,9 @@ AmendmentTableImpl::injectJson(
if (!fs.name.empty())
v[jss::name] = fs.name;
v[jss::supported] = fs.supported;
v[jss::supported] = fs.supported && !fs.ignored;
if (fs.ignored)
v["ignored"] = true;
if (!fs.enabled && isAdmin)
{
if (fs.vote == AmendmentVote::obsolete)
@@ -1113,10 +1184,11 @@ make_AmendmentTable(
std::vector<AmendmentTable::FeatureInfo> const& supported,
Section const& enabled,
Section const& vetoed,
beast::Journal journal)
beast::Journal journal,
Section const& ignored)
{
return std::make_unique<AmendmentTableImpl>(
app, majorityTime, supported, enabled, vetoed, journal);
app, majorityTime, supported, enabled, vetoed, journal, ignored);
}
} // namespace ripple
+28 -20
View File
@@ -258,42 +258,50 @@ saveValidatedLedger(
ledger->info().txHash == ledger->txMap().getHash().as_uint256(),
"ripple::detail::saveValidatedLedger : transaction hash match");
bool const pinned = app.getLedgerMaster().isPinned(seq);
// Save the ledger header in the hashed object store
{
Serializer s(128);
s.add32(HashPrefix::ledgerMaster);
addRaw(ledger->info(), s);
// Use pinnedLEDGER type for pinned ledgers, hotLEDGER for others
auto ledgerType =
app.getLedgerMaster().isPinned(seq) ? pinnedLEDGER : hotLEDGER;
auto ledgerType = pinned ? pinnedLEDGER : hotLEDGER;
app.getNodeStore().store(
ledgerType, std::move(s.modData()), ledger->info().hash, seq);
}
// Pinned (catalogue) ledgers are saved in bulk: they are never cached
// or published from here, so their AcceptedLedger skips the per-tx JSON
// and is not built at all when there are no tx tables to fill.
std::shared_ptr<AcceptedLedger> aLedger;
try
if (!pinned || app.config().useTxTables())
{
aLedger = app.getAcceptedLedgerCache().fetch(ledger->info().hash);
if (!aLedger)
try
{
aLedger = std::make_shared<AcceptedLedger>(ledger, app);
// Only cache if the ledger is NOT in the pinned range
if (!app.getLedgerMaster().isPinned(ledger->info().seq))
aLedger = app.getAcceptedLedgerCache().fetch(ledger->info().hash);
if (!aLedger)
{
app.getAcceptedLedgerCache().canonicalize_replace_client(
ledger->info().hash, aLedger);
aLedger = std::make_shared<AcceptedLedger>(
ledger, app, /*withJson=*/!pinned);
// Only cache if the ledger is NOT in the pinned range
if (!pinned)
{
app.getAcceptedLedgerCache().canonicalize_replace_client(
ledger->info().hash, aLedger);
}
}
}
}
catch (std::exception const&)
{
JLOG(j.warn()) << "An accepted ledger was missing nodes";
app.getLedgerMaster().failedSave(seq, ledger->info().hash);
// Clients can now trust the database for information about this
// ledger sequence.
app.pendingSaves().finishWork(seq);
return false;
catch (std::exception const&)
{
JLOG(j.warn()) << "An accepted ledger was missing nodes";
app.getLedgerMaster().failedSave(seq, ledger->info().hash);
// Clients can now trust the database for information about this
// ledger sequence.
app.pendingSaves().finishWork(seq);
return false;
}
}
{
+3 -2
View File
@@ -158,8 +158,9 @@ ClaimReward::preclaim(PreclaimContext const& ctx)
auto const& hookOn =
hook::getHookOn(hook, sleDef, sfHookOnIncoming);
auto const hookName = ctx.view.rules().enabled(fix20260929)
? hook[~sfHookName]
auto const hookName = ctx.view.rules().enabled(fix20260929) &&
hook.isFieldPresent(sfHookName)
? std::optional<Blob>(hook.getFieldVL(sfHookName))
: std::nullopt;
if (hook::canHook(ctx.tx, hookOn, hookName))
{
+9 -1
View File
@@ -1740,7 +1740,15 @@ SetHook::setHook()
{
if (newHookName->size() == 0)
{
newHook.makeFieldAbsent(sfHookName);
// Erase the field outright rather than
// makeFieldAbsent: newHook carries no template, so
// makeFieldAbsent leaves an STI_NOTPRESENT
// placeholder in the in-memory entry that every
// reader must then special-case, and it throws
// (-> tefEXCEPTION) when the hook has no name to
// remove. delField matches the serialized form of
// the entry and is a no-op on a nameless hook.
newHook.delField(sfHookName);
}
else
{
+6 -2
View File
@@ -287,7 +287,9 @@ Transactor::calculateHookChainFee(
// LCOV_EXCL_STOP
}
auto const hookName = hookObj[~sfHookName];
std::optional<Blob> const hookName = hookObj.isFieldPresent(sfHookName)
? std::optional<Blob>(hookObj.getFieldVL(sfHookName))
: std::nullopt;
uint32_t flags = 0;
if (hookObj.isFieldPresent(sfFlags))
@@ -1409,7 +1411,9 @@ Transactor::executeHookChain(
// LCOV_EXCL_STOP
}
auto const hookName = hookObj[~sfHookName];
std::optional<Blob> const hookName = hookObj.isFieldPresent(sfHookName)
? std::optional<Blob>(hookObj.getFieldVL(sfHookName))
: std::nullopt;
// check if the hook can fire
uint256 hookOn = hook::getHookOn(
+15 -55
View File
@@ -18,61 +18,21 @@
//==============================================================================
#include <xrpld/app/tx/applySteps.h>
#include <xrpld/app/tx/detail/AMMBid.h>
#include <xrpld/app/tx/detail/AMMClawback.h>
#include <xrpld/app/tx/detail/AMMCreate.h>
#include <xrpld/app/tx/detail/AMMDelete.h>
#include <xrpld/app/tx/detail/AMMDeposit.h>
#include <xrpld/app/tx/detail/AMMVote.h>
#include <xrpld/app/tx/detail/AMMWithdraw.h>
#include <xrpld/app/tx/detail/ApplyContext.h>
#include <xrpld/app/tx/detail/CancelCheck.h>
#include <xrpld/app/tx/detail/CancelOffer.h>
#include <xrpld/app/tx/detail/CashCheck.h>
#include <xrpld/app/tx/detail/Change.h>
#include <xrpld/app/tx/detail/ClaimReward.h>
#include <xrpld/app/tx/detail/Clawback.h>
#include <xrpld/app/tx/detail/CreateCheck.h>
#include <xrpld/app/tx/detail/CreateOffer.h>
#include <xrpld/app/tx/detail/CreateTicket.h>
#include <xrpld/app/tx/detail/Credentials.h>
#include <xrpld/app/tx/detail/Cron.h>
#include <xrpld/app/tx/detail/CronSet.h>
#include <xrpld/app/tx/detail/DID.h>
#include <xrpld/app/tx/detail/DeleteAccount.h>
#include <xrpld/app/tx/detail/DeleteOracle.h>
#include <xrpld/app/tx/detail/DepositPreauth.h>
#include <xrpld/app/tx/detail/Escrow.h>
#include <xrpld/app/tx/detail/GenesisMint.h>
#include <xrpld/app/tx/detail/Import.h>
#include <xrpld/app/tx/detail/Invoke.h>
#include <xrpld/app/tx/detail/LedgerStateFix.h>
#include <xrpld/app/tx/detail/MPTokenAuthorize.h>
#include <xrpld/app/tx/detail/MPTokenIssuanceCreate.h>
#include <xrpld/app/tx/detail/MPTokenIssuanceDestroy.h>
#include <xrpld/app/tx/detail/MPTokenIssuanceSet.h>
#include <xrpld/app/tx/detail/NFTokenAcceptOffer.h>
#include <xrpld/app/tx/detail/NFTokenBurn.h>
#include <xrpld/app/tx/detail/NFTokenCancelOffer.h>
#include <xrpld/app/tx/detail/NFTokenCreateOffer.h>
#include <xrpld/app/tx/detail/NFTokenMint.h>
#include <xrpld/app/tx/detail/NFTokenModify.h>
#include <xrpld/app/tx/detail/PayChan.h>
#include <xrpld/app/tx/detail/Payment.h>
#include <xrpld/app/tx/detail/PermissionedDomainDelete.h>
#include <xrpld/app/tx/detail/PermissionedDomainSet.h>
#include <xrpld/app/tx/detail/Remit.h>
#include <xrpld/app/tx/detail/SetAccount.h>
#include <xrpld/app/tx/detail/SetHook.h>
#include <xrpld/app/tx/detail/SetManifest.h>
#include <xrpld/app/tx/detail/SetOracle.h>
#include <xrpld/app/tx/detail/SetRegularKey.h>
#include <xrpld/app/tx/detail/SetRemarks.h>
#include <xrpld/app/tx/detail/SetSignerList.h>
#include <xrpld/app/tx/detail/SetTrust.h>
#include <xrpld/app/tx/detail/URIToken.h>
#include <xrpld/app/tx/detail/XChainBridge.h>
#include <xrpld/app/tx/detail/XahauGenesis.h>
#pragma push_macro("TRANSACTION")
#undef TRANSACTION
// Do nothing
#define TRANSACTION(...)
#define TRANSACTION_INCLUDE 1
#include <xrpl/protocol/detail/transactions.macro>
#undef TRANSACTION
#pragma pop_macro("TRANSACTION")
// DO NOT INCLUDE TRANSACTOR HEADER FILES HERE.
// See the instructions at the top of transactions.macro instead.
#include <xrpl/protocol/TxFormats.h>
#include <stdexcept>
+3
View File
@@ -102,6 +102,9 @@ struct ConfigSection
#define SECTION_VALIDATORS "validators"
#define SECTION_VALIDATOR_TOKEN "validator_token"
#define SECTION_VETO_AMENDMENTS "veto_amendments"
// Testnet-only: amendments enabled on-ledger that this node should not treat
// as unsupported (i.e. should not become amendment blocked over).
#define SECTION_IGNORE_AMENDMENTS "ignore_amendments"
#define SECTION_WORKERS "workers"
} // namespace ripple
+5 -1
View File
@@ -405,7 +405,11 @@ Config::setup(
HTTPClient::initializeSSLContext(*this, j_);
if (RUN_STANDALONE)
// Standalone never fetches history, so ledger_history is normally
// meaningless there. The rwdb null node store (nullBackend) uses it
// to size the in-memory retained-ledger window, and SHAMapStoreImp
// rejects zero for it, so keep the configured value in that case.
if (RUN_STANDALONE && !nullBackend())
LEDGER_HISTORY = 0;
std::string ledgerTxDbType;
+17
View File
@@ -851,6 +851,22 @@ private:
return jvRequest;
}
// catalogue_unpin <ledger_index_min> <ledger_index_max>
Json::Value
parseCatalogueUnpin(Json::Value const& jvParams)
{
Json::Value jvRequest(Json::objectValue);
if (jvParams.size() >= 2)
{
// asUInt() throws on non-numeric input rather than yielding 0
jvRequest[jss::ledger_index_min] = jvParams[0u].asUInt();
jvRequest[jss::ledger_index_max] = jvParams[1u].asUInt();
}
return jvRequest;
}
// catalogue_status - no parameters required
Json::Value
parseCatalogueStatus(Json::Value const& jvParams)
@@ -1470,6 +1486,7 @@ public:
{"catalogue_create", &RPCParser::parseCatalogueCreate, 3, 4},
{"catalogue_load", &RPCParser::parseCatalogueLoad, 1, 2},
{"catalogue_status", &RPCParser::parseCatalogueStatus, 0, 0},
{"catalogue_unpin", &RPCParser::parseCatalogueUnpin, 2, 2},
{"channel_authorize", &RPCParser::parseChannelAuthorize, 3, 4},
{"channel_verify", &RPCParser::parseChannelVerify, 4, 4},
{"connect", &RPCParser::parseConnect, 1, 2},
+16
View File
@@ -121,6 +121,22 @@ public:
virtual void
sync() = 0;
/** Make every object previously passed to store() or storeBatch()
durable before returning.
@return true if, on return, those writes are as durable as this
backend ever makes them (trivially so for in-memory
backends); false if the backend cannot do this on demand and
they become durable later on its own schedule (e.g. NuDB's
commit thread).
*/
virtual bool
syncDurable()
{
sync();
return false;
}
/** Visit every object in the database
This is usually called during import.
@note This routine will not be called concurrently with itself
@@ -186,6 +186,13 @@ public:
{
}
bool
syncDurable() override
{
// Nothing here outlives the process, so there is nothing to wait for.
return true;
}
void
for_each(std::function<void(std::shared_ptr<NodeObject>)> f) override
{
@@ -291,6 +291,9 @@ public:
void
sync() override
{
// NuDB commits on its context thread (about once a second) and
// offers no public barrier while that thread runs, so this cannot
// force durability; syncDurable() therefore reports false.
}
void
@@ -81,6 +81,13 @@ public:
{
}
bool
syncDurable() override
{
// Nothing here outlives the process, so there is nothing to wait for.
return true;
}
void
for_each(std::function<void(std::shared_ptr<NodeObject>)> f) override
{
@@ -164,6 +164,13 @@ public:
{
}
bool
syncDurable() override
{
// Nothing here outlives the process, so there is nothing to wait for.
return true;
}
void
for_each(std::function<void(std::shared_ptr<NodeObject>)> f) override
{
@@ -383,6 +383,33 @@ public:
void
sync() override
{
(void)syncDurable();
}
bool
syncDurable() override
{
// Drain anything queued through store() into RocksDB, then flush
// and fsync the WAL so every write so far survives a crash.
m_batch.waitForWriting();
if (!m_db)
return false;
auto const ret = m_db->FlushWAL(/*sync=*/true);
if (ret.ok())
return true;
// Some user-supplied options (e.g. allow_mmap_writes) make WAL sync
// unavailable. Report "not durable on demand" rather than failing:
// callers then fall back to the non-durable checkpoint path.
if (ret.IsNotSupported())
{
JLOG(m_journal.warn())
<< "RocksDB cannot sync its WAL on demand: " << ret.ToString();
return false;
}
Throw<std::runtime_error>("sync failed: " + ret.ToString());
}
void
+5 -2
View File
@@ -74,13 +74,16 @@ public:
int
getWriteLoad();
/** Block until every object passed to store() so far has been handed
to the callback's writeBatch(). */
void
waitForWriting();
private:
void
performScheduledTask() override;
void
writeBatch();
void
waitForWriting();
private:
using LockType = std::recursive_mutex;
@@ -91,6 +91,36 @@ DatabasePinnedImp::store(
}
}
void
DatabasePinnedImp::storePinnedBatch(Batch const& batch)
{
if (batch.empty())
return;
std::uint64_t bytes = 0;
for (auto const& obj : batch)
{
XRPL_ASSERT(
obj && !isPinnedType(obj->getType()),
"ripple::NodeStore::DatabasePinnedImp::storePinnedBatch : "
"serializable type");
bytes += obj->getData().size();
}
// The persistent backends allowed for pinned_type (NuDB, RocksDB, and
// rwdb/memory in standalone) are safe to write from several threads,
// so this may overlap a BatchWriter flush of single store() calls.
persistent_->storeBatch(batch);
pinnedStoreCount_ += batch.size();
storeStats(batch.size(), bytes);
}
bool
DatabasePinnedImp::syncPinned()
{
return persistent_->syncDurable();
}
std::shared_ptr<NodeObject>
DatabasePinnedImp::fetchNodeObject(
uint256 const& hash,
@@ -112,7 +142,11 @@ DatabasePinnedImp::fetchNodeObject(
{
// Try persistent backend first (pinned data)
if (auto obj = tryPersistent(hash, fetchReport))
{
if (duplicate)
copyToRotating(*obj, hash, ledgerSeq);
return obj;
}
// Fallback: try rotating backends
return rotating_.fetchNodeObject(
@@ -130,7 +164,30 @@ DatabasePinnedImp::fetchNodeObject(
return obj;
// Fallback: try persistent backend (pinned data)
return tryPersistent(hash, fetchReport);
auto obj = tryPersistent(hash, fetchReport);
// `duplicate` is set by online_delete's copy of the current validated
// state into the new writable backend. A hot ledger can share nodes
// that exist only in persistent storage (e.g. after catalogue_load
// switched the LCL to its last ledger); copy those too, so the hot
// store becomes self-contained after one rotation and pinned history
// can be unpinned and its store discarded without breaking live state.
// The pinned-first path above does the same, so the copy happens
// whichever backend is tried first.
if (obj && duplicate)
copyToRotating(*obj, hash, ledgerSeq);
return obj;
}
void
DatabasePinnedImp::copyToRotating(
NodeObject const& obj,
uint256 const& hash,
std::uint32_t ledgerSeq)
{
Blob data(obj.getData());
rotating_.store(obj.getType(), std::move(data), hash, ledgerSeq);
}
void
@@ -262,8 +319,8 @@ DatabasePinnedImp::tryPersistent(uint256 const& hash, FetchReport& fetchReport)
if (status == ok && nodeObject)
{
fetchReport.wasFound = true;
// Note: We do NOT copy pinned data to rotating storage even if
// duplicate=true. Pinned data stays in persistent storage.
// Copying into rotating storage for duplicate=true is handled by
// the caller (see fetchNodeObject's default path).
}
return nodeObject;
}
@@ -121,6 +121,20 @@ public:
std::uint32_t ledgerSeq) override;
void
sync() override;
/** Write node objects of pinned history straight to the persistent
backend in one batch. Objects must carry hot (serializable) types.
Used by catalogue_load, which builds whole batches per ledger.
*/
void
storePinnedBatch(Batch const& batch);
/** Make everything written to the persistent backend durable.
@return false if the backend cannot guarantee that on demand.
*/
bool
syncPinned();
bool
storeLedger(std::shared_ptr<Ledger const> const& srcLedger) override;
void
@@ -141,6 +155,14 @@ private:
bool
likelyPinned(std::uint32_t ledgerSeq) const;
/** Copy an object found only in persistent storage into the writable
rotating backend (the `duplicate` path of fetchNodeObject). */
void
copyToRotating(
NodeObject const& obj,
uint256 const& hash,
std::uint32_t ledgerSeq);
std::shared_ptr<NodeObject>
tryPersistent(uint256 const& hash, FetchReport& fetchReport);
+65 -13
View File
@@ -213,13 +213,19 @@ deserializeSHAMapFromStream(
CatalogueInputStream& stream,
SHAMapNodeType nodeType,
NodeObjectType flushType,
SHAMap::FlushSink const* sink,
bool allowRemoval,
beast::Journal const& j)
{
try
{
auto deserializeLeaf = [&shaMap, &stream, &j, nodeType, allowRemoval](
SHAMapNodeType& parsedType) -> bool {
// Reused across leaves: resize() keeps capacity, so steady-state
// parsing does no per-leaf allocation beyond the SHAMapItem itself.
std::vector<std::uint8_t> data;
auto deserializeLeaf =
[&shaMap, &stream, &j, &data, nodeType, allowRemoval](
SHAMapNodeType& parsedType) -> bool {
stream.read(reinterpret_cast<char*>(&parsedType), 1);
if (parsedType == SHAMapNodeType::tnTERMINAL)
@@ -248,7 +254,8 @@ deserializeSHAMapFromStream(
return false;
}
if (!shaMap.hasItem(key))
// delItem walks once and reports absence itself.
if (!shaMap.delItem(key))
{
JLOG(j.error())
<< "Deserialization: removal of key " << to_string(key)
@@ -256,7 +263,6 @@ deserializeSHAMapFromStream(
return false;
}
shaMap.delItem(key);
return true;
}
@@ -279,7 +285,7 @@ deserializeSHAMapFromStream(
return false;
}
std::vector<std::uint8_t> data(size);
data.resize(size);
stream.read(reinterpret_cast<char*>(data.data()), size);
if (stream.fail())
@@ -290,11 +296,8 @@ deserializeSHAMapFromStream(
return false;
}
auto item = make_shamapitem(key, makeSlice(data));
if (shaMap.hasItem(key))
return shaMap.updateGiveItem(nodeType, std::move(item));
return shaMap.addGiveItem(nodeType, std::move(item));
auto item = make_shamapitem(key, Slice(data.data(), size));
return shaMap.upsertGiveItem(nodeType, std::move(item));
};
auto lastParsed = SHAMapNodeType::tnREMOVE;
@@ -308,7 +311,10 @@ deserializeSHAMapFromStream(
return false;
}
shaMap.flushDirty(flushType);
if (sink)
shaMap.flushDirty(flushType, *sink);
else
shaMap.flushDirty(flushType);
return true;
}
catch (std::exception const& e)
@@ -345,7 +351,13 @@ deserializeStateMapFromStream(
beast::Journal const& j)
{
return deserializeSHAMapFromStream(
stateMap, stream, SHAMapNodeType::tnACCOUNT_STATE, flushType, true, j);
stateMap,
stream,
SHAMapNodeType::tnACCOUNT_STATE,
flushType,
nullptr,
true,
j);
}
bool
@@ -356,7 +368,47 @@ deserializeTxMapFromStream(
beast::Journal const& j)
{
return deserializeSHAMapFromStream(
txMap, stream, SHAMapNodeType::tnTRANSACTION_MD, flushType, false, j);
txMap,
stream,
SHAMapNodeType::tnTRANSACTION_MD,
flushType,
nullptr,
false,
j);
}
bool
deserializeStateMapFromStream(
SHAMap& stateMap,
CatalogueInputStream& stream,
SHAMap::FlushSink const& sink,
beast::Journal const& j)
{
return deserializeSHAMapFromStream(
stateMap,
stream,
SHAMapNodeType::tnACCOUNT_STATE,
hotACCOUNT_NODE,
&sink,
true,
j);
}
bool
deserializeTxMapFromStream(
SHAMap& txMap,
CatalogueInputStream& stream,
SHAMap::FlushSink const& sink,
beast::Journal const& j)
{
return deserializeSHAMapFromStream(
txMap,
stream,
SHAMapNodeType::tnTRANSACTION_MD,
hotTRANSACTION_NODE,
&sink,
false,
j);
}
} // namespace RPC
+18
View File
@@ -59,6 +59,24 @@ deserializeTxMapFromStream(
NodeObjectType flushType,
beast::Journal const& j);
/** As above, but dirty nodes are flushed to `sink` (tagged with
hotACCOUNT_NODE / hotTRANSACTION_NODE) instead of the nodestore, and are
not canonicalized into the TreeNodeCache. See SHAMap::flushDirty(t, sink).
*/
bool
deserializeStateMapFromStream(
SHAMap& stateMap,
CatalogueInputStream& stream,
SHAMap::FlushSink const& sink,
beast::Journal const& j);
bool
deserializeTxMapFromStream(
SHAMap& txMap,
CatalogueInputStream& stream,
SHAMap::FlushSink const& sink,
beast::Journal const& j);
} // namespace RPC
} // namespace ripple
+1
View File
@@ -199,6 +199,7 @@ Handler const handlerArray[]{
{"catalogue_create", byRef(&doCatalogueCreate), Role::ADMIN, NO_CONDITION},
{"catalogue_status", byRef(&doCatalogueStatus), Role::ADMIN, NO_CONDITION},
{"catalogue_load", byRef(&doCatalogueLoad), Role::ADMIN, NO_CONDITION},
{"catalogue_unpin", byRef(&doCatalogueUnpin), Role::ADMIN, NO_CONDITION},
};
class HandlerTable
File diff suppressed because it is too large. Load diff
+2
View File
@@ -184,6 +184,8 @@ Json::Value
doCatalogueStatus(RPC::JsonContext&);
Json::Value
doCatalogueLoad(RPC::JsonContext&);
Json::Value
doCatalogueUnpin(RPC::JsonContext&);
} // namespace ripple
#endif
+44 -3
View File
@@ -36,6 +36,7 @@
#include <xrpl/beast/utility/instrumentation.h>
#include <boost/iostreams/filtering_stream.hpp>
#include <cassert>
#include <functional>
#include <stack>
#include <vector>
@@ -212,6 +213,19 @@ public:
SHAMapNodeType type,
boost::intrusive_ptr<SHAMapItem const> item);
/** Replace the item with the same key if present, otherwise add it.
Equivalent to `hasItem(key) ? updateGiveItem(..) : addGiveItem(..)`
but walks the tree once when the key already exists.
@return false if the key exists with a different leaf type, or if
the add fails.
*/
bool
upsertGiveItem(
SHAMapNodeType type,
boost::intrusive_ptr<SHAMapItem const> item);
// Save a copy if you need to extend the life
// of the SHAMapItem beyond this SHAMap
boost::intrusive_ptr<SHAMapItem const> const&
@@ -367,6 +381,26 @@ public:
int
flushDirty(NodeObjectType t);
/** Receives each node written by flushDirty(t, sink): the type passed
to flushDirty, the node serialized with its hash prefix, and its
hash.
*/
using FlushSink =
std::function<void(NodeObjectType, Blob&&, uint256 const&)>;
/** Flush modified nodes to `sink` instead of the nodestore and convert
them to shared.
Unlike flushDirty(t), written nodes are NOT canonicalized into the
family's TreeNodeCache. Bulk writers of history (catalogue_load)
must use this: every cached inner node holds strong references to
its children, so canonicalizing pins every superseded node version
in memory until the cache sweeps it, which on a fast import grows
without practical bound.
*/
int
flushDirty(NodeObjectType t, FlushSink const& sink);
void
walkMap(std::vector<SHAMapMissingNode>& missingNodes, int maxMissing) const;
bool
@@ -449,9 +483,13 @@ private:
std::shared_ptr<Node>
preFlushNode(std::shared_ptr<Node> node) const;
/** write and canonicalize modified node */
/** write and canonicalize modified node; if `sink` is set, hand the
serialized node to it instead and do not canonicalize */
std::shared_ptr<SHAMapTreeNode>
writeNode(NodeObjectType t, std::shared_ptr<SHAMapTreeNode> node) const;
writeNode(
NodeObjectType t,
std::shared_ptr<SHAMapTreeNode> node,
FlushSink const* sink = nullptr) const;
// returns the first item at or below this node
SHAMapLeafNode*
@@ -534,7 +572,10 @@ private:
Delta& differences,
int& maxCount) const;
int
walkSubTree(bool doWrite, NodeObjectType t);
walkSubTree(
bool doWrite,
NodeObjectType t,
FlushSink const* sink = nullptr);
// Structure to track information about call to
// getMissingNodes while it's in progress
+65 -6
View File
@@ -1059,6 +1059,42 @@ SHAMap::updateGiveItem(
return true;
}
bool
SHAMap::upsertGiveItem(
SHAMapNodeType type,
boost::intrusive_ptr<SHAMapItem const> item)
{
XRPL_ASSERT(
state_ != SHAMapState::Immutable,
"ripple::SHAMap::upsertGiveItem : not immutable");
uint256 const tag = item->key();
SharedPtrNodeStack stack;
auto const leaf = walkTowardsKey(tag, &stack);
if (!leaf || leaf->peekItem()->key() != tag)
return addGiveItem(type, std::move(item));
// From here this mirrors updateGiveItem, reusing the walk above.
auto node = std::static_pointer_cast<SHAMapLeafNode>(stack.top().first);
auto const nodeID = stack.top().second;
stack.pop();
if (node->getType() != type)
{
JLOG(journal_.fatal()) << "SHAMap::upsertGiveItem: cross-type change!";
return false;
}
node = unshareNode(std::move(node), nodeID);
if (node->setItem(std::move(item)))
dirtyUp(stack, tag, node);
return true;
}
bool
SHAMap::fetchRoot(SHAMapHash const& hash, SHAMapSyncFilter* filter)
{
@@ -1108,10 +1144,24 @@ SHAMap::fetchRoot(SHAMapHash const& hash, SHAMapSyncFilter* filter)
first call SHAMapTreeNode::unshare().
*/
std::shared_ptr<SHAMapTreeNode>
SHAMap::writeNode(NodeObjectType t, std::shared_ptr<SHAMapTreeNode> node) const
SHAMap::writeNode(
NodeObjectType t,
std::shared_ptr<SHAMapTreeNode> node,
FlushSink const* sink) const
{
XRPL_ASSERT(
node->cowid() == 0, "ripple::SHAMap::writeNode : valid input node");
if (sink)
{
// Bulk-write path: no TreeNodeCache, no direct store. The caller
// owns persistence (see flushDirty(t, sink)).
Serializer s;
node->serializeWithPrefix(s);
(*sink)(t, std::move(s.modData()), node->getHash().as_uint256());
return node;
}
XRPL_ASSERT(backed_, "ripple::SHAMap::writeNode : is backed");
canonicalize(node->getHash(), node);
@@ -1159,10 +1209,19 @@ SHAMap::flushDirty(NodeObjectType t)
}
int
SHAMap::walkSubTree(bool doWrite, NodeObjectType t)
SHAMap::flushDirty(NodeObjectType t, FlushSink const& sink)
{
// The sink decides where nodes go, so this writes whether or not the
// map is backed.
return walkSubTree(true, t, &sink);
}
int
SHAMap::walkSubTree(bool doWrite, NodeObjectType t, FlushSink const* sink)
{
XRPL_ASSERT(
!doWrite || backed_, "ripple::SHAMap::walkSubTree : valid input");
!doWrite || backed_ || sink,
"ripple::SHAMap::walkSubTree : valid input");
int flushed = 0;
@@ -1176,7 +1235,7 @@ SHAMap::walkSubTree(bool doWrite, NodeObjectType t)
root_->unshare();
if (doWrite)
root_ = writeNode(t, std::move(root_));
root_ = writeNode(t, std::move(root_), sink);
return 1;
}
@@ -1245,7 +1304,7 @@ SHAMap::walkSubTree(bool doWrite, NodeObjectType t)
child->unshare();
if (doWrite)
child = writeNode(t, std::move(child));
child = writeNode(t, std::move(child), sink);
node->shareChild(branch, child);
}
@@ -1261,7 +1320,7 @@ SHAMap::walkSubTree(bool doWrite, NodeObjectType t)
if (doWrite)
node = std::static_pointer_cast<SHAMapInnerNode>(
writeNode(t, std::move(node)));
writeNode(t, std::move(node), sink));
++flushed;