Compare commits

...

3 Commits

Author SHA1 Message Date
Richard Holland
4221267dce debug acctx tests under release builder 2025-02-04 16:50:04 +11:00
RichardAH
317bd4bc6e add strict filtering to account_tx api (#429) 2025-02-03 17:56:08 +10:00
RichardAH
2fd465bb3f fix20250131 (#428)
Co-authored-by: Denis Angell <dangell@transia.co>
2025-02-03 10:33:19 +10:00
16 changed files with 354 additions and 39 deletions

View File

@@ -3,6 +3,7 @@
#include <iostream>
#include <map>
#include <memory>
#include <optional>
#include <ostream>
#include <stack>
#include <string>
@@ -271,7 +272,19 @@ check_guard(
int guard_func_idx,
int last_import_idx,
GuardLog guardLog,
std::string guardLogAccStr)
std::string guardLogAccStr,
/* RH NOTE:
* rules version is a bit field, so rule update 1 is 0x01, update 2 is 0x02
* and update 3 is 0x04 ideally at rule version 3 all bits so far are set
* (0b111) so the ruleVersion = 7, however if a specific rule update must be
* rolled back due to unforeseen behaviour then this may no longer be the
* case. using a bit field here leaves us flexible to rollback changes that
* might have unforeseen consequences, without also rolling back further
* changes that are fine.
*/
uint64_t rulesVersion = 0
)
{
#define MAX_GUARD_CALLS 1024
uint32_t guard_count = 0;
@@ -621,11 +634,17 @@ check_guard(
}
else if (fc_type == 10) // memory.copy
{
if (rulesVersion & 0x02U)
GUARD_ERROR("Memory.copy instruction is not allowed.");
REQUIRE(2);
ADVANCE(2);
}
else if (fc_type == 11) // memory.fill
{
if (rulesVersion & 0x02U)
GUARD_ERROR("Memory.fill instruction is not allowed.");
ADVANCE(1);
}
else if (fc_type <= 7) // numeric instructions
@@ -807,6 +826,15 @@ validateGuards(
std::vector<uint8_t> const& wasm,
GuardLog guardLog,
std::string guardLogAccStr,
/* RH NOTE:
* rules version is a bit field, so rule update 1 is 0x01, update 2 is 0x02
* and update 3 is 0x04 ideally at rule version 3 all bits so far are set
* (0b111) so the ruleVersion = 7, however if a specific rule update must be
* rolled back due to unforeseen behaviour then this may no longer be the
* case. using a bit field here leaves us flexible to rollback changes that
* might have unforeseen consequences, without also rolling back further
* changes that are fine.
*/
uint64_t rulesVersion = 0)
{
uint64_t byteCount = wasm.size();
@@ -1477,7 +1505,8 @@ validateGuards(
guard_import_number,
last_import_number,
guardLog,
guardLogAccStr);
guardLogAccStr,
rulesVersion);
if (!valid)
return {};

View File

@@ -79,7 +79,7 @@ main(int argc, char** argv)
close(fd);
auto result = validateGuards(hook, std::cout, "", 1);
auto result = validateGuards(hook, std::cout, "", 3);
if (!result)
{

View File

@@ -69,6 +69,7 @@ public:
std::uint32_t offset;
std::uint32_t limit;
bool bUnlimited;
bool strict;
};
struct AccountTxPageOptions
@@ -79,6 +80,7 @@ public:
std::optional<AccountTxMarker> marker;
std::uint32_t limit;
bool bAdmin;
bool strict;
};
using AccountTx =
@@ -101,6 +103,7 @@ public:
bool forward = false;
uint32_t limit = 0;
std::optional<AccountTxMarker> marker;
bool strict;
};
struct AccountTxResult

View File

@@ -43,6 +43,62 @@ private:
std::map<uint256, AccountTx> transactionMap_;
std::map<AccountID, AccountTxData> accountTxMap_;
// helper function to scan for an account ID inside the tx and meta blobs
// used for strict filtering of account_tx
bool
isAccountInvolvedInTx(AccountID const& account, AccountTx const& accountTx)
{
auto const& txn = accountTx.first;
auto const& meta = accountTx.second;
// Search metadata, excluding RegularKey false positives
Blob const metaBlob = meta->getAsObject().getSerializer().peekData();
if (metaBlob.size() >= account.size())
{
auto it = metaBlob.begin();
while (true)
{
// Find next occurrence of account
it = std::search(
it,
metaBlob.end(),
account.data(),
account.data() + account.size());
if (it == metaBlob.end())
break;
// Check if this is a RegularKey field (0x8814 prefix)
if (it >= metaBlob.begin() + 2)
{
auto prefix = *(it - 2);
auto prefix2 = *(it - 1);
if (prefix != 0x88 || prefix2 != 0x14)
{
// Found account not preceded by RegularKey prefix
return true;
}
}
else
{
// Too close to start to be RegularKey
return true;
}
++it; // Move past this occurrence
}
}
// Search transaction blob
Blob const txnBlob = txn->getSTransaction()->getSerializer().peekData();
return txnBlob.size() >= account.size() &&
std::search(
txnBlob.begin(),
txnBlob.end(),
account.data(),
account.data() + account.size()) != txnBlob.end();
}
public:
RWDBDatabase(Application& app, Config const& config, JobQueue& jobQueue)
: app_(app), useTxTables_(config.useTxTables())
@@ -193,7 +249,17 @@ public:
std::size_t count = 0;
for (const auto& [_, accountData] : accountTxMap_)
{
count += accountData.transactions.size();
for (const auto& tx : accountData.transactions)
{
// RH NOTE: options isn't provided to this function
// but this function is probably only used internally
// so make it reflect the true number (unfiltered)
// if (options.strict &&
// !isAccountInvolvedInTx(options.account, tx))
// continue;
count++;
}
}
return count;
}
@@ -607,12 +673,17 @@ public:
{
for (const auto& [txSeq, txIndex] : txIt->second)
{
AccountTx const accountTx = accountData.transactions[txIndex];
if (options.strict &&
!isAccountInvolvedInTx(options.account, accountTx))
continue;
if (skipped < options.offset)
{
++skipped;
continue;
}
AccountTx const accountTx = accountData.transactions[txIndex];
std::uint32_t const inLedger = rangeCheckedCast<std::uint32_t>(
accountTx.second->getLgrSeq());
accountTx.first->setStatus(COMMITTED);
@@ -652,13 +723,18 @@ public:
innerRIt != rIt->second.rend();
++innerRIt)
{
AccountTx const accountTx =
accountData.transactions[innerRIt->second];
if (options.strict &&
!isAccountInvolvedInTx(options.account, accountTx))
continue;
if (skipped < options.offset)
{
++skipped;
continue;
}
AccountTx const accountTx =
accountData.transactions[innerRIt->second];
std::uint32_t const inLedger = rangeCheckedCast<std::uint32_t>(
accountTx.second->getLgrSeq());
accountTx.first->setLedger(inLedger);
@@ -694,12 +770,19 @@ public:
{
for (const auto& [txSeq, txIndex] : txIt->second)
{
AccountTx const accountTx = accountData.transactions[txIndex];
if (options.strict &&
!isAccountInvolvedInTx(options.account, accountTx))
continue;
const auto& [txn, txMeta] = accountTx;
if (skipped < options.offset)
{
++skipped;
continue;
}
const auto& [txn, txMeta] = accountData.transactions[txIndex];
result.emplace_back(
txn->getSTransaction()->getSerializer().peekData(),
txMeta->getAsObject().getSerializer().peekData(),
@@ -738,13 +821,20 @@ public:
innerRIt != rIt->second.rend();
++innerRIt)
{
AccountTx const accountTx =
accountData.transactions[innerRIt->second];
if (options.strict &&
!isAccountInvolvedInTx(options.account, accountTx))
continue;
const auto& [txn, txMeta] = accountTx;
if (skipped < options.offset)
{
++skipped;
continue;
}
const auto& [txn, txMeta] =
accountData.transactions[innerRIt->second];
result.emplace_back(
txn->getSTransaction()->getSerializer().peekData(),
txMeta->getAsObject().getSerializer().peekData(),
@@ -838,18 +928,23 @@ public:
return {newmarker, total};
}
Blob rawTxn = accountData.transactions[index]
.first->getSTransaction()
AccountTx const& accountTx =
accountData.transactions[index];
Blob rawTxn = accountTx.first->getSTransaction()
->getSerializer()
.peekData();
Blob rawMeta = accountData.transactions[index]
.second->getAsObject()
Blob rawMeta = accountTx.second->getAsObject()
.getSerializer()
.peekData();
if (rawMeta.size() == 0)
onUnsavedLedger(ledgerSeq);
if (options.strict &&
!isAccountInvolvedInTx(options.account, accountTx))
continue;
onTransaction(
rangeCheckedCast<std::uint32_t>(ledgerSeq),
"COMMITTED",
@@ -893,18 +988,23 @@ public:
return {newmarker, total};
}
Blob rawTxn = accountData.transactions[index]
.first->getSTransaction()
AccountTx const& accountTx =
accountData.transactions[index];
Blob rawTxn = accountTx.first->getSTransaction()
->getSerializer()
.peekData();
Blob rawMeta = accountData.transactions[index]
.second->getAsObject()
Blob rawMeta = accountTx.second->getAsObject()
.getSerializer()
.peekData();
if (rawMeta.size() == 0)
onUnsavedLedger(ledgerSeq);
if (options.strict &&
!isAccountInvolvedInTx(options.account, accountTx))
continue;
onTransaction(
rangeCheckedCast<std::uint32_t>(ledgerSeq),
"COMMITTED",

View File

@@ -27,6 +27,7 @@
#include <ripple/app/rdb/backend/detail/Node.h>
#include <ripple/basics/BasicConfig.h>
#include <ripple/basics/StringUtilities.h>
#include <ripple/basics/strHex.h>
#include <ripple/core/DatabaseCon.h>
#include <ripple/core/SociDB.h>
#include <ripple/json/to_string.h>
@@ -758,14 +759,34 @@ transactionsSQL(
options.minLedger);
}
// Convert account ID to hex string for binary search
std::string accountHex =
strHex(options.account.data(), options.account.size());
std::string sql;
// For metadata search:
// 1. Look for account ID not preceded by 8814 (RegularKey field)
// 2. OR look for account in raw transaction
std::string filterClause = options.strict ? "AND (("
"hex(TxnMeta) LIKE '%" +
accountHex +
"%' AND "
"hex(TxnMeta) NOT LIKE '%8814" +
accountHex +
"%'"
") OR hex(RawTxn) LIKE '%" +
accountHex + "%')"
: "";
if (count)
sql = boost::str(
boost::format("SELECT %s FROM AccountTransactions "
"WHERE Account = '%s' %s %s LIMIT %u, %u;") %
selection % toBase58(options.account) % maxClause % minClause %
beast::lexicalCastThrow<std::string>(options.offset) %
"INNER JOIN Transactions ON Transactions.TransID = "
"AccountTransactions.TransID "
"WHERE Account = '%s' %s %s %s LIMIT %u, %u;") %
selection % toBase58(options.account) % filterClause % maxClause %
minClause % beast::lexicalCastThrow<std::string>(options.offset) %
beast::lexicalCastThrow<std::string>(numberOfResults));
else
sql = boost::str(
@@ -773,15 +794,16 @@ transactionsSQL(
"SELECT %s FROM "
"AccountTransactions INNER JOIN Transactions "
"ON Transactions.TransID = AccountTransactions.TransID "
"WHERE Account = '%s' %s %s "
"WHERE Account = '%s' %s %s %s "
"ORDER BY AccountTransactions.LedgerSeq %s, "
"AccountTransactions.TxnSeq %s, AccountTransactions.TransID %s "
"LIMIT %u, %u;") %
selection % toBase58(options.account) % maxClause % minClause %
selection % toBase58(options.account) % filterClause % maxClause %
minClause % (descending ? "DESC" : "ASC") %
(descending ? "DESC" : "ASC") % (descending ? "DESC" : "ASC") %
(descending ? "DESC" : "ASC") %
beast::lexicalCastThrow<std::string>(options.offset) %
beast::lexicalCastThrow<std::string>(numberOfResults));
JLOG(j.trace()) << "txSQL query: " << sql;
return sql;
}
@@ -1114,6 +1136,21 @@ accountTxPage(
if (limit_used > 0)
newmarker = options.marker;
// Convert account ID to hex string for binary search
std::string accountHex =
strHex(options.account.data(), options.account.size());
// Add metadata search filter similar to transactionsSQL
std::string filterClause = options.strict
? " AND ((hex(TxnMeta) LIKE '%" + accountHex +
"%' "
"AND hex(TxnMeta) NOT LIKE '%8814" +
accountHex +
"%') "
"OR hex(RawTxn) LIKE '%" +
accountHex + "%')"
: "";
static std::string const prefix(
R"(SELECT AccountTransactions.LedgerSeq,AccountTransactions.TxnSeq,
Status,RawTxn,TxnMeta
@@ -1132,12 +1169,12 @@ accountTxPage(
{
sql = boost::str(
boost::format(
prefix + (R"(AccountTransactions.LedgerSeq BETWEEN %u AND %u
prefix + (R"(AccountTransactions.LedgerSeq BETWEEN %u AND %u %s
ORDER BY AccountTransactions.LedgerSeq %s,
AccountTransactions.TxnSeq %s
LIMIT %u;)")) %
toBase58(options.account) % options.minLedger % options.maxLedger %
order % order % queryLimit);
filterClause % order % order % queryLimit);
}
else
{
@@ -1150,25 +1187,25 @@ accountTxPage(
auto b58acct = toBase58(options.account);
sql = boost::str(
boost::format((
R"(SELECT AccountTransactions.LedgerSeq,AccountTransactions.TxnSeq,
Status,RawTxn,TxnMeta
R"(SELECT AccountTransactions.LedgerSeq,AccountTransactions.TxnSeq,Status,RawTxn,TxnMeta
FROM AccountTransactions, Transactions WHERE
(AccountTransactions.TransID = Transactions.TransID AND
AccountTransactions.Account = '%s' AND
AccountTransactions.LedgerSeq BETWEEN %u AND %u)
AccountTransactions.LedgerSeq BETWEEN %u AND %u) %s
UNION
SELECT AccountTransactions.LedgerSeq,AccountTransactions.TxnSeq,Status,RawTxn,TxnMeta
FROM AccountTransactions, Transactions WHERE
(AccountTransactions.TransID = Transactions.TransID AND
AccountTransactions.Account = '%s' AND
AccountTransactions.LedgerSeq = %u AND
AccountTransactions.TxnSeq %s %u)
AccountTransactions.TxnSeq %s %u) %s
ORDER BY AccountTransactions.LedgerSeq %s,
AccountTransactions.TxnSeq %s
LIMIT %u;
)")) %
b58acct % minLedger % maxLedger % b58acct % findLedger % compare %
findSeq % order % order % queryLimit);
b58acct % minLedger % maxLedger % filterClause % b58acct %
findLedger % compare % findSeq % filterClause % order % order %
queryLimit);
}
{

View File

@@ -587,7 +587,8 @@ Change::activateXahauGenesis()
wasmBytes, // wasm to verify
loggerStream,
"rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh",
ctx_.view().rules().enabled(featureHooksUpdate1) ? 1 : 0);
(ctx_.view().rules().enabled(featureHooksUpdate1) ? 1 : 0) +
(ctx_.view().rules().enabled(fix20250131) ? 2 : 0));
if (!result)
{

View File

@@ -72,6 +72,16 @@ Remit::preflight(PreflightContext const& ctx)
return temREDUNDANT;
}
if (ctx.rules.enabled(fix20250131))
{
if (!dstID || dstID == noAccount())
{
JLOG(ctx.j.warn())
<< "Malformed transaction: Remit to invalid account.";
return temMALFORMED;
}
}
if (ctx.tx.isFieldPresent(sfInform))
{
AccountID const infID = ctx.tx.getAccountID(sfInform);

View File

@@ -479,7 +479,8 @@ SetHook::validateHookSetEntry(SetHookCtx& ctx, STObject const& hookSetObj)
hook, // wasm to verify
logger,
hsacc,
ctx.rules.enabled(featureHooksUpdate1) ? 1 : 0);
(ctx.rules.enabled(featureHooksUpdate1) ? 1 : 0) +
(ctx.rules.enabled(fix20250131) ? 2 : 0));
if (ctx.j.trace())
{

View File

@@ -40,6 +40,17 @@ strHex(FwdIt begin, FwdIt end)
return result;
}
template <class FwdIt>
std::string
strHex(FwdIt begin, std::size_t length)
{
std::string result;
result.reserve(2 * length);
boost::algorithm::hex(
begin, std::next(begin, length), std::back_inserter(result));
return result;
}
template <class T, class = decltype(std::declval<T>().begin())>
std::string
strHex(T const& from)

View File

@@ -74,7 +74,7 @@ namespace detail {
// Feature.cpp. Because it's only used to reserve storage, and determine how
// large to make the FeatureBitset, it MAY be larger. It MUST NOT be less than
// the actual number of amendments. A LogicError on startup will verify this.
static constexpr std::size_t numFeatures = 76;
static constexpr std::size_t numFeatures = 77;
/** Amendments that this server supports and the default voting behavior.
Whether they are enabled depends on the Rules defined in the validated
@@ -364,6 +364,7 @@ extern uint256 const fix240911;
extern uint256 const fixFloatDivide;
extern uint256 const fixReduceImport;
extern uint256 const fixXahauV3;
extern uint256 const fix20250131;
} // namespace ripple

View File

@@ -469,7 +469,8 @@ REGISTER_FIX (fixPageCap, Supported::yes, VoteBehavior::De
REGISTER_FIX (fix240911, Supported::yes, VoteBehavior::DefaultYes);
REGISTER_FIX (fixFloatDivide, Supported::yes, VoteBehavior::DefaultYes);
REGISTER_FIX (fixReduceImport, Supported::yes, VoteBehavior::DefaultYes);
REGISTER_FIX (fixXahauV3, Supported::yes, VoteBehavior::DefaultNo);
REGISTER_FIX (fixXahauV3, Supported::yes, VoteBehavior::DefaultYes);
REGISTER_FIX (fix20250131, Supported::yes, VoteBehavior::DefaultYes);
// The following amendments are obsolete, but must remain supported
// because they could potentially get enabled.

View File

@@ -240,7 +240,9 @@ TxMeta::addRaw(Serializer& s, TER result, std::uint32_t index)
{
mResult = TERtoInt(result);
mIndex = index;
assert((mResult == 0) || ((mResult > 100) && (mResult <= 255)));
assert(
(mResult == 0 || mResult == 1) ||
((mResult > 100) && (mResult <= 255)));
mNodes.sort([](STObject const& o1, STObject const& o2) {
return o1.getFieldH256(sfLedgerIndex) < o2.getFieldH256(sfLedgerIndex);

View File

@@ -223,7 +223,8 @@ doAccountTxHelp(RPC::Context& context, AccountTxArgs const& args)
result.ledgerRange.max,
result.marker,
args.limit,
isUnlimited(context.role)};
isUnlimited(context.role),
args.strict};
auto const db =
dynamic_cast<SQLiteDatabase*>(&context.app.getRelationalDatabase());
@@ -369,6 +370,9 @@ doAccountTxJson(RPC::JsonContext& context)
args.forward =
params.isMember(jss::forward) && params[jss::forward].asBool();
args.strict =
params.isMember(jss::strict) ? params[jss::strict].asBool() : true;
if (!params.isMember(jss::account))
return rpcError(rpcINVALID_PARAMS);

View File

@@ -5544,7 +5544,6 @@ public:
testTSH(sa - fixXahauV1 - fixXahauV2);
testTSH(sa - fixXahauV2);
testTSH(sa);
testEmittedTxn(sa - fixXahauV2);
testEmittedTxn(sa);
}
};

View File

@@ -1138,6 +1138,54 @@ public:
: preHookCount + 66);
}
void
testFillCopy(FeatureBitset features)
{
testcase("Test fill/copy");
// a hook containing memory.fill instruction
std::string hookFill =
"0061736d0100000001130360027f7f017f60037f7f7e017e60017f017e02"
"170203656e76025f67000003656e76066163636570740001030201020503"
"0100020621057f01418088040b7f004180080b7f004180080b7f00418088"
"040b7f004180080b07080104686f6f6b00020aa4800001a0800001017e23"
"01412a41e400fc0b004101410110001a41004100420010011a20010b";
// a hook containing memory.copy instruction
std::string hookCopy =
"0061736d0100000001130360027f7f017f60037f7f7e017e60017f017e02"
"170203656e76025f67000003656e76066163636570740001030201020503"
"0100020621057f01418088040b7f004180080b7f004180080b7f00418088"
"040b7f004180080b07080104686f6f6b00020aa5800001a1800001017e23"
"00230141e400fc0a00004101410110001a41004100420010011a20010b";
using namespace jtx;
for (int withFix = 0; withFix < 2; ++withFix)
{
auto f = withFix ? features : features - fix20250131;
Env env{*this, f};
auto const alice = Account{"alice"};
env.fund(XRP(10000), alice);
auto const bob = Account{"bob"};
env.fund(XRP(10000), bob);
env(ripple::test::jtx::hook(alice, {{hso(hookFill)}}, 0),
M(withFix ? "hookFill - with fix" : "hookFill - zonder fix"),
HSFEE,
withFix ? ter(temMALFORMED) : ter(tesSUCCESS));
env(ripple::test::jtx::hook(bob, {{hso(hookCopy)}}, 0),
M(withFix ? "hookCopy - with fix" : "hookCopy - zonder fix"),
HSFEE,
withFix ? ter(temMALFORMED) : ter(tesSUCCESS));
env.close();
}
}
void
testCreate(FeatureBitset features)
{
@@ -11973,6 +12021,8 @@ public:
testNSDeletePartial(features);
testPageCap(features);
testFillCopy(features);
testWasm(features);
test_accept(features);
test_rollback(features);

View File

@@ -245,6 +245,72 @@ class AccountTx_test : public beast::unit_test::suite
p[jss::ledger_hash] = to_string(env.closed()->info().parentHash);
BEAST_EXPECT(noTxs(env.rpc("json", "account_tx", to_string(p))));
}
// Strict
{
Account S1{"S1"};
Account S2{"S2"};
Account S3{"S3"};
env.fund(XRP(10000), S1);
env.fund(XRP(10000), S2);
env.fund(XRP(10000), S3);
env.close();
// Regular key set
env(regkey(S1, S2));
env.close();
// we'll make a payment between S1 and S3
env(pay(S1, S3, XRP(100)));
env.close();
auto hasTxs = [](Json::Value const& j, bool strict) {
if (!j.isMember(jss::result) ||
j[jss::result][jss::status] != "success")
return false;
std::cout << "hasTx " << (strict ? "strict" : "not strict")
<< ":\n"
<< to_string(j) << "\n";
if (strict)
{
return (j[jss::result][jss::transactions].size() == 3) &&
(j[jss::result][jss::transactions][0u][jss::tx]
[jss::TransactionType] == jss::SetRegularKey) &&
(j[jss::result][jss::transactions][1u][jss::tx]
[jss::TransactionType] == jss::AccountSet) &&
(j[jss::result][jss::transactions][2u][jss::tx]
[jss::TransactionType] == jss::Payment);
}
return (j[jss::result][jss::transactions].size() == 4) &&
(j[jss::result][jss::transactions][0u][jss::tx]
[jss::TransactionType] == jss::Payment) &&
(j[jss::result][jss::transactions][1u][jss::tx]
[jss::TransactionType] == jss::SetRegularKey) &&
(j[jss::result][jss::transactions][2u][jss::tx]
[jss::TransactionType] == jss::AccountSet) &&
(j[jss::result][jss::transactions][3u][jss::tx]
[jss::TransactionType] == jss::Payment);
};
Json::Value p{jParms};
p[jss::account] = S2.human();
BEAST_EXPECT(
hasTxs(env.rpc("json", "account_tx", to_string(p)), true));
p[jss::strict] = true;
BEAST_EXPECT(
hasTxs(env.rpc("json", "account_tx", to_string(p)), true));
p[jss::strict] = false;
BEAST_EXPECT(
hasTxs(env.rpc("json", "account_tx", to_string(p)), false));
}
}
void