Compare commits

..

1 Commits

Author SHA1 Message Date
Richard Holland
dc17cb44a7 partial 2026-01-29 12:05:07 +11:00
10 changed files with 496 additions and 2225 deletions

2
.gitignore vendored
View File

@@ -121,5 +121,3 @@ CMakeUserPresets.json
bld.rippled/
generated
guard_checker
guard_checker.dSYM

View File

@@ -5,7 +5,6 @@
#include <memory>
#include <optional>
#include <ostream>
#include <set>
#include <stack>
#include <string>
#include <string_view>
@@ -283,8 +282,7 @@ check_guard(
* might have unforeseen consequences, without also rolling back further
* changes that are fine.
*/
uint64_t rulesVersion = 0,
std::set<int>* out_callees = nullptr
uint64_t rulesVersion = 0
)
{
@@ -494,27 +492,17 @@ check_guard(
{
REQUIRE(1);
uint64_t callee_idx = LEB();
// record user-defined function calls if tracking is enabled
// disallow calling of user defined functions inside a hook
if (callee_idx > last_import_idx)
{
if (out_callees != nullptr)
{
// record the callee for call graph analysis
out_callees->insert(callee_idx);
}
else
{
// if not tracking, maintain original behavior: reject
GUARDLOG(hook::log::CALL_ILLEGAL)
<< "GuardCheck "
<< "Hook calls a function outside of the whitelisted "
"imports "
<< "codesec: " << codesec << " hook byte offset: " << i
<< "\n";
GUARDLOG(hook::log::CALL_ILLEGAL)
<< "GuardCheck "
<< "Hook calls a function outside of the whitelisted "
"imports "
<< "codesec: " << codesec << " hook byte offset: " << i
<< "\n";
return {};
}
return {};
}
// enforce guard call limit
@@ -849,42 +837,6 @@ validateGuards(
*/
uint64_t rulesVersion = 0)
{
// Structure to track function call graph information
struct FunctionInfo
{
int func_idx;
std::set<int> callees; // functions this function calls
std::set<int> callers; // functions that call this function
bool has_loops; // whether this function contains loops
uint64_t local_wce; // local worst-case execution count
uint64_t total_wce; // total WCE including callees
bool wce_calculated; // whether total_wce has been computed
bool in_calculation; // for cycle detection in WCE calculation
FunctionInfo()
: func_idx(-1)
, has_loops(false)
, local_wce(0)
, total_wce(0)
, wce_calculated(false)
, in_calculation(false)
{
}
FunctionInfo(int idx, uint64_t local_wce_val, bool has_loops_val)
: func_idx(idx)
, has_loops(has_loops_val)
, local_wce(local_wce_val)
, total_wce(0)
, wce_calculated(false)
, in_calculation(false)
{
}
};
// Call graph: maps function index to its information
std::map<int, FunctionInfo> call_graph;
uint64_t byteCount = wasm.size();
// 63 bytes is the smallest possible valid hook wasm
@@ -1224,12 +1176,6 @@ validateGuards(
if (DEBUG_GUARD)
printf("Function map: func %d -> type %d\n", j, type_idx);
func_type_map[j] = type_idx;
// Step 4: Initialize FunctionInfo for each user-defined
// function func_idx starts from last_import_number + 1
int actual_func_idx = last_import_number + 1 + j;
call_graph[actual_func_idx] = FunctionInfo();
call_graph[actual_func_idx].func_idx = actual_func_idx;
}
}
@@ -1271,6 +1217,9 @@ validateGuards(
return {};
}
int64_t maxInstrCountHook = 0;
int64_t maxInstrCountCbak = 0;
// second pass... where we check all the guard function calls follow the
// guard rules minimal other validation in this pass because first pass
// caught most of it
@@ -1304,7 +1253,6 @@ validateGuards(
std::optional<
std::reference_wrapper<std::vector<uint8_t> const>>
first_signature;
bool helper_function = false;
if (auto const& usage = import_type_map.find(j);
usage != import_type_map.end())
{
@@ -1340,7 +1288,7 @@ validateGuards(
}
}
}
else if (j == hook_type_idx) // hook() or cbak() function type
else if (j == hook_type_idx)
{
// pass
}
@@ -1353,8 +1301,7 @@ validateGuards(
<< "Codesec: " << section_type << " "
<< "Local: " << j << " "
<< "Offset: " << i << "\n";
// return {};
helper_function = true;
return {};
}
int param_count = parseLeb128(wasm, i, &i);
@@ -1371,19 +1318,12 @@ validateGuards(
return {};
}
}
else if (helper_function)
{
// pass
}
else if (param_count != (*first_signature).get().size() - 1)
{
GUARDLOG(hook::log::FUNC_TYPE_INVALID)
<< "Malformed transaction. "
<< "Hook API: " << *first_name
<< " has the wrong number of parameters.\n"
<< "param_count: " << param_count << " "
<< "first_signature: "
<< (*first_signature).get().size() - 1 << "\n";
<< " has the wrong number of parameters.\n";
return {};
}
@@ -1430,10 +1370,6 @@ validateGuards(
return {};
}
}
else if (helper_function)
{
// pass
}
else if ((*first_signature).get()[k + 1] != param_type)
{
GUARDLOG(hook::log::FUNC_PARAM_INVALID)
@@ -1510,10 +1446,6 @@ validateGuards(
return {};
}
}
else if (helper_function)
{
// pass
}
else if ((*first_signature).get()[0] != result_type)
{
GUARDLOG(hook::log::FUNC_RETURN_INVALID)
@@ -1565,17 +1497,6 @@ validateGuards(
// execution to here means we are up to the actual expr for the
// codesec/function
// Step 5: Calculate actual function index and prepare callees
// tracking
int actual_func_idx = last_import_number + 1 + j;
std::set<int>* out_callees_ptr = nullptr;
// Only track callees if this function is in the call_graph
if (call_graph.find(actual_func_idx) != call_graph.end())
{
out_callees_ptr = &call_graph[actual_func_idx].callees;
}
auto valid = check_guard(
wasm,
j,
@@ -1585,188 +1506,33 @@ validateGuards(
last_import_number,
guardLog,
guardLogAccStr,
rulesVersion,
out_callees_ptr);
rulesVersion);
if (!valid)
return {};
// Step 5: Store local WCE and build bidirectional call
// relationships
if (call_graph.find(actual_func_idx) != call_graph.end())
if (hook_func_idx && *hook_func_idx == j)
maxInstrCountHook = *valid;
else if (cbak_func_idx && *cbak_func_idx == j)
maxInstrCountCbak = *valid;
else
{
call_graph[actual_func_idx].local_wce = *valid;
// Build bidirectional relationships: for each callee, add
// this function as a caller
for (int callee_idx : call_graph[actual_func_idx].callees)
{
if (call_graph.find(callee_idx) != call_graph.end())
{
call_graph[callee_idx].callers.insert(
actual_func_idx);
}
}
if (DEBUG_GUARD)
printf(
"code section: %d not hook_func_idx: %d or "
"cbak_func_idx: %d\n",
j,
*hook_func_idx,
(cbak_func_idx ? *cbak_func_idx : -1));
// assert(false);
}
// Note: We will calculate total WCE later after processing all
// functions
i = code_end;
}
}
i = next_section;
}
// Step 6: Cycle detection using DFS
// Lambda function for DFS-based cycle detection
std::set<int> visited;
std::set<int> rec_stack;
std::function<bool(int)> detect_cycles_dfs = [&](int func_idx) -> bool {
if (rec_stack.find(func_idx) != rec_stack.end())
{
// Found a cycle: func_idx is already in the recursion stack
return true;
}
// execution to here means guards are installed correctly
if (visited.find(func_idx) != visited.end())
{
// Already visited and no cycle found from this node
return false;
}
visited.insert(func_idx);
rec_stack.insert(func_idx);
// Check all callees
if (call_graph.find(func_idx) != call_graph.end())
{
for (int callee_idx : call_graph[func_idx].callees)
{
if (detect_cycles_dfs(callee_idx))
{
return true;
}
}
}
rec_stack.erase(func_idx);
return false;
};
// Run cycle detection on all user-defined functions
for (const auto& [func_idx, func_info] : call_graph)
{
if (detect_cycles_dfs(func_idx))
{
GUARDLOG(hook::log::CALL_ILLEGAL)
<< "GuardCheck: Recursive function calls detected. "
<< "Hooks cannot contain recursive or mutually recursive "
"functions.\n";
return {};
}
}
// Step 7: Calculate total WCE for each function using bottom-up approach
// Lambda function for recursive WCE calculation with memoization
std::function<uint64_t(int)> calculate_function_wce =
[&](int func_idx) -> uint64_t {
// Check if function exists in call graph
if (call_graph.find(func_idx) == call_graph.end())
{
// This is an imported function, WCE = 0 (already accounted for)
return 0;
}
FunctionInfo& func_info = call_graph[func_idx];
// If already calculated, return cached result
if (func_info.wce_calculated)
{
return func_info.total_wce;
}
// Detect circular dependency in WCE calculation (should not happen
// after cycle detection)
if (func_info.in_calculation)
{
GUARDLOG(hook::log::CALL_ILLEGAL)
<< "GuardCheck: Internal error - circular dependency detected "
"during WCE calculation.\n";
return 0xFFFFFFFFU; // Return large value to trigger overflow error
}
func_info.in_calculation = true;
// Start with local WCE
uint64_t total = func_info.local_wce;
// Add WCE of all callees
for (int callee_idx : func_info.callees)
{
uint64_t callee_wce = calculate_function_wce(callee_idx);
// Check for overflow
if (total > 0xFFFFU || callee_wce > 0xFFFFU ||
(total + callee_wce) > 0xFFFFU)
{
func_info.in_calculation = false;
return 0xFFFFFFFFU; // Signal overflow
}
total += callee_wce;
}
func_info.total_wce = total;
func_info.wce_calculated = true;
func_info.in_calculation = false;
return total;
};
// Calculate WCE for hook and cbak functions
int64_t hook_wce_actual = 0;
int64_t cbak_wce_actual = 0;
if (hook_func_idx)
{
int actual_hook_idx = last_import_number + 1 + *hook_func_idx;
hook_wce_actual = calculate_function_wce(actual_hook_idx);
if (hook_wce_actual >= 0xFFFFU)
{
GUARDLOG(hook::log::INSTRUCTION_EXCESS)
<< "GuardCheck: hook() function exceeds maximum instruction "
"count (65535). "
<< "Total WCE including called functions: " << hook_wce_actual
<< "\n";
return {};
}
if (DEBUG_GUARD)
printf("hook() total WCE: %ld\n", hook_wce_actual);
}
if (cbak_func_idx)
{
int actual_cbak_idx = last_import_number + 1 + *cbak_func_idx;
cbak_wce_actual = calculate_function_wce(actual_cbak_idx);
if (cbak_wce_actual >= 0xFFFFU)
{
GUARDLOG(hook::log::INSTRUCTION_EXCESS)
<< "GuardCheck: cbak() function exceeds maximum instruction "
"count (65535). "
<< "Total WCE including called functions: " << cbak_wce_actual
<< "\n";
return {};
}
if (DEBUG_GUARD)
printf("cbak() total WCE: %ld\n", cbak_wce_actual);
}
// execution to here means guards are installed correctly and WCE is within
// limits
return std::pair<uint64_t, uint64_t>{hook_wce_actual, cbak_wce_actual};
return std::pair<uint64_t, uint64_t>{maxInstrCountHook, maxInstrCountCbak};
}

View File

@@ -479,11 +479,24 @@ private:
to reach consensus. Update our position only on the timer, and in this
phase.
If we have consensus, move to the accepted phase.
If we have consensus, move to the shuffle phase.
*/
void
phaseEstablish();
/** Handle shuffle phase.
In the shuffle phase, UNLReport nodes exchange entropy to build
a consensus entropy that is then used as an RNG source for Hooks.
The entropy is injected as a ttSHUFFLE psuedo into the final ledger
If we have consensus, move to the accepted phase.
*/
void
phaseShuffle();
/** Evaluate whether pausing increases likelihood of validation.
*
* As a validator that has previously synced to the network, if our most
@@ -588,6 +601,10 @@ private:
// Peer proposed positions for the current round
hash_map<NodeID_t, PeerPosition_t> currPeerPositions_;
// our and our peers' entropy as per TMShuffle, used in phaseShuffle
std::optional<uint256> ourEntropy_;
hash_map<NodeID_t, std::pair<uint256, uint256>> currPeerEntropy_;
// Recently received peer positions, available when transitioning between
// ledgers or rounds
hash_map<NodeID_t, std::deque<PeerPosition_t>> recentPeerPositions_;
@@ -832,6 +849,10 @@ Consensus<Adaptor>::timerEntry(NetClock::time_point const& now)
{
phaseEstablish();
}
else if (phase_ == ConsensusPhase::shuffle)
{
phaseShuffle();
}
}
template <class Adaptor>
@@ -1291,8 +1312,12 @@ Consensus<Adaptor>::phaseEstablish()
adaptor_.updateOperatingMode(currPeerPositions_.size());
prevProposers_ = currPeerPositions_.size();
prevRoundTime_ = result_->roundTime.read();
phase_ = ConsensusPhase::accepted;
JLOG(j_.debug()) << "transitioned to ConsensusPhase::accepted";
// RHTODO: guard with amendment
phase_ = ConsensusPhase::shuffle;
JLOG(j_.debug()) << "transitioned to ConsensusPhase::shuffle";
/*
adaptor_.onAccept(
*result_,
previousLedger_,
@@ -1300,6 +1325,60 @@ Consensus<Adaptor>::phaseEstablish()
rawCloseTimes_,
mode_.get(),
getJson(true));
*/
}
template <class Adaptor>
void
Consensus<Adaptor>::phaseShuffle()
{
// can only establish consensus if we already took a stance
assert(result_);
using namespace std::chrono;
ConsensusParms const& parms = adaptor_.parms();
result_->roundTime.tick(clock_.now());
result_->proposers = currPeerPositions_.size();
convergePercent_ = result_->roundTime.read() * 100 /
std::max<milliseconds>(prevRoundTime_, parms.avMIN_CONSENSUS_TIME);
// Give everyone a chance to take an initial position
if (result_->roundTime.read() < parms.ledgerMIN_CONSENSUS)
return;
updateOurPositions();
// Nothing to do if too many laggards or we don't have consensus.
if (shouldPause() || !haveConsensus())
return;
if (!haveCloseTimeConsensus_)
{
JLOG(j_.info()) << "We have TX consensus but not CT consensus";
return;
}
JLOG(j_.info()) << "Converge cutoff (" << currPeerPositions_.size()
<< " participants)";
adaptor_.updateOperatingMode(currPeerPositions_.size());
prevProposers_ = currPeerPositions_.size();
prevRoundTime_ = result_->roundTime.read();
// RHTODO: guard with amendment
phase_ = ConsensusPhase::shuffle;
JLOG(j_.debug()) << "transitioned to ConsensusPhase::shuffle";
/*
adaptor_.onAccept(
*result_,
previousLedger_,
closeResolution_,
rawCloseTimes_,
mode_.get(),
getJson(true));
*/
}
template <class Adaptor>

View File

@@ -87,15 +87,15 @@ to_string(ConsensusMode m)
/** Phases of consensus for a single ledger round.
@code
"close" "accept"
open ------- > establish ---------> accepted
^ | |
|---------------| |
^ "startRound" |
|------------------------------------|
"close" "shuffle" "accept"
open ------- > establish -------> shuffle ---------> accepted
^ | |
|---------------| |
^ "startRound" |
|-----------------------------------------------------|
@endcode
The typical transition goes from open to establish to accepted and
The typical transition goes from open to establish to shuffle to accepted and
then a call to startRound begins the process anew. However, if a wrong prior
ledger is detected and recovered during the establish or accept phase,
consensus will internally go back to open (see Consensus::handleWrongLedger).
@@ -107,6 +107,9 @@ enum class ConsensusPhase {
//! Establishing consensus by exchanging proposals with our peers
establish,
//! Negotitate featureRNG entropy
shuffle,
//! We have accepted a new last closed ledger and are waiting on a call
//! to startRound to begin the next consensus round. No changes
//! to consensus phase occur while in this phase.
@@ -122,6 +125,8 @@ to_string(ConsensusPhase p)
return "open";
case ConsensusPhase::establish:
return "establish";
case ConsensusPhase::shuffle:
return "shuffle";
case ConsensusPhase::accepted:
return "accepted";
default:

View File

@@ -1094,6 +1094,7 @@ trustTransferLockedBalance(
}
return tesSUCCESS;
}
} // namespace ripple
#endif

View File

@@ -1918,6 +1918,100 @@ PeerImp::onMessage(std::shared_ptr<protocol::TMLedgerData> const& m)
app_.getInboundLedgers().gotLedgerData(ledgerHash, shared_from_this(), m);
}
void
PeerImp::onMessage(std::shared_ptr<protocol::TMShuffle> const& m)
{
protocol::TMShuffle& shuf = *m;
auto const sig = makeSlice(shf.signature());
// Preliminary check for the validity of the signature: A DER encoded
// signature can't be longer than 72 bytes.
if ((std::clamp<std::size_t>(sig.size(), 64, 72) != sig.size()) ||
(publicKeyType(makeSlice(shf.nodepubkey())) != KeyType::secp256k1))
{
JLOG(p_journal_.warn()) << "Shuffle: malformed";
fee_ = Resource::feeInvalidSignature;
return;
}
if (!stringIsUint256Sized(shf.nodeentropy()) ||
!stringIsUint256Sized(shf.consensusentropy()) ||
!stringIsUint256Sized(shf.previousledger()))
{
JLOG(p_journal_.warn()) << "Shuffle: malformed";
fee_ = Resource::feeInvalidRequest;
return;
}
PublicKey const publicKey{makeSlice(shf.nodepubkey())};
auto const isTrusted = app_.validators().trusted(publicKey);
if (!isTrusted)
return;
uint256 const prevLedger{shf.previousledger()};
uint32_t const shuffleSeq{shf.shuffleseq()};
uint256 const nodeEntropy{shf.nodeentropy()};
uint256 const consensusEntropy{shf.consensusentropy()};
uint256 const suppression = sha512Half(std::string("TMShuffle", sig));
if (auto [added, relayed] =
app_.getHashRouter().addSuppressionPeerWithStatus(suppression, id_);
!added)
{
// Count unique messages (Slots has it's own 'HashRouter'), which a peer
// receives within IDLED seconds since the message has been relayed.
if (reduceRelayReady() && relayed &&
(stopwatch().now() - *relayed) < reduce_relay::IDLED)
overlay_.updateSlotAndSquelch(
suppression, publicKey, id_, protocol::mtSHUFFLE);
JLOG(p_journal_.trace()) << "Shuffle: duplicate";
return;
}
if (!isTrusted)
{
if (tracking_.load() == Tracking::diverged)
{
JLOG(p_journal_.debug())
<< "Proposal: Dropping untrusted (peer divergence)";
return;
}
if (!cluster() && app_.getFeeTrack().isLoadedLocal())
{
JLOG(p_journal_.debug()) << "Proposal: Dropping untrusted (load)";
return;
}
}
JLOG(p_journal_.trace())
<< "Proposal: " << (isTrusted ? "trusted" : "untrusted");
auto proposal = RCLCxPeerPos(
publicKey,
sig,
suppression,
RCLCxPeerPos::Proposal{
prevLedger,
set.proposeseq(),
proposeHash,
closeTime,
app_.timeKeeper().closeTime(),
calcNodeID(app_.validatorManifests().getMasterKey(publicKey))});
std::weak_ptr<PeerImp> weak = shared_from_this();
app_.getJobQueue().addJob(
isTrusted ? jtPROPOSAL_t : jtPROPOSAL_ut,
"recvPropose->checkPropose",
[weak, isTrusted, m, proposal]() {
if (auto peer = weak.lock())
peer->checkPropose(isTrusted, m, proposal);
});
}
void
PeerImp::onMessage(std::shared_ptr<protocol::TMProposeSet> const& m)
{

View File

@@ -450,3 +450,12 @@ message TMHaveTransactions
repeated bytes hashes = 1;
}
message TMShuffle
{
required bytes nodeEntropy = 1;
required bytes consensusEntropy = 2;
required uint32 shuffleSeq = 3;
required bytes nodePubKey = 4;
required bytes previousledger = 5;
required bytes signature = 6; // signature of above fields
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -58,21 +58,8 @@ cat $INPUT_FILE | tr '\n' '\f' |
then
echo '#include "api.h"' > "$WASM_DIR/test-$COUNTER-gen.c"
tr '\f' '\n' <<< $line >> "$WASM_DIR/test-$COUNTER-gen.c"
DECLARED="`tr '\f' '\n' <<< $line \
| grep -E '(extern|static|define) ' \
| grep -Eo '[a-z\-\_]+ *\(' \
| grep -v 'sizeof' \
| sed -E 's/[^a-z\-\_]//g' \
| grep -vE '^__attribute__$' \
| sort | uniq`"
USED="`tr '\f' '\n' <<< $line \
| grep -vE '(extern|static|define) ' \
| grep -Eo '[a-z\-\_]+\(' \
| grep -v 'sizeof' \
| sed -E 's/[^a-z\-\_]//g' \
| grep -vE '^(__attribute__|hook|cbak)$' \
| sort | uniq`"
DECLARED="`tr '\f' '\n' <<< $line | grep -E '(extern|define) ' | grep -Eo '[a-z\-\_]+ *\(' | grep -v 'sizeof' | sed -E 's/[^a-z\-\_]//g' | sort | uniq`"
USED="`tr '\f' '\n' <<< $line | grep -vE '(extern|define) ' | grep -Eo '[a-z\-\_]+\(' | grep -v 'sizeof' | sed -E 's/[^a-z\-\_]//g' | grep -vE '^(hook|cbak)' | sort | uniq`"
ONCE="`echo $DECLARED $USED | tr ' ' '\n' | sort | uniq -c | grep '1 ' | sed -E 's/^ *1 //g'`"
FILTER="`echo $DECLARED | tr ' ' '|' | sed -E 's/\|$//g'`"
UNDECL="`echo $ONCE | grep -v -E $FILTER 2>/dev/null || echo ''`"
@@ -82,7 +69,7 @@ cat $INPUT_FILE | tr '\n' '\f' |
echo "$line"
exit 1
fi
wasmcc -x c /dev/stdin -o /dev/stdout -O2 -Wl,--allow-undefined,--export=hook,--export=cbak <<< "`tr '\f' '\n' <<< $line`" |
wasmcc -x c /dev/stdin -o /dev/stdout -O2 -Wl,--allow-undefined <<< "`tr '\f' '\n' <<< $line`" |
hook-cleaner - - 2>/dev/null |
xxd -p -u -c 10 |
sed -E 's/../0x&U,/g' | sed -E 's/^/ /g' >> $OUTPUT_FILE