Compare commits

..

14 Commits

Author SHA1 Message Date
Pratik Mankawde
c0cc42a464 separate ubsan and asan
Signed-off-by: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
2026-04-02 16:16:02 +01:00
Pratik Mankawde
59f34924f0 coverage build on gcc-13, ubsan on gcc-15
Signed-off-by: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
2026-04-02 14:05:48 +01:00
Pratik Mankawde
e031c775e4 Merge branch 'develop' into pratik/Fix-ubsan-flagged-issues 2026-04-02 14:02:52 +01:00
Pratik Mankawde
6e68f14cf5 n being size_t issue
Signed-off-by: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
2026-03-30 12:43:12 +01:00
Pratik Mankawde
e56be3c1ed minor fix
Signed-off-by: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
2026-03-30 12:37:31 +01:00
Pratik Mankawde
d564c1dda9 minor fix
Signed-off-by: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
2026-03-30 12:20:57 +01:00
Pratik Mankawde
5c6858afe9 code review changes
Signed-off-by: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
2026-03-30 12:05:45 +01:00
Pratik Mankawde
9389350e17 Merge branch 'develop' into pratik/Fix-ubsan-flagged-issues 2026-03-30 11:26:15 +01:00
Pratik Mankawde
ac936648fe Merge branch 'develop' into pratik/Fix-ubsan-flagged-issues
Signed-off-by: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
2026-03-25 10:48:44 +00:00
Pratik Mankawde
23f1e1b0e6 put address back
Signed-off-by: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
2026-03-24 15:19:51 +00:00
Pratik Mankawde
1fc79cec53 more cleanup
Signed-off-by: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
2026-03-24 15:18:23 +00:00
Pratik Mankawde
09ce8125a0 more fixes some reverts
Signed-off-by: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
2026-03-24 14:59:53 +00:00
Pratik Mankawde
6468412621 Merge branch 'develop' into pratik/Fix-ubsan-flagged-issues
Signed-off-by: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
2026-03-24 14:35:58 +00:00
Pratik Mankawde
fe8403f9c4 Fix UBSan-flagged undefined behavior and clean up sanitizer suppressions
- Fix signed integer overflow UB in negation operations by performing
  negation in unsigned domain before casting back to signed. Applies to
  IOUAmount, XRPAmount, MPTAmount, and throughout STAmount (operator+,
  set, canonicalize, xrp/iou/mpt accessors, constructors).
- Fix post-decrement loop patterns that cause unsigned integer overflow
  (e.g. `while(n--)`) by replacing with `while(n > 0) { --n; ... }` or
  `for` loops in DecayingSample.h, varint.h, base64.cpp, BasicApp.cpp,
  and yield_to.h.
- Add Counts::adjustCounter() helper in PeerFinder to safely adjust
  size_t counters by signed values without triggering UBSan.
- Fix uninitialized member in ValidatorSite_test and remove
  overflow-dependent initialization in LexicalCast_test.
- Drastically reduce ubsan.supp by removing broad per-file suppressions
  now that the underlying issues are fixed. Keep only targeted
  suppressions for external libraries (RocksDB, protobuf, gRPC, nudb,
  snappy, abseil) and intentional unsigned wraps in rippled (STAmount
  arithmetic, nft::cipheredTaxon).
- Remove UBSAN_OPTIONS runtime suppressions file from CI workflow.
- Enable UBSan builds for gcc-13 in addition to clang-20 in CI matrix.
- Add fPIC handling in conanfile.py when Address sanitizer is active.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-11 11:37:28 +00:00
30 changed files with 225 additions and 943 deletions

View File

@@ -44,7 +44,6 @@ libxrpl.tx > xrpl.server
libxrpl.tx > xrpl.tx
test.app > test.jtx
test.app > test.rpc
test.app > test.shamap
test.app > test.toplevel
test.app > test.unit_test
test.app > xrpl.basics
@@ -60,7 +59,6 @@ test.app > xrpl.protocol
test.app > xrpl.rdb
test.app > xrpl.resource
test.app > xrpl.server
test.app > xrpl.shamap
test.app > xrpl.tx
test.basics > test.jtx
test.basics > test.unit_test

View File

@@ -64,7 +64,7 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
if os["distro_version"] == "bookworm":
if (
f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-13"
and build_type == "Release"
and build_type == "Debug"
and architecture["platform"] == "linux/amd64"
):
cmake_args = f"-DUNIT_TEST_REFERENCE_FEE=500 {cmake_args}"
@@ -197,7 +197,7 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
# linux/amd64
if (
f"{os['distro_name']}-{os['distro_version']}" == "debian-bookworm"
and f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-15"
and f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-13"
and build_type == "Debug"
and architecture["platform"] == "linux/amd64"
):
@@ -236,21 +236,36 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
# names get truncated.
# Add Address and Thread (both coupled with UB) sanitizers for specific bookworm distros.
# GCC-Asan rippled-embedded tests are failing because of https://github.com/google/sanitizers/issues/856
if (
os["distro_version"] == "bookworm"
and f"{os['compiler_name']}-{os['compiler_version']}" == "clang-20"
):
# Add ASAN + UBSAN configuration.
if os[
"distro_version"
] == "bookworm" and f"{os['compiler_name']}-{os['compiler_version']}" in [
"gcc-15",
"clang-20",
]:
# Add ASAN configuration.
configurations.append(
{
"config_name": config_name + "-asan-ubsan",
"config_name": config_name + "-asan",
"cmake_args": cmake_args,
"cmake_target": cmake_target,
"build_only": build_only,
"build_type": build_type,
"os": os,
"architecture": architecture,
"sanitizers": "address,undefinedbehavior",
"sanitizers": "address",
}
)
# Add UBSAN configuration.
configurations.append(
{
"config_name": config_name + "-ubsan",
"cmake_args": cmake_args,
"cmake_target": cmake_target,
"build_only": build_only,
"build_type": build_type,
"os": os,
"architecture": architecture,
"sanitizers": "undefinedbehavior",
}
)
# TSAN is deactivated due to seg faults with latest compilers.

View File

@@ -67,8 +67,10 @@ private:
}
else
{
while (elapsed--)
for (; elapsed > 0; --elapsed)
{
m_value -= (m_value + Window - 1) / Window;
}
}
}

View File

@@ -11,7 +11,6 @@
#include <limits>
#include <stdexcept>
#include <string>
#include <string_view>
#include <type_traits>
#include <vector>
@@ -232,11 +231,4 @@ makeSlice(std::basic_string<char, Traits, Alloc> const& s)
return Slice(s.data(), s.size());
}
template <class Traits>
Slice
makeSlice(std::basic_string_view<char, Traits> const& s)
{
return Slice(s.data(), s.size());
}
} // namespace xrpl

View File

@@ -44,7 +44,7 @@ public:
: work_(boost::asio::make_work_guard(ios_))
{
threads_.reserve(concurrency);
while (concurrency--)
for (std::size_t i = 0; i < concurrency; ++i)
threads_.emplace_back([&] { ios_.run(); });
}

View File

@@ -53,8 +53,9 @@ read_varint(void const* buf, std::size_t buflen, std::size_t& t)
return 1;
}
auto const used = n;
while (n--)
while (n > 0)
{
--n;
auto const d = p[n];
auto const t0 = t;
t *= 127;

View File

@@ -244,15 +244,7 @@ message TMGetObjectByHash {
message TMLedgerNode {
required bytes nodedata = 1;
// Used when protocol version <2.3. Not set for ledger base data.
optional bytes nodeid = 2;
// Used when protocol version >=2.3. Neither value is set for ledger base data.
oneof reference {
bytes id = 3; // Set for inner nodes.
uint32 depth = 4; // Set for leaf nodes.
}
optional bytes nodeid = 2; // missing for ledger base data
}
enum TMLedgerInfoType {

View File

@@ -16,7 +16,6 @@
#include <set>
#include <stack>
#include <tuple>
#include <vector>
namespace xrpl {
@@ -254,7 +253,7 @@ public:
bool
getNodeFat(
SHAMapNodeID const& wanted,
std::vector<std::tuple<SHAMapNodeID, Blob, bool>>& data,
std::vector<std::pair<SHAMapNodeID, Blob>>& data,
bool fatLeaves,
std::uint32_t depth) const;
@@ -281,45 +280,10 @@ public:
void
serializeRoot(Serializer& s) const;
/** Add a root node to the SHAMap during synchronization.
*
* This function is used when receiving the root node of a SHAMap from a peer during ledger
* synchronization. The node must already have been deserialized.
*
* @param hash The expected hash of the root node.
* @param rootNode A deserialized root node to add.
* @param filter Optional sync filter to track received nodes.
* @return Status indicating whether the node was useful, duplicate, or invalid.
*
* @note This function expects the rootNode to be a valid, deserialized SHAMapTreeNode. The
* caller is responsible for deserialization and basic validation before calling this
* function.
*/
SHAMapAddNode
addRootNode(
SHAMapHash const& hash,
intr_ptr::SharedPtr<SHAMapTreeNode> rootNode,
SHAMapSyncFilter const* filter);
/** Add a known node at a specific position in the SHAMap during synchronization.
*
* This function is used when receiving nodes from peers during ledger synchronization. The node
* is inserted at the position specified by nodeID. The node must already have been
* deserialized.
*
* @param nodeID The position in the tree where this node belongs.
* @param treeNode A deserialized tree node to add.
* @param filter Optional sync filter to track received nodes.
* @return Status indicating whether the node was useful, duplicate, or invalid.
*
* @note This function expects that the caller has already validated that the nodeID is
* consistent with the node's content.
*/
addRootNode(SHAMapHash const& hash, Slice const& rootNode, SHAMapSyncFilter* filter);
SHAMapAddNode
addKnownNode(
SHAMapNodeID const& nodeID,
intr_ptr::SharedPtr<SHAMapTreeNode> treeNode,
SHAMapSyncFilter const* filter);
addKnownNode(SHAMapNodeID const& nodeID, Slice const& rawNode, SHAMapSyncFilter* filter);
// status functions
void
@@ -380,11 +344,11 @@ private:
intr_ptr::SharedPtr<SHAMapTreeNode>
fetchNodeNT(SHAMapHash const& hash) const;
intr_ptr::SharedPtr<SHAMapTreeNode>
fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter const* filter) const;
fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter* filter) const;
intr_ptr::SharedPtr<SHAMapTreeNode>
fetchNode(SHAMapHash const& hash) const;
intr_ptr::SharedPtr<SHAMapTreeNode>
checkFilter(SHAMapHash const& hash, SHAMapSyncFilter const* filter) const;
checkFilter(SHAMapHash const& hash, SHAMapSyncFilter* filter) const;
/** Update hashes up to the root */
void
@@ -456,7 +420,7 @@ private:
descendAsync(
SHAMapInnerNode* parent,
int branch,
SHAMapSyncFilter const* filter,
SHAMapSyncFilter* filter,
bool& pending,
descendCallback&&) const;
@@ -465,7 +429,7 @@ private:
SHAMapInnerNode* parent,
SHAMapNodeID const& parentID,
int branch,
SHAMapSyncFilter const* filter) const;
SHAMapSyncFilter* filter) const;
// Non-storing
// Does not hook the returned node to its parent

View File

@@ -11,7 +11,6 @@ float-cast-overflow:external
float-divide-by-zero:external
function:external
implicit-integer-sign-change:external
implicit-signed-integer-truncation::external
implicit-signed-integer-truncation:external
implicit-unsigned-integer-truncation:external
integer-divide-by-zero:external
@@ -71,145 +70,15 @@ vla-bound:boost
vptr_check:boost
vptr:boost
# Google protobuf
# Google protobuf - intentional overflows in hash functions
undefined:protobuf
# Suppress UBSan errors in rippled code by source file path
undefined:src/libxrpl/basics/base64.cpp
undefined:src/libxrpl/basics/Number.cpp
undefined:src/libxrpl/beast/utility/beast_Journal.cpp
undefined:src/libxrpl/crypto/RFC1751.cpp
undefined:src/libxrpl/ledger/ApplyView.cpp
undefined:src/libxrpl/ledger/View.cpp
undefined:src/libxrpl/protocol/Permissions.cpp
undefined:src/libxrpl/protocol/STAmount.cpp
undefined:src/libxrpl/protocol/STPathSet.cpp
undefined:src/libxrpl/protocol/tokens.cpp
undefined:src/libxrpl/shamap/SHAMap.cpp
undefined:src/test/app/Batch_test.cpp
undefined:src/test/app/Invariants_test.cpp
undefined:src/test/app/NFToken_test.cpp
undefined:src/test/app/Offer_test.cpp
undefined:src/test/app/Path_test.cpp
undefined:src/test/basics/XRPAmount_test.cpp
undefined:src/test/beast/LexicalCast_test.cpp
undefined:src/test/jtx/impl/acctdelete.cpp
undefined:src/test/ledger/SkipList_test.cpp
undefined:src/test/rpc/Subscribe_test.cpp
undefined:src/tests/libxrpl/basics/RangeSet.cpp
undefined:src/xrpld/app/main/BasicApp.cpp
undefined:src/xrpld/app/main/BasicApp.cpp
undefined:src/xrpld/app/misc/detail/AmendmentTable.cpp
undefined:src/xrpld/app/misc/NetworkOPs.cpp
undefined:src/libxrpl/json/json_value.cpp
undefined:src/xrpld/app/paths/detail/StrandFlow.h
undefined:src/xrpld/app/tx/detail/NFTokenMint.cpp
undefined:src/xrpld/app/tx/detail/OracleSet.cpp
undefined:src/xrpld/core/detail/JobQueue.cpp
undefined:src/xrpld/core/detail/Workers.cpp
undefined:src/xrpld/rpc/detail/Role.cpp
undefined:src/xrpld/rpc/handlers/GetAggregatePrice.cpp
undefined:xrpl/basics/base_uint.h
undefined:xrpl/basics/DecayingSample.h
undefined:xrpl/beast/test/yield_to.h
undefined:xrpl/beast/xor_shift_engine.h
undefined:xrpl/nodestore/detail/varint.h
undefined:xrpl/peerfinder/detail/Counts.h
undefined:xrpl/protocol/nft.h
# basic_string.h:483:51: runtime error: unsigned integer overflow
unsigned-integer-overflow:basic_string.h
unsigned-integer-overflow:bits/chrono.h
unsigned-integer-overflow:bits/random.h
unsigned-integer-overflow:bits/random.tcc
unsigned-integer-overflow:bits/stl_algobase.h
unsigned-integer-overflow:bits/uniform_int_dist.h
unsigned-integer-overflow:string_view
# runtime error: unsigned integer overflow: 0 - 1 cannot be represented in type 'std::size_t' (aka 'unsigned long')
unsigned-integer-overflow:src/libxrpl/basics/base64.cpp
unsigned-integer-overflow:src/libxrpl/basics/Number.cpp
unsigned-integer-overflow:src/libxrpl/crypto/RFC1751.cpp
unsigned-integer-overflow:rc/libxrpl/json/json_value.cpp
unsigned-integer-overflow:src/libxrpl/ledger/ApplyView.cpp
unsigned-integer-overflow:src/libxrpl/ledger/View.cpp
unsigned-integer-overflow:src/libxrpl/protocol/Permissions.cpp
unsigned-integer-overflow:src/libxrpl/protocol/STAmount.cpp
unsigned-integer-overflow:src/libxrpl/protocol/STPathSet.cpp
unsigned-integer-overflow:src/libxrpl/protocol/tokens.cpp
unsigned-integer-overflow:src/libxrpl/shamap/SHAMap.cpp
unsigned-integer-overflow:src/test/app/Batch_test.cpp
unsigned-integer-overflow:src/test/app/Invariants_test.cpp
unsigned-integer-overflow:src/test/app/NFToken_test.cpp
unsigned-integer-overflow:src/test/app/Offer_test.cpp
unsigned-integer-overflow:src/test/app/Path_test.cpp
unsigned-integer-overflow:src/test/basics/XRPAmount_test.cpp
unsigned-integer-overflow:src/test/beast/LexicalCast_test.cpp
unsigned-integer-overflow:src/test/jtx/impl/acctdelete.cpp
unsigned-integer-overflow:src/test/ledger/SkipList_test.cpp
unsigned-integer-overflow:src/test/rpc/Subscribe_test.cpp
unsigned-integer-overflow:src/tests/libxrpl/basics/RangeSet.cpp
unsigned-integer-overflow:src/xrpld/app/main/BasicApp.cpp
unsigned-integer-overflow:src/xrpld/app/misc/detail/AmendmentTable.cpp
unsigned-integer-overflow:src/xrpld/app/misc/NetworkOPs.cpp
unsigned-integer-overflow:src/xrpld/app/paths/detail/StrandFlow.h
unsigned-integer-overflow:src/xrpld/app/tx/detail/NFTokenMint.cpp
unsigned-integer-overflow:src/xrpld/app/tx/detail/OracleSet.cpp
unsigned-integer-overflow:src/xrpld/rpc/detail/Role.cpp
unsigned-integer-overflow:src/xrpld/rpc/handlers/GetAggregatePrice.cpp
unsigned-integer-overflow:xrpl/basics/base_uint.h
unsigned-integer-overflow:xrpl/basics/DecayingSample.h
unsigned-integer-overflow:xrpl/beast/test/yield_to.h
unsigned-integer-overflow:xrpl/beast/xor_shift_engine.h
unsigned-integer-overflow:xrpl/nodestore/detail/varint.h
unsigned-integer-overflow:xrpl/peerfinder/detail/Counts.h
unsigned-integer-overflow:xrpl/protocol/nft.h
# Rippled intentional overflows and operations
# STAmount uses intentional negation of INT64_MIN and overflow in arithmetic
signed-integer-overflow:src/libxrpl/protocol/STAmount.cpp
unsigned-integer-overflow:src/libxrpl/protocol/STAmount.cpp
# XRPAmount test intentional overflows
signed-integer-overflow:src/test/basics/XRPAmount_test.cpp
# Peerfinder intentional overflow in counter arithmetic
unsigned-integer-overflow:src/xrpld/peerfinder/detail/Counts.h
# Signed integer overflow suppressions
signed-integer-overflow:src/test/beast/LexicalCast_test.cpp
# External library suppressions
unsigned-integer-overflow:nudb/detail/xxhash.hpp
# Loan_test.cpp intentional underflow in test arithmetic
unsigned-integer-overflow:src/test/app/Loan_test.cpp
undefined:src/test/app/Loan_test.cpp
# Source tree restructured paths (libxrpl/tx/transactors/)
# These duplicate the xrpld/app/tx/detail entries above for the new layout
unsigned-integer-overflow:src/libxrpl/tx/transactors/oracle/OracleSet.cpp
undefined:src/libxrpl/tx/transactors/oracle/OracleSet.cpp
unsigned-integer-overflow:src/libxrpl/tx/transactors/nft/NFTokenMint.cpp
undefined:src/libxrpl/tx/transactors/nft/NFTokenMint.cpp
# Protobuf intentional overflows in hash functions
# Protobuf uses intentional unsigned overflow for hash computation (stringpiece.h:393)
unsigned-integer-overflow:google/protobuf/stubs/stringpiece.h
# gRPC intentional overflows
# gRPC uses intentional overflow in timer calculations
# gRPC intentional overflows in timer calculations
unsigned-integer-overflow:grpc
unsigned-integer-overflow:timer_manager.cc
# Standard library intentional overflows
# These are intentional overflows in random number generation and character conversion
unsigned-integer-overflow:__random/seed_seq.h
unsigned-integer-overflow:__charconv/traits.h
# Suppress errors in RocksDB
# RocksDB uses intentional unsigned integer overflows in hash functions and CRC calculations
# RocksDB intentional unsigned integer overflows in hash functions and CRC calculations
unsigned-integer-overflow:rocks*/*/util/xxhash.h
unsigned-integer-overflow:rocks*/*/util/xxph3.h
unsigned-integer-overflow:rocks*/*/util/hash.cc
@@ -221,13 +90,14 @@ unsigned-integer-overflow:rocks*/*/table/format.cc
unsigned-integer-overflow:rocks*/*/table/block_based/block_based_table_builder.cc
unsigned-integer-overflow:rocks*/*/table/block_based/reader_common.cc
unsigned-integer-overflow:rocks*/*/db/version_set.cc
# RocksDB misaligned loads (intentional for performance on ARM64)
alignment:rocks*/*/util/crc32c_arm64.cc
undefined:rocks.*/*/util/crc32c_arm64.cc
undefined:rocks.*/*/util/xxhash.h
# nudb intentional overflows in hash functions
unsigned-integer-overflow:nudb/detail/xxhash.hpp
alignment:nudb/detail/xxhash.hpp
undefined:nudb
# Snappy compression library intentional overflows
unsigned-integer-overflow:snappy.cc
@@ -239,10 +109,39 @@ unsigned-integer-overflow:absl/base/internal/low_level_alloc.cc
unsigned-integer-overflow:absl/hash/internal/hash.h
unsigned-integer-overflow:absl/container/internal/raw_hash_set.h
# Standard library intentional overflows in chrono duration arithmetic
# Standard library intentional overflows
unsigned-integer-overflow:basic_string.h
unsigned-integer-overflow:bits/chrono.h
unsigned-integer-overflow:bits/random.h
unsigned-integer-overflow:bits/random.tcc
unsigned-integer-overflow:bits/stl_algobase.h
unsigned-integer-overflow:bits/uniform_int_dist.h
unsigned-integer-overflow:string_view
unsigned-integer-overflow:__random/seed_seq.h
unsigned-integer-overflow:__charconv/traits.h
unsigned-integer-overflow:__chrono/duration.h
# Suppress undefined errors in RocksDB and nudb
undefined:rocks.*/*/util/crc32c_arm64.cc
undefined:rocks.*/*/util/xxhash.h
undefined:nudb
# =============================================================================
# Rippled code suppressions
# =============================================================================
# Signed integer negation (-value) in amount types.
# INT64_MIN cannot occur in practice due to domain invariants (mantissa ranges
# are well within int64_t bounds), but UBSan flags the pattern as potential
# signed overflow.
signed-integer-overflow:IOUAmount
signed-integer-overflow:XRPAmount
signed-integer-overflow:MPTAmount
signed-integer-overflow:STAmount
# STAmount::operator+ signed addition — operands are bounded by total supply
# (~10^17 for XRP, ~10^18 for MPT) so overflow cannot occur in practice.
signed-integer-overflow:operator+*STAmount*
# STAmount::getRate uses unsigned shift and addition
unsigned-integer-overflow:getRate*
# STAmount::serialize uses unsigned bitwise operations
unsigned-integer-overflow:*STAmount*serialize*
# nft::cipheredTaxon uses intentional uint32 wraparound (LCG permutation)
unsigned-integer-overflow:cipheredTaxon

View File

@@ -107,7 +107,7 @@ encode(void* dest, void const* src, std::size_t len)
char const* in = static_cast<char const*>(src);
auto const tab = base64::get_alphabet();
for (auto n = len / 3; n != 0u; --n)
for (auto n = len / 3; n > 0; --n)
{
*out++ = tab[(in[0] & 0xfc) >> 2];
*out++ = tab[((in[0] & 0x03) << 4) + ((in[1] & 0xf0) >> 4)];

View File

@@ -179,7 +179,7 @@ SHAMap::finishFetch(SHAMapHash const& hash, std::shared_ptr<NodeObject> const& o
// See if a sync filter has a node
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMap::checkFilter(SHAMapHash const& hash, SHAMapSyncFilter const* filter) const
SHAMap::checkFilter(SHAMapHash const& hash, SHAMapSyncFilter* filter) const
{
if (auto nodeData = filter->getNode(hash))
{
@@ -205,7 +205,7 @@ SHAMap::checkFilter(SHAMapHash const& hash, SHAMapSyncFilter const* filter) cons
// Get a node without throwing
// Used on maps where missing nodes are expected
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMap::fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter const* filter) const
SHAMap::fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter* filter) const
{
auto node = cacheLookup(hash);
if (node)
@@ -318,7 +318,7 @@ SHAMap::descend(
SHAMapInnerNode* parent,
SHAMapNodeID const& parentID,
int branch,
SHAMapSyncFilter const* filter) const
SHAMapSyncFilter* filter) const
{
XRPL_ASSERT(parent->isInner(), "xrpl::SHAMap::descend : valid parent input");
XRPL_ASSERT(
@@ -347,7 +347,7 @@ SHAMapTreeNode*
SHAMap::descendAsync(
SHAMapInnerNode* parent,
int branch,
SHAMapSyncFilter const* filter,
SHAMapSyncFilter* filter,
bool& pending,
descendCallback&& callback) const
{

View File

@@ -122,9 +122,7 @@ selectBranch(SHAMapNodeID const& id, uint256 const& hash)
SHAMapNodeID
SHAMapNodeID::createID(int depth, uint256 const& key)
{
XRPL_ASSERT(
depth >= 0 && depth <= SHAMap::leafDepth,
"xrpl::SHAMapNodeID::createID : valid branch input");
XRPL_ASSERT((depth >= 0) && (depth < 65), "xrpl::SHAMapNodeID::createID : valid branch input");
return SHAMapNodeID(depth, key & depthMask(depth));
}

View File

@@ -392,7 +392,7 @@ SHAMap::getMissingNodes(int max, SHAMapSyncFilter* filter)
bool
SHAMap::getNodeFat(
SHAMapNodeID const& wanted,
std::vector<std::tuple<SHAMapNodeID, Blob, bool>>& data,
std::vector<std::pair<SHAMapNodeID, Blob>>& data,
bool fatLeaves,
std::uint32_t depth) const
{
@@ -438,7 +438,7 @@ SHAMap::getNodeFat(
// Add this node to the reply
s.erase();
node->serializeForWire(s);
data.emplace_back(std::make_tuple(nodeID, s.getData(), node->isLeaf()));
data.emplace_back(std::make_pair(nodeID, s.getData()));
if (node->isInner())
{
@@ -468,8 +468,7 @@ SHAMap::getNodeFat(
// Just include this node
s.erase();
childNode->serializeForWire(s);
data.emplace_back(
std::make_tuple(childID, s.getData(), childNode->isLeaf()));
data.emplace_back(std::make_pair(childID, s.getData()));
}
}
}
@@ -487,18 +486,8 @@ SHAMap::serializeRoot(Serializer& s) const
}
SHAMapAddNode
SHAMap::addRootNode(
SHAMapHash const& hash,
intr_ptr::SharedPtr<SHAMapTreeNode> rootNode,
SHAMapSyncFilter const* filter)
SHAMap::addRootNode(SHAMapHash const& hash, Slice const& rootNode, SHAMapSyncFilter* filter)
{
XRPL_ASSERT(rootNode, "xrpl::SHAMap::addRootNode : non-null root node");
if (!rootNode)
{
JLOG(journal_.error()) << "Null node received";
return SHAMapAddNode::invalid();
}
// we already have a root_ node
if (root_->getHash().isNonZero())
{
@@ -508,16 +497,14 @@ SHAMap::addRootNode(
}
XRPL_ASSERT(cowid_ >= 1, "xrpl::SHAMap::addRootNode : valid cowid");
if (rootNode->getHash() != hash)
{
JLOG(journal_.warn()) << "Corrupt node received";
auto node = SHAMapTreeNode::makeFromWire(rootNode);
if (!node || node->getHash() != hash)
return SHAMapAddNode::invalid();
}
if (backed_)
canonicalize(hash, rootNode);
canonicalize(hash, node);
root_ = std::move(rootNode);
root_ = node;
if (root_->isLeaf())
clearSynching();
@@ -534,23 +521,9 @@ SHAMap::addRootNode(
}
SHAMapAddNode
SHAMap::addKnownNode(
SHAMapNodeID const& nodeID,
intr_ptr::SharedPtr<SHAMapTreeNode> treeNode,
SHAMapSyncFilter const* filter)
SHAMap::addKnownNode(SHAMapNodeID const& node, Slice const& rawNode, SHAMapSyncFilter* filter)
{
XRPL_ASSERT(!nodeID.isRoot(), "xrpl::SHAMap::addKnownNode : valid node input");
if (nodeID.isRoot())
{
JLOG(journal_.error()) << "Root node received";
return SHAMapAddNode::invalid();
}
XRPL_ASSERT(treeNode, "xrpl::SHAMap::addKnownNode : non-null tree node");
if (!treeNode)
{
JLOG(journal_.error()) << "Null node received";
return SHAMapAddNode::invalid();
}
XRPL_ASSERT(!node.isRoot(), "xrpl::SHAMap::addKnownNode : valid node input");
if (!isSynching())
{
@@ -564,14 +537,14 @@ SHAMap::addKnownNode(
while (currNode->isInner() &&
!safe_downcast<SHAMapInnerNode*>(currNode)->isFullBelow(generation) &&
(currNodeID.getDepth() < nodeID.getDepth()))
(currNodeID.getDepth() < node.getDepth()))
{
int const branch = selectBranch(currNodeID, nodeID.getNodeID());
int const branch = selectBranch(currNodeID, node.getNodeID());
XRPL_ASSERT(branch >= 0, "xrpl::SHAMap::addKnownNode : valid branch");
auto inner = safe_downcast<SHAMapInnerNode*>(currNode);
if (inner->isEmptyBranch(branch))
{
JLOG(journal_.warn()) << "Add known node for empty branch" << nodeID;
JLOG(journal_.warn()) << "Add known node for empty branch" << node;
return SHAMapAddNode::invalid();
}
@@ -587,44 +560,67 @@ SHAMap::addKnownNode(
if (currNode != nullptr)
continue;
if (childHash != treeNode->getHash())
auto newNode = SHAMapTreeNode::makeFromWire(rawNode);
if (!newNode || childHash != newNode->getHash())
{
JLOG(journal_.warn()) << "Corrupt node received";
return SHAMapAddNode::invalid();
}
// In rare cases, a node can still be corrupt even after hash
// validation. For leaf nodes, we perform an additional check to
// ensure the node's position in the tree is consistent with its
// content to prevent inconsistencies that could
// propagate further down the line.
if (newNode->isLeaf())
{
auto const& actualKey =
safe_downcast<SHAMapLeafNode const*>(newNode.get())->peekItem()->key();
// Validate that this leaf belongs at the target position
auto const expectedNodeID = SHAMapNodeID::createID(node.getDepth(), actualKey);
if (expectedNodeID.getNodeID() != node.getNodeID())
{
JLOG(journal_.debug())
<< "Leaf node position mismatch: "
<< "expected=" << expectedNodeID.getNodeID() << ", actual=" << node.getNodeID();
return SHAMapAddNode::invalid();
}
}
// Inner nodes must be at a level strictly less than 64
// but leaf nodes (while notionally at level 64) can be
// at any depth up to and including 64:
if ((currNodeID.getDepth() > leafDepth) ||
(treeNode->isInner() && currNodeID.getDepth() == leafDepth))
(newNode->isInner() && currNodeID.getDepth() == leafDepth))
{
// Map is provably invalid
state_ = SHAMapState::Invalid;
return SHAMapAddNode::useful();
}
if (currNodeID != nodeID)
if (currNodeID != node)
{
// Either this node is broken or we didn't request it (yet)
JLOG(journal_.warn()) << "unable to hook node " << nodeID;
JLOG(journal_.warn()) << "unable to hook node " << node;
JLOG(journal_.info()) << " stuck at " << currNodeID;
JLOG(journal_.info()) << "got depth=" << nodeID.getDepth()
JLOG(journal_.info()) << "got depth=" << node.getDepth()
<< ", walked to= " << currNodeID.getDepth();
return SHAMapAddNode::useful();
}
if (backed_)
canonicalize(childHash, treeNode);
canonicalize(childHash, newNode);
treeNode = prevNode->canonicalizeChild(branch, std::move(treeNode));
newNode = prevNode->canonicalizeChild(branch, std::move(newNode));
if (filter != nullptr)
{
Serializer s;
treeNode->serializeWithPrefix(s);
newNode->serializeWithPrefix(s);
filter->gotNode(
false, childHash, ledgerSeq_, std::move(s.modData()), treeNode->getType());
false, childHash, ledgerSeq_, std::move(s.modData()), newNode->getType());
}
return SHAMapAddNode::useful();

View File

@@ -1,356 +0,0 @@
#include <test/shamap/common.h>
#include <xrpld/app/ledger/detail/LedgerNodeHelpers.h>
#include <xrpl/beast/unit_test.h>
#include <xrpl/protocol/messages.h>
#include <xrpl/shamap/SHAMap.h>
#include <xrpl/shamap/SHAMapAccountStateLeafNode.h>
#include <xrpl/shamap/SHAMapInnerNode.h>
#include <xrpl/shamap/SHAMapItem.h>
#include <xrpl/shamap/SHAMapTreeNode.h>
#include <bit>
namespace xrpl {
namespace tests {
class LedgerNodeHelpers_test : public beast::unit_test::suite
{
// Helper function to create a simple SHAMapItem for testing.
static boost::intrusive_ptr<SHAMapItem>
makeTestItem(std::uint32_t seed)
{
Serializer s;
s.add32(seed);
s.add32(seed + 1);
s.add32(seed + 2);
return make_shamapitem(s.getSHA512Half(), s.slice());
}
// Helper function to serialize a tree node to wire format.
static std::string
serializeNode(intr_ptr::SharedPtr<SHAMapTreeNode> const& node)
{
Serializer s;
node->serializeForWire(s);
auto const slice = s.slice();
return std::string(std::bit_cast<char const*>(slice.data()), slice.size());
}
void
testValidateLedgerNode()
{
// In the tests below the validity of the content of the node data and ID fields is not
// checked - only that the fields have values when expected. The content of the fields is
// verified in the other tests in this file.
testcase("validateLedgerNode");
// Invalid: missing all fields.
{
protocol::TMLedgerNode const node;
BEAST_EXPECT(!validateLedgerNode(node));
}
// Invalid: missing `nodedata` field.
{
protocol::TMLedgerNode node;
node.set_nodeid("test_nodeid");
BEAST_EXPECT(!validateLedgerNode(node));
}
// Invalid: missing `nodedata` field.
{
protocol::TMLedgerNode node;
node.set_id("test_nodeid");
BEAST_EXPECT(!validateLedgerNode(node));
}
// Invalid: missing `nodedata` field.
{
protocol::TMLedgerNode node;
node.set_depth(1);
BEAST_EXPECT(!validateLedgerNode(node));
}
// Valid: legacy `nodeid` field.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_nodeid("test_nodeid");
BEAST_EXPECT(validateLedgerNode(node));
}
// Invalid: has both legacy `nodeid` and new `id` fields.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_nodeid("test_nodeid");
node.set_id("test_nodeid");
BEAST_EXPECT(!validateLedgerNode(node));
}
// Invalid: has both legacy `nodeid` and new `depth` fields.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_nodeid("test_nodeid");
node.set_depth(5);
BEAST_EXPECT(!validateLedgerNode(node));
}
// Valid: new `id` field.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_id("test_id");
BEAST_EXPECT(validateLedgerNode(node));
}
// Valid: new `depth` field.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_depth(5);
BEAST_EXPECT(validateLedgerNode(node));
}
// Valid: `depth` at minimum depth.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_depth(0);
BEAST_EXPECT(validateLedgerNode(node));
}
// Valid: `depth` at arbitrary depth between minimum and maximum.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_depth(10);
BEAST_EXPECT(validateLedgerNode(node));
}
// Valid: `depth` at maximum depth.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_depth(SHAMap::leafDepth);
BEAST_EXPECT(validateLedgerNode(node));
}
// Invalid: `depth` is greater than maximum depth.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_depth(SHAMap::leafDepth + 1);
BEAST_EXPECT(!validateLedgerNode(node));
}
}
void
testGetTreeNode()
{
testcase("getTreeNode");
// Valid: inner node. It must have at least one child for `serializeNode` to work.
{
auto const innerNode = intr_ptr::make_shared<SHAMapInnerNode>(1);
auto const childNode = intr_ptr::make_shared<SHAMapInnerNode>(1);
innerNode->setChild(0, childNode);
auto const innerData = serializeNode(innerNode);
auto const result = getTreeNode(innerData);
BEAST_EXPECT(result.has_value());
BEAST_EXPECT((*result)->isInner());
}
// Valid: leaf node.
{
auto const leafItem = makeTestItem(12345);
auto const leafNode = intr_ptr::make_shared<SHAMapAccountStateLeafNode>(leafItem, 1);
auto const leafData = serializeNode(leafNode);
auto result = getTreeNode(leafData);
BEAST_EXPECT(result.has_value());
BEAST_EXPECT((*result)->isLeaf());
}
// Invalid: empty data.
{
auto const result = getTreeNode("");
BEAST_EXPECT(!result.has_value());
}
// Invalid: garbage data.
{
auto const result = getTreeNode("invalid");
BEAST_EXPECT(!result.has_value());
}
// Invalid: truncated data.
{
auto const leafItem = makeTestItem(54321);
auto const leafNode = intr_ptr::make_shared<SHAMapAccountStateLeafNode>(leafItem, 1);
// Truncate the data to trigger an exception in SHAMapTreeNode::makeAccountState when
// the data is used to deserialize the node.
uint256 const tag;
auto const leafData = serializeNode(leafNode).substr(0, tag.bytes - 1);
auto const result = getTreeNode(leafData);
BEAST_EXPECT(!result.has_value());
}
}
void
testGetSHAMapNodeID()
{
testcase("getSHAMapNodeID");
{
// Tests using inner nodes at various depths.
auto const innerNode = intr_ptr::make_shared<SHAMapInnerNode>(1);
auto const childNode = intr_ptr::make_shared<SHAMapInnerNode>(1);
innerNode->setChild(0, childNode);
auto const innerData = serializeNode(innerNode);
// Valid: legacy `nodeid` field at arbitrary depth.
{
auto const innerDepth = 3;
auto const innerID = SHAMapNodeID::createID(innerDepth, uint256{});
protocol::TMLedgerNode node;
node.set_nodedata(innerData);
node.set_nodeid(innerID.getRawString());
auto const result = getSHAMapNodeID(node, innerNode);
BEAST_EXPECT(result.has_value());
BEAST_EXPECT(*result == innerID);
}
// Valid: new `id` field at minimum depth.
{
auto const innerDepth = 0;
auto const innerID = SHAMapNodeID::createID(innerDepth, uint256{});
protocol::TMLedgerNode node;
node.set_nodedata(innerData);
node.set_id(innerID.getRawString());
auto const result = getSHAMapNodeID(node, innerNode);
BEAST_EXPECT(result.has_value());
BEAST_EXPECT(*result == innerID);
}
// Invalid: new `depth` field should not be used for inner nodes.
{
protocol::TMLedgerNode node;
node.set_nodedata(innerData);
node.set_depth(10);
auto const result = getSHAMapNodeID(node, innerNode);
BEAST_EXPECT(!result.has_value());
}
}
{
// Tests using leaf nodes at various depths.
auto const leafItem = makeTestItem(12345);
auto const leafNode = intr_ptr::make_shared<SHAMapAccountStateLeafNode>(leafItem, 1);
auto const leafData = serializeNode(leafNode);
auto const leafKey = leafItem->key();
// Valid: legacy `nodeid` field at arbitrary depth.
{
auto const leafDepth = 5;
auto const leafID = SHAMapNodeID::createID(leafDepth, leafKey);
protocol::TMLedgerNode ledgerNode;
ledgerNode.set_nodedata(leafData);
ledgerNode.set_nodeid(leafID.getRawString());
auto result = getSHAMapNodeID(ledgerNode, leafNode);
BEAST_EXPECT(result.has_value());
BEAST_EXPECT(*result == leafID);
}
// Invalid: new `id` field should not be used for leaf nodes.
{
auto const leafDepth = 5;
auto const leafID = SHAMapNodeID::createID(leafDepth, leafKey);
protocol::TMLedgerNode ledgerNode;
ledgerNode.set_nodedata(leafData);
ledgerNode.set_id(leafID.getRawString());
auto result = getSHAMapNodeID(ledgerNode, leafNode);
BEAST_EXPECT(!result.has_value());
}
// Valid: new `depth` field at minimum depth.
{
auto const leafDepth = 0;
auto const leafID = SHAMapNodeID::createID(leafDepth, leafKey);
protocol::TMLedgerNode node;
node.set_nodedata(leafData);
node.set_depth(leafDepth);
auto result = getSHAMapNodeID(node, leafNode);
BEAST_EXPECT(result.has_value());
BEAST_EXPECT(*result == leafID);
}
// Valid: new `depth` field at arbitrary depth between minimum and maximum.
{
auto const leafDepth = 10;
auto const leafID = SHAMapNodeID::createID(leafDepth, leafKey);
protocol::TMLedgerNode ledgerNode;
ledgerNode.set_nodedata(leafData);
ledgerNode.set_depth(leafDepth);
auto result = getSHAMapNodeID(ledgerNode, leafNode);
BEAST_EXPECT(result.has_value());
BEAST_EXPECT(*result == leafID);
}
// Valid: new `depth` field at maximum depth.
// Note that we do not test a depth greater than the maximum depth, because the proto
// message is assumed to have been validated by the time the getSHAMapNodeID function is
// called.
{
auto const leafDepth = SHAMap::leafDepth;
auto const leafID = SHAMapNodeID::createID(leafDepth, leafKey);
protocol::TMLedgerNode node;
node.set_nodedata(leafData);
node.set_depth(leafDepth);
auto result = getSHAMapNodeID(node, leafNode);
BEAST_EXPECT(result.has_value());
BEAST_EXPECT(*result == leafID);
}
// Invalid: legacy `nodeid` field where the node ID is inconsistent with the key.
{
auto const otherItem = makeTestItem(54321);
auto const otherNode =
intr_ptr::make_shared<SHAMapAccountStateLeafNode>(otherItem, 1);
auto const otherData = serializeNode(otherNode);
auto const otherKey = otherItem->key();
auto const otherDepth = 1;
auto const otherID = SHAMapNodeID::createID(otherDepth, otherKey);
protocol::TMLedgerNode ledgerNode;
ledgerNode.set_nodedata(otherData);
ledgerNode.set_nodeid(otherID.getRawString());
auto result = getSHAMapNodeID(ledgerNode, leafNode);
BEAST_EXPECT(!result.has_value());
}
}
}
public:
void
run() override
{
testValidateLedgerNode();
testGetTreeNode();
testGetSHAMapNodeID();
}
};
BEAST_DEFINE_TESTSUITE(LedgerNodeHelpers, app, xrpl);
} // namespace tests
} // namespace xrpl

View File

@@ -19,7 +19,7 @@ public:
testInteger(IntType in)
{
std::string s;
IntType out(in + 1);
IntType out{}; // Initialize to avoid relying on overflow behavior
expect(lexicalCastChecked(s, in));
expect(lexicalCastChecked(out, s));

View File

@@ -60,8 +60,8 @@ public:
negotiateProtocolVersion("RTXP/1.2, XRPL/2.0, XRPL/2.1") == make_protocol(2, 1));
BEAST_EXPECT(negotiateProtocolVersion("XRPL/2.2") == make_protocol(2, 2));
BEAST_EXPECT(
negotiateProtocolVersion("RTXP/1.2, XRPL/2.3, XRPL/2.4, XRPL/999.999") ==
make_protocol(2, 3));
negotiateProtocolVersion("RTXP/1.2, XRPL/2.2, XRPL/2.3, XRPL/999.999") ==
make_protocol(2, 2));
BEAST_EXPECT(negotiateProtocolVersion("XRPL/999.999, WebSocket/1.0") == std::nullopt);
BEAST_EXPECT(negotiateProtocolVersion("") == std::nullopt);
}

View File

@@ -99,17 +99,14 @@ public:
destination.setSynching();
{
std::vector<std::tuple<SHAMapNodeID, Blob, bool>> a;
std::vector<std::pair<SHAMapNodeID, Blob>> a;
BEAST_EXPECT(source.getNodeFat(SHAMapNodeID(), a, rand_bool(eng_), rand_int(eng_, 2)));
unexpected(a.empty(), "NodeSize");
auto node = SHAMapTreeNode::makeFromWire(makeSlice(std::get<1>(a[0])));
if (!node)
fail("", __FILE__, __LINE__);
BEAST_EXPECT(
destination.addRootNode(source.getHash(), std::move(node), nullptr).isGood());
BEAST_EXPECT(destination.addRootNode(source.getHash(), makeSlice(a[0].second), nullptr)
.isGood());
}
do
@@ -123,7 +120,7 @@ public:
break;
// get as many nodes as possible based on this information
std::vector<std::tuple<SHAMapNodeID, Blob, bool>> b;
std::vector<std::pair<SHAMapNodeID, Blob>> b;
for (auto& it : nodesMissing)
{
@@ -145,10 +142,7 @@ public:
// Don't use BEAST_EXPECT here b/c it will be called a
// non-deterministic number of times and the number of tests run
// should be deterministic
auto node = SHAMapTreeNode::makeFromWire(makeSlice(std::get<1>(b[i])));
if (!node)
fail("", __FILE__, __LINE__);
if (!destination.addKnownNode(std::get<0>(b[i]), std::move(node), nullptr)
if (!destination.addKnownNode(b[i].first, makeSlice(b[i].second), nullptr)
.isUseful())
fail("", __FILE__, __LINE__);
}

View File

@@ -9,7 +9,6 @@
#include <mutex>
#include <set>
#include <string_view>
#include <utility>
namespace xrpl {
@@ -132,16 +131,16 @@ private:
processData(std::shared_ptr<Peer> peer, protocol::TMLedgerData& data);
bool
takeHeader(std::string_view data);
takeHeader(std::string const& data);
void
receiveNode(protocol::TMLedgerData& packet, SHAMapAddNode& san);
receiveNode(protocol::TMLedgerData& packet, SHAMapAddNode&);
bool
takeTxRootNode(std::string_view data, SHAMapAddNode& san);
takeTxRootNode(Slice const& data, SHAMapAddNode&);
bool
takeAsRootNode(std::string_view data, SHAMapAddNode& san);
takeAsRootNode(Slice const& data, SHAMapAddNode&);
std::vector<uint256>
neededTxHashes(int max, SHAMapSyncFilter* filter) const;

View File

@@ -3,7 +3,6 @@
#include <xrpld/app/ledger/InboundLedgers.h>
#include <xrpld/app/ledger/LedgerMaster.h>
#include <xrpld/app/ledger/TransactionStateSF.h>
#include <xrpld/app/ledger/detail/LedgerNodeHelpers.h>
#include <xrpld/app/main/Application.h>
#include <xrpld/overlay/Overlay.h>
@@ -763,7 +762,7 @@ InboundLedger::filterNodes(
*/
// data must not have hash prefix
bool
InboundLedger::takeHeader(std::string_view data)
InboundLedger::takeHeader(std::string const& data)
{
// Return value: true=normal, false=bad data
JLOG(journal_.trace()) << "got header acquiring ledger " << hash_;
@@ -851,31 +850,20 @@ InboundLedger::receiveNode(protocol::TMLedgerData& packet, SHAMapAddNode& san)
{
auto const f = filter.get();
for (auto const& ledger_node : packet.nodes())
for (auto const& node : packet.nodes())
{
auto treeNode = getTreeNode(ledger_node.nodedata());
if (!treeNode)
{
JLOG(journal_.warn()) << "Got invalid node data";
san.incInvalid();
return;
}
auto const nodeID = deserializeSHAMapNodeID(node.nodeid());
auto const nodeID = getSHAMapNodeID(ledger_node, *treeNode);
if (!nodeID)
{
JLOG(journal_.warn()) << "Got invalid node id";
san.incInvalid();
return;
}
throw std::runtime_error("data does not properly deserialize");
if (nodeID->isRoot())
{
san += map.addRootNode(rootHash, std::move(*treeNode), f);
san += map.addRootNode(rootHash, makeSlice(node.nodedata()), f);
}
else
{
san += map.addKnownNode(*nodeID, std::move(*treeNode), f);
san += map.addKnownNode(*nodeID, makeSlice(node.nodedata()), f);
}
if (!san.isGood())
@@ -915,7 +903,7 @@ InboundLedger::receiveNode(protocol::TMLedgerData& packet, SHAMapAddNode& san)
Call with a lock
*/
bool
InboundLedger::takeAsRootNode(std::string_view data, SHAMapAddNode& san)
InboundLedger::takeAsRootNode(Slice const& data, SHAMapAddNode& san)
{
if (failed_ || mHaveState)
{
@@ -931,17 +919,9 @@ InboundLedger::takeAsRootNode(std::string_view data, SHAMapAddNode& san)
// LCOV_EXCL_STOP
}
auto treeNode = getTreeNode(data);
if (!treeNode)
{
JLOG(journal_.warn()) << "Got invalid node data";
san.incInvalid();
return false;
}
AccountStateSF filter(mLedger->stateMap().family().db(), app_.getLedgerMaster());
san += mLedger->stateMap().addRootNode(
SHAMapHash{mLedger->header().accountHash}, std::move(*treeNode), &filter);
san +=
mLedger->stateMap().addRootNode(SHAMapHash{mLedger->header().accountHash}, data, &filter);
return san.isGood();
}
@@ -949,7 +929,7 @@ InboundLedger::takeAsRootNode(std::string_view data, SHAMapAddNode& san)
Call with a lock
*/
bool
InboundLedger::takeTxRootNode(std::string_view data, SHAMapAddNode& san)
InboundLedger::takeTxRootNode(Slice const& data, SHAMapAddNode& san)
{
if (failed_ || mHaveTransactions)
{
@@ -965,17 +945,8 @@ InboundLedger::takeTxRootNode(std::string_view data, SHAMapAddNode& san)
// LCOV_EXCL_STOP
}
auto treeNode = getTreeNode(data);
if (!treeNode)
{
JLOG(journal_.warn()) << "Got invalid node data";
san.incInvalid();
return false;
}
TransactionStateSF filter(mLedger->txMap().family().db(), app_.getLedgerMaster());
san += mLedger->txMap().addRootNode(
SHAMapHash{mLedger->header().txHash}, std::move(*treeNode), &filter);
san += mLedger->txMap().addRootNode(SHAMapHash{mLedger->header().txHash}, data, &filter);
return san.isGood();
}
@@ -1072,13 +1043,13 @@ InboundLedger::processData(std::shared_ptr<Peer> peer, protocol::TMLedgerData& p
}
if (!mHaveState && (packet.nodes().size() > 1) &&
!takeAsRootNode(packet.nodes(1).nodedata(), san))
!takeAsRootNode(makeSlice(packet.nodes(1).nodedata()), san))
{
JLOG(journal_.warn()) << "Included AS root invalid";
}
if (!mHaveTransactions && (packet.nodes().size() > 2) &&
!takeTxRootNode(packet.nodes(2).nodedata(), san))
!takeTxRootNode(makeSlice(packet.nodes(2).nodedata()), san))
{
JLOG(journal_.warn()) << "Included TX root invalid";
}
@@ -1109,13 +1080,13 @@ InboundLedger::processData(std::shared_ptr<Peer> peer, protocol::TMLedgerData& p
ScopedLockType const sl(mtx_);
// Verify nodes are complete
for (auto const& ledger_node : packet.nodes())
// Verify node IDs and data are complete
for (auto const& node : packet.nodes())
{
if (!validateLedgerNode(ledger_node))
if (!node.has_nodeid() || !node.has_nodedata())
{
JLOG(journal_.warn()) << "Got malformed ledger node";
peer->charge(Resource::feeMalformedRequest, "ledger_node");
JLOG(journal_.warn()) << "Got bad node";
peer->charge(Resource::feeMalformedRequest, "ledger_data bad node");
return -1;
}
}

View File

@@ -1,6 +1,5 @@
#include <xrpld/app/ledger/InboundLedgers.h>
#include <xrpld/app/ledger/LedgerMaster.h>
#include <xrpld/app/ledger/detail/LedgerNodeHelpers.h>
#include <xrpld/app/main/Application.h>
#include <xrpl/basics/DecayingSample.h>
@@ -224,21 +223,23 @@ public:
Serializer s;
try
{
for (auto const& ledger_node : packet_ptr->nodes())
for (int i = 0; i < packet_ptr->nodes().size(); ++i)
{
if (!validateLedgerNode(ledger_node))
auto const& node = packet_ptr->nodes(i);
if (!node.has_nodeid() || !node.has_nodedata())
return;
auto const treeNode = getTreeNode(ledger_node.nodedata());
if (!treeNode)
auto newNode = SHAMapTreeNode::makeFromWire(makeSlice(node.nodedata()));
if (!newNode)
return;
auto const tn = *treeNode;
s.erase();
tn->serializeWithPrefix(s);
newNode->serializeWithPrefix(s);
app_.getLedgerMaster().addFetchPack(
tn->getHash().as_uint256(), std::make_shared<Blob>(s.begin(), s.end()));
newNode->getHash().as_uint256(), std::make_shared<Blob>(s.begin(), s.end()));
}
}
catch (std::exception const&) // NOLINT(bugprone-empty-catch)

View File

@@ -1,6 +1,5 @@
#include <xrpld/app/ledger/InboundLedgers.h>
#include <xrpld/app/ledger/InboundTransactions.h>
#include <xrpld/app/ledger/detail/LedgerNodeHelpers.h>
#include <xrpld/app/ledger/detail/TransactionAcquire.h>
#include <xrpld/app/main/Application.h>
@@ -132,35 +131,26 @@ public:
return;
}
std::vector<std::pair<SHAMapNodeID, intr_ptr::SharedPtr<SHAMapTreeNode>>> data;
std::vector<std::pair<SHAMapNodeID, Slice>> data;
data.reserve(packet.nodes().size());
for (auto const& ledger_node : packet.nodes())
for (auto const& node : packet.nodes())
{
if (!validateLedgerNode(ledger_node))
if (!node.has_nodeid() || !node.has_nodedata())
{
JLOG(j_.warn()) << "Got malformed ledger node";
peer->charge(Resource::feeMalformedRequest, "ledger_node");
peer->charge(Resource::feeMalformedRequest, "ledger_data");
return;
}
auto treeNode = getTreeNode(ledger_node.nodedata());
if (!treeNode)
auto const id = deserializeSHAMapNodeID(node.nodeid());
if (!id)
{
JLOG(j_.warn()) << "Got invalid node data";
peer->charge(Resource::feeInvalidData, "node_data");
peer->charge(Resource::feeInvalidData, "ledger_data");
return;
}
auto const nodeID = getSHAMapNodeID(ledger_node, *treeNode);
if (!nodeID)
{
JLOG(j_.warn()) << "Got invalid node id";
peer->charge(Resource::feeInvalidData, "node_id");
return;
}
data.emplace_back(std::make_pair(*nodeID, std::move(*treeNode)));
data.emplace_back(std::make_pair(*id, makeSlice(node.nodedata())));
}
if (!ta->takeNodes(data, peer).isUseful())

View File

@@ -1,95 +0,0 @@
#include <xrpld/app/ledger/detail/LedgerNodeHelpers.h>
#include <xrpl/basics/IntrusivePointer.h>
#include <xrpl/basics/Slice.h>
#include <xrpl/basics/safe_cast.h>
#include <xrpl/beast/utility/instrumentation.h>
#include <xrpl/protocol/messages.h>
#include <xrpl/shamap/SHAMap.h>
#include <xrpl/shamap/SHAMapLeafNode.h>
#include <xrpl/shamap/SHAMapNodeID.h>
#include <xrpl/shamap/SHAMapTreeNode.h>
#include <optional>
#include <string>
namespace xrpl {
bool
validateLedgerNode(protocol::TMLedgerNode const& ledger_node)
{
if (!ledger_node.has_nodedata())
return false;
if (ledger_node.has_nodeid())
return !ledger_node.has_id() && !ledger_node.has_depth();
return ledger_node.has_id() ||
(ledger_node.has_depth() && ledger_node.depth() <= SHAMap::leafDepth);
}
std::optional<intr_ptr::SharedPtr<SHAMapTreeNode>>
getTreeNode(std::string_view data)
{
auto const slice = makeSlice(data);
try
{
auto treeNode = SHAMapTreeNode::makeFromWire(slice);
if (!treeNode)
return std::nullopt;
return treeNode;
}
catch (std::exception const&)
{
return std::nullopt;
}
}
std::optional<SHAMapNodeID>
getSHAMapNodeID(
protocol::TMLedgerNode const& ledger_node,
intr_ptr::SharedPtr<SHAMapTreeNode> const& treeNode)
{
if (ledger_node.has_id() || ledger_node.has_depth())
{
if (treeNode->isInner())
{
if (!ledger_node.has_id())
return std::nullopt;
return deserializeSHAMapNodeID(ledger_node.id());
}
if (treeNode->isLeaf())
{
if (!ledger_node.has_depth())
return std::nullopt;
auto const key =
safe_downcast<SHAMapLeafNode const*>(treeNode.get())->peekItem()->key();
return SHAMapNodeID::createID(ledger_node.depth(), key);
}
UNREACHABLE("xrpl::getSHAMapNodeID : tree node is neither inner nor leaf");
return std::nullopt;
}
if (!ledger_node.has_nodeid())
return std::nullopt;
auto nodeID = deserializeSHAMapNodeID(ledger_node.nodeid());
if (!nodeID.has_value())
return std::nullopt;
if (treeNode->isLeaf())
{
auto const key = safe_downcast<SHAMapLeafNode const*>(treeNode.get())->peekItem()->key();
auto const expected_id = SHAMapNodeID::createID(static_cast<int>(nodeID->getDepth()), key);
if (nodeID->getNodeID() != expected_id.getNodeID())
return std::nullopt;
}
return nodeID;
}
} // namespace xrpl

View File

@@ -1,75 +0,0 @@
#pragma once
#include <xrpl/basics/IntrusivePointer.h>
#include <xrpl/shamap/SHAMapNodeID.h>
#include <xrpl/shamap/SHAMapTreeNode.h>
#include <optional>
#include <string_view>
namespace protocol {
class TMLedgerNode;
} // namespace protocol
namespace xrpl {
/**
* @brief Validates a ledger node proto message.
*
* This function checks whether a ledger node has the expected fields (for non-ledger base data):
* - The node must have `nodedata`.
* - If the legacy `nodeid` field is present then the new `id` and `depth` fields must not be
* present.
* - If the new `id` or `depth` fields are present (it is a oneof field, so only one of the two can
* be set) then the legacy `nodeid` must not be present.
* - If the `depth` field is present then it must be between 0 and SHAMap::leafDepth (inclusive).
*
* @param ledger_node The ledger node to validate.
* @return true if the ledger node has the expected fields, false otherwise.
*/
[[nodiscard]] bool
validateLedgerNode(protocol::TMLedgerNode const& ledger_node);
/**
* @brief Deserializes a SHAMapTreeNode from wire format data.
*
* This function attempts to create a SHAMapTreeNode from the provided data string. If the data is
* malformed or deserialization fails, the function returns std::nullopt instead of throwing an
* exception.
*
* @param data The serialized node data in wire format.
* @return An optional containing the deserialized tree node if successful, or std::nullopt if
* deserialization fails.
*/
[[nodiscard]] std::optional<intr_ptr::SharedPtr<SHAMapTreeNode>>
getTreeNode(std::string_view data);
/**
* @brief Extracts or reconstructs the SHAMapNodeID from a ledger node proto message.
*
* This function retrieves the SHAMapNodeID for a tree node, with behavior that depends on which
* field is set and the node type (inner vs. leaf).
*
* When the legacy `nodeid` field is set in the message:
* - For all nodes: Deserializes the node ID from the field.
* - For leaf nodes: Validates that the node ID is consistent with the leaf's key.
*
* When the new `id` or `depth` field is set in the message:
* - For inner nodes: Deserializes the node ID from the `id` field. Note that root nodes are also
* inner nodes.
* - For leaf nodes: Reconstructs the node ID using both the depth from the `depth` field and the
* key from the leaf node's item.
*
* @param ledger_node The validated protocol message containing the ledger node data.
* @param treeNode The deserialized tree node (inner or leaf node).
* @return An optional containing the node ID if extraction/reconstruction succeeds, or std::nullopt
* if the required fields are missing or validation fails.
* @note This function expects that the caller has already validated the ledger node by calling the
* `validateLedgerNode` function and obtained a valid tree node by calling `getTreeNode`.
*/
[[nodiscard]] std::optional<SHAMapNodeID>
getSHAMapNodeID(
protocol::TMLedgerNode const& ledger_node,
intr_ptr::SharedPtr<SHAMapTreeNode> const& treeNode);
} // namespace xrpl

View File

@@ -158,7 +158,7 @@ TransactionAcquire::trigger(std::shared_ptr<Peer> const& peer)
SHAMapAddNode
TransactionAcquire::takeNodes(
std::vector<std::pair<SHAMapNodeID, intr_ptr::SharedPtr<SHAMapTreeNode>>> const& data,
std::vector<std::pair<SHAMapNodeID, Slice>> const& data,
std::shared_ptr<Peer> const& peer)
{
ScopedLockType const sl(mtx_);
@@ -182,7 +182,7 @@ TransactionAcquire::takeNodes(
ConsensusTransSetSF sf(app_, app_.getTempNodeCache());
for (auto& d : data)
for (auto const& d : data)
{
if (d.first.isRoot())
{

View File

@@ -20,8 +20,8 @@ public:
SHAMapAddNode
takeNodes(
std::vector<std::pair<SHAMapNodeID, intr_ptr::SharedPtr<SHAMapTreeNode>>> const& data,
std::shared_ptr<Peer> const& peer);
std::vector<std::pair<SHAMapNodeID, Slice>> const& data,
std::shared_ptr<Peer> const&);
void
init(int startPeers);

View File

@@ -9,10 +9,10 @@ BasicApp::BasicApp(std::size_t numberOfThreads)
work_.emplace(boost::asio::make_work_guard(io_context_));
threads_.reserve(numberOfThreads);
while ((numberOfThreads--) != 0u)
for (std::size_t i = 0; i < numberOfThreads; ++i)
{
threads_.emplace_back([this, numberOfThreads]() {
beast::setCurrentThreadName("io svc #" + std::to_string(numberOfThreads));
threads_.emplace_back([this, i]() {
beast::setCurrentThreadName("io svc #" + std::to_string(i));
this->io_context_.run();
});
}

View File

@@ -17,7 +17,6 @@ enum class ProtocolFeature {
ValidatorListPropagation,
ValidatorList2Propagation,
LedgerReplay,
LedgerNodeDepth,
};
/** Represents a peer connection in the overlay. */

View File

@@ -30,7 +30,6 @@
#include <mutex>
#include <numeric>
#include <sstream>
#include <tuple>
using namespace std::chrono_literals;
@@ -499,8 +498,6 @@ PeerImp::supportsFeature(ProtocolFeature f) const
return protocol_ >= make_protocol(2, 1);
case ProtocolFeature::ValidatorList2Propagation:
return protocol_ >= make_protocol(2, 2);
case ProtocolFeature::LedgerNodeDepth:
return protocol_ >= make_protocol(2, 3);
case ProtocolFeature::LedgerReplay:
return ledgerReplayEnabled_;
}
@@ -3391,19 +3388,13 @@ PeerImp::processLedgerRequest(std::shared_ptr<protocol::TMGetLedger> const& m)
std::uint32_t const defaultDepth = isHighLatency() ? 2 : 1;
auto const queryDepth{m->has_querydepth() ? m->querydepth() : defaultDepth};
std::vector<std::tuple<SHAMapNodeID, Blob, bool>> data;
auto const useLedgerNodeDepth = supportsFeature(ProtocolFeature::LedgerNodeDepth);
std::vector<std::pair<SHAMapNodeID, Blob>> data;
for (int i = 0;
i < m->nodeids_size() && ledgerData.nodes_size() < Tuning::softMaxReplyNodes;
++i)
{
auto const shaMapNodeId{deserializeSHAMapNodeID(m->nodeids(i))};
if (!shaMapNodeId.has_value())
{
JLOG(p_journal_.error()) << "processLedgerRequest: Invalid SHAMap node ID";
return;
}
data.clear();
data.reserve(Tuning::softMaxReplyNodes);
@@ -3419,28 +3410,9 @@ PeerImp::processLedgerRequest(std::shared_ptr<protocol::TMGetLedger> const& m)
{
if (ledgerData.nodes_size() >= Tuning::hardMaxReplyNodes)
break;
protocol::TMLedgerNode* node{ledgerData.add_nodes()};
auto const& nodeData = std::get<1>(d);
node->set_nodedata(nodeData.data(), nodeData.size());
// When the LedgerNodeDepth protocol feature is not supported by the peer,
// we always set the `nodeid` field. However, when it is supported then we
// set the `id` field for inner nodes and the `depth` field for leaf nodes.
auto const& nodeID = std::get<0>(d);
if (!useLedgerNodeDepth)
{
node->set_nodeid(nodeID.getRawString());
}
else if (std::get<2>(d))
{
node->set_depth(nodeID.getDepth());
}
else
{
node->set_id(nodeID.getRawString());
}
node->set_nodeid(d.first.getRawString());
node->set_nodedata(d.second.data(), d.second.size());
}
}
else

View File

@@ -20,7 +20,6 @@ namespace xrpl {
constexpr ProtocolVersion const supportedProtocolList[]{
{2, 1},
{2, 2},
{2, 3},
};
// This ugly construct ensures that supportedProtocolList is sorted in strictly

View File

@@ -9,6 +9,9 @@
namespace xrpl {
namespace PeerFinder {
/** Direction of a slot count adjustment. */
enum class CountAdjustment : int { Decrement = -1, Increment = 1 };
/** Manages the count of available connections for the various slots. */
class Counts
{
@@ -17,14 +20,14 @@ public:
void
add(Slot const& s)
{
adjust(s, 1);
adjust(s, CountAdjustment::Increment);
}
/** Removes the slot state and properties from the slot counts. */
void
remove(Slot const& s)
{
adjust(s, -1);
adjust(s, CountAdjustment::Decrement);
}
/** Returns `true` if the slot can become active. */
@@ -211,21 +214,40 @@ public:
//--------------------------------------------------------------------------
private:
/** Increments or decrements a counter based on the adjustment direction. */
template <typename T>
static void
adjustCounter(T& counter, CountAdjustment dir)
{
switch (dir)
{
case CountAdjustment::Increment:
++counter;
break;
case CountAdjustment::Decrement:
--counter;
break;
}
}
// Adjusts counts based on the specified slot, in the direction indicated.
// Using ++/-- instead of += on std::size_t counters avoids UBSan
// unsigned-integer-overflow from implicit conversion of -1 to SIZE_MAX.
// A decrement on a zero counter is a real bug that UBSan should catch.
void
adjust(Slot const& s, int const n)
adjust(Slot const& s, CountAdjustment const dir)
{
if (s.fixed())
m_fixed += n;
adjustCounter(m_fixed, dir);
if (s.reserved())
m_reserved += n;
adjustCounter(m_reserved, dir);
switch (s.state())
{
case Slot::accept:
XRPL_ASSERT(s.inbound(), "xrpl::PeerFinder::Counts::adjust : input is inbound");
m_acceptCount += n;
adjustCounter(m_acceptCount, dir);
break;
case Slot::connect:
@@ -234,24 +256,28 @@ private:
!s.inbound(),
"xrpl::PeerFinder::Counts::adjust : input is not "
"inbound");
m_attempts += n;
adjustCounter(m_attempts, dir);
break;
case Slot::active:
if (s.fixed())
m_fixed_active += n;
adjustCounter(m_fixed_active, dir);
if (!s.fixed() && !s.reserved())
{
if (s.inbound())
m_in_active += n;
{
adjustCounter(m_in_active, dir);
}
else
m_out_active += n;
{
adjustCounter(m_out_active, dir);
}
}
m_active += n;
adjustCounter(m_active, dir);
break;
case Slot::closing:
m_closingCount += n;
adjustCounter(m_closingCount, dir);
break;
// LCOV_EXCL_START