mirror of
https://github.com/XRPLF/rippled.git
synced 2026-04-04 02:52:38 +00:00
Compare commits
165 Commits
bthomee/no
...
ximinez/on
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a34d1e5537 | ||
|
|
78a122943d | ||
|
|
06135e7203 | ||
|
|
f20425fa4f | ||
|
|
3e94546acc | ||
|
|
3b088ed0dc | ||
|
|
26182ed52e | ||
|
|
de2a3e10f5 | ||
|
|
e17f8554fc | ||
|
|
386a7192ba | ||
|
|
12cc6e424d | ||
|
|
c9deecf1b7 | ||
|
|
ddd1b49f38 | ||
|
|
c1b2a24005 | ||
|
|
86d88eca31 | ||
|
|
ef09eaea00 | ||
|
|
c504cfb291 | ||
|
|
0c217dfa2b | ||
|
|
b0198d2566 | ||
|
|
7eee8ca802 | ||
|
|
2a079a0154 | ||
|
|
40989c1178 | ||
|
|
addc831eb3 | ||
|
|
b4efc6d116 | ||
|
|
125d075d6e | ||
|
|
370a775479 | ||
|
|
1a2ee706eb | ||
|
|
2a981357ba | ||
|
|
1ae475e724 | ||
|
|
a3e9401fbc | ||
|
|
9091469f9e | ||
|
|
17fa54f1f9 | ||
|
|
8fb5347c2d | ||
|
|
6739bf998f | ||
|
|
6eea38ba67 | ||
|
|
e9cf88b359 | ||
|
|
645b203476 | ||
|
|
be2aff1f4c | ||
|
|
56ed237e82 | ||
|
|
fd7b0fd135 | ||
|
|
e700994891 | ||
|
|
c76f7029ac | ||
|
|
d535c5fb2a | ||
|
|
54f860463e | ||
|
|
950434b8ff | ||
|
|
ee365e876d | ||
|
|
c6c59834b9 | ||
|
|
63b47914b8 | ||
|
|
9e02e5be2e | ||
|
|
093cd70fa1 | ||
|
|
376d65a483 | ||
|
|
a0d9a2458e | ||
|
|
456f639cf7 | ||
|
|
2c559ec2f3 | ||
|
|
619c81f463 | ||
|
|
f1490df960 | ||
|
|
7bdf74de98 | ||
|
|
1743d6fb98 | ||
|
|
ca7a5bb926 | ||
|
|
ce8b1a3f1e | ||
|
|
486fa75a10 | ||
|
|
f8d68cd3d3 | ||
|
|
ef7a3f5606 | ||
|
|
4f84ed7490 | ||
|
|
d534103131 | ||
|
|
82dff3c2ce | ||
|
|
30d73eb5ba | ||
|
|
1b2754bac2 | ||
|
|
cf80cafc75 | ||
|
|
b8897d51de | ||
|
|
3ff25eeb65 | ||
|
|
2bbfc4e786 | ||
|
|
2b1eb052e6 | ||
|
|
360e214e54 | ||
|
|
2618afed94 | ||
|
|
698ba2c788 | ||
|
|
b614e99588 | ||
|
|
fe8e4af2fa | ||
|
|
0a897f1528 | ||
|
|
cf8a3f5779 | ||
|
|
db39a39868 | ||
|
|
37a03d28c2 | ||
|
|
19d275425a | ||
|
|
88e9045602 | ||
|
|
5adbc536b6 | ||
|
|
e27af94ba9 | ||
|
|
43fe1e7e9c | ||
|
|
f456a858c8 | ||
|
|
084c3aa88e | ||
|
|
34f9b63921 | ||
|
|
bd3de79817 | ||
|
|
304eee2259 | ||
|
|
9e729b7f59 | ||
|
|
dd141468c4 | ||
|
|
933147c21f | ||
|
|
9201a4f591 | ||
|
|
5adb1e9b8b | ||
|
|
4df84d7988 | ||
|
|
cd87c0968b | ||
|
|
8a8e7c90bf | ||
|
|
e806069065 | ||
|
|
ce948cbec0 | ||
|
|
6ed34b3294 | ||
|
|
7161a235ca | ||
|
|
71463810de | ||
|
|
e997219a85 | ||
|
|
895cc13fa6 | ||
|
|
8d3c3ca29a | ||
|
|
9829553807 | ||
|
|
e551f9731a | ||
|
|
fd827bf58b | ||
|
|
5a3baba34d | ||
|
|
c78f5b160f | ||
|
|
485f78761a | ||
|
|
23cd2f7b21 | ||
|
|
5753266c43 | ||
|
|
4722d2607d | ||
|
|
85b5b4f855 | ||
|
|
a16f492f0f | ||
|
|
3633dc632c | ||
|
|
b3b30c3a86 | ||
|
|
c78a7684f4 | ||
|
|
cf83d92630 | ||
|
|
a56b1274d8 | ||
|
|
ae4bdd0492 | ||
|
|
e90102dd3b | ||
|
|
71f0e8db3d | ||
|
|
638929373a | ||
|
|
8440654377 | ||
|
|
9fa66c4741 | ||
|
|
38a9235145 | ||
|
|
c7a3cc9108 | ||
|
|
248337908d | ||
|
|
3d003619fd | ||
|
|
f163dca12c | ||
|
|
6e0ce458e5 | ||
|
|
5fae8480f1 | ||
|
|
e6587d374a | ||
|
|
376cc404e0 | ||
|
|
9898ca638f | ||
|
|
34b46d8f7c | ||
|
|
fe7d0798a7 | ||
|
|
0cecc09d71 | ||
|
|
e091d55561 | ||
|
|
69cf18158b | ||
|
|
6513c53817 | ||
|
|
e13baa58a5 | ||
|
|
951056fe9b | ||
|
|
67700ea6bd | ||
|
|
e5442cf3f1 | ||
|
|
da68076f04 | ||
|
|
b24116a118 | ||
|
|
f67398c6bf | ||
|
|
43d3eb1a24 | ||
|
|
0993315ed5 | ||
|
|
0bc383ada9 | ||
|
|
1841ceca43 | ||
|
|
2714cebabd | ||
|
|
e184db4ce2 | ||
|
|
ac6dc6943c | ||
|
|
ddd53806df | ||
|
|
e629a1f70e | ||
|
|
68076d969c | ||
|
|
d3009d3e1c | ||
|
|
54f7f3c894 |
@@ -44,7 +44,6 @@ libxrpl.tx > xrpl.server
|
||||
libxrpl.tx > xrpl.tx
|
||||
test.app > test.jtx
|
||||
test.app > test.rpc
|
||||
test.app > test.shamap
|
||||
test.app > test.toplevel
|
||||
test.app > test.unit_test
|
||||
test.app > xrpl.basics
|
||||
@@ -60,7 +59,6 @@ test.app > xrpl.protocol
|
||||
test.app > xrpl.rdb
|
||||
test.app > xrpl.resource
|
||||
test.app > xrpl.server
|
||||
test.app > xrpl.shamap
|
||||
test.app > xrpl.tx
|
||||
test.basics > test.jtx
|
||||
test.basics > test.unit_test
|
||||
|
||||
@@ -1033,10 +1033,11 @@
|
||||
# The online delete process checks periodically
|
||||
# that xrpld is still in sync with the network,
|
||||
# and that the validated ledger is less than
|
||||
# 'age_threshold_seconds' old. If not, then continue
|
||||
# 'age_threshold_seconds' old, and that all
|
||||
# recent ledgers are available. If not, then continue
|
||||
# sleeping for this number of seconds and
|
||||
# checking until healthy.
|
||||
# Default is 5.
|
||||
# Default is 1.
|
||||
#
|
||||
# Notes:
|
||||
# The 'node_db' entry configures the primary, persistent storage.
|
||||
|
||||
@@ -11,7 +11,6 @@
|
||||
#include <limits>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <type_traits>
|
||||
#include <vector>
|
||||
|
||||
@@ -232,11 +231,4 @@ makeSlice(std::basic_string<char, Traits, Alloc> const& s)
|
||||
return Slice(s.data(), s.size());
|
||||
}
|
||||
|
||||
template <class Traits>
|
||||
Slice
|
||||
makeSlice(std::basic_string_view<char, Traits> const& s)
|
||||
{
|
||||
return Slice(s.data(), s.size());
|
||||
}
|
||||
|
||||
} // namespace xrpl
|
||||
|
||||
@@ -244,15 +244,7 @@ message TMGetObjectByHash {
|
||||
|
||||
message TMLedgerNode {
|
||||
required bytes nodedata = 1;
|
||||
|
||||
// Used when protocol version <2.3. Not set for ledger base data.
|
||||
optional bytes nodeid = 2;
|
||||
|
||||
// Used when protocol version >=2.3. Neither value is set for ledger base data.
|
||||
oneof reference {
|
||||
bytes id = 3; // Set for inner nodes.
|
||||
uint32 depth = 4; // Set for leaf nodes.
|
||||
}
|
||||
optional bytes nodeid = 2; // missing for ledger base data
|
||||
}
|
||||
|
||||
enum TMLedgerInfoType {
|
||||
|
||||
@@ -16,7 +16,6 @@
|
||||
|
||||
#include <set>
|
||||
#include <stack>
|
||||
#include <tuple>
|
||||
#include <vector>
|
||||
|
||||
namespace xrpl {
|
||||
@@ -254,7 +253,7 @@ public:
|
||||
bool
|
||||
getNodeFat(
|
||||
SHAMapNodeID const& wanted,
|
||||
std::vector<std::tuple<SHAMapNodeID, Blob, bool>>& data,
|
||||
std::vector<std::pair<SHAMapNodeID, Blob>>& data,
|
||||
bool fatLeaves,
|
||||
std::uint32_t depth) const;
|
||||
|
||||
@@ -281,45 +280,10 @@ public:
|
||||
void
|
||||
serializeRoot(Serializer& s) const;
|
||||
|
||||
/** Add a root node to the SHAMap during synchronization.
|
||||
*
|
||||
* This function is used when receiving the root node of a SHAMap from a peer during ledger
|
||||
* synchronization. The node must already have been deserialized.
|
||||
*
|
||||
* @param hash The expected hash of the root node.
|
||||
* @param rootNode A deserialized root node to add.
|
||||
* @param filter Optional sync filter to track received nodes.
|
||||
* @return Status indicating whether the node was useful, duplicate, or invalid.
|
||||
*
|
||||
* @note This function expects the rootNode to be a valid, deserialized SHAMapTreeNode. The
|
||||
* caller is responsible for deserialization and basic validation before calling this
|
||||
* function.
|
||||
*/
|
||||
SHAMapAddNode
|
||||
addRootNode(
|
||||
SHAMapHash const& hash,
|
||||
intr_ptr::SharedPtr<SHAMapTreeNode> rootNode,
|
||||
SHAMapSyncFilter const* filter);
|
||||
|
||||
/** Add a known node at a specific position in the SHAMap during synchronization.
|
||||
*
|
||||
* This function is used when receiving nodes from peers during ledger synchronization. The node
|
||||
* is inserted at the position specified by nodeID. The node must already have been
|
||||
* deserialized.
|
||||
*
|
||||
* @param nodeID The position in the tree where this node belongs.
|
||||
* @param treeNode A deserialized tree node to add.
|
||||
* @param filter Optional sync filter to track received nodes.
|
||||
* @return Status indicating whether the node was useful, duplicate, or invalid.
|
||||
*
|
||||
* @note This function expects that the caller has already validated that the nodeID is
|
||||
* consistent with the node's content.
|
||||
*/
|
||||
addRootNode(SHAMapHash const& hash, Slice const& rootNode, SHAMapSyncFilter* filter);
|
||||
SHAMapAddNode
|
||||
addKnownNode(
|
||||
SHAMapNodeID const& nodeID,
|
||||
intr_ptr::SharedPtr<SHAMapTreeNode> treeNode,
|
||||
SHAMapSyncFilter const* filter);
|
||||
addKnownNode(SHAMapNodeID const& nodeID, Slice const& rawNode, SHAMapSyncFilter* filter);
|
||||
|
||||
// status functions
|
||||
void
|
||||
@@ -380,11 +344,11 @@ private:
|
||||
intr_ptr::SharedPtr<SHAMapTreeNode>
|
||||
fetchNodeNT(SHAMapHash const& hash) const;
|
||||
intr_ptr::SharedPtr<SHAMapTreeNode>
|
||||
fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter const* filter) const;
|
||||
fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter* filter) const;
|
||||
intr_ptr::SharedPtr<SHAMapTreeNode>
|
||||
fetchNode(SHAMapHash const& hash) const;
|
||||
intr_ptr::SharedPtr<SHAMapTreeNode>
|
||||
checkFilter(SHAMapHash const& hash, SHAMapSyncFilter const* filter) const;
|
||||
checkFilter(SHAMapHash const& hash, SHAMapSyncFilter* filter) const;
|
||||
|
||||
/** Update hashes up to the root */
|
||||
void
|
||||
@@ -456,7 +420,7 @@ private:
|
||||
descendAsync(
|
||||
SHAMapInnerNode* parent,
|
||||
int branch,
|
||||
SHAMapSyncFilter const* filter,
|
||||
SHAMapSyncFilter* filter,
|
||||
bool& pending,
|
||||
descendCallback&&) const;
|
||||
|
||||
@@ -465,7 +429,7 @@ private:
|
||||
SHAMapInnerNode* parent,
|
||||
SHAMapNodeID const& parentID,
|
||||
int branch,
|
||||
SHAMapSyncFilter const* filter) const;
|
||||
SHAMapSyncFilter* filter) const;
|
||||
|
||||
// Non-storing
|
||||
// Does not hook the returned node to its parent
|
||||
|
||||
@@ -179,7 +179,7 @@ SHAMap::finishFetch(SHAMapHash const& hash, std::shared_ptr<NodeObject> const& o
|
||||
|
||||
// See if a sync filter has a node
|
||||
intr_ptr::SharedPtr<SHAMapTreeNode>
|
||||
SHAMap::checkFilter(SHAMapHash const& hash, SHAMapSyncFilter const* filter) const
|
||||
SHAMap::checkFilter(SHAMapHash const& hash, SHAMapSyncFilter* filter) const
|
||||
{
|
||||
if (auto nodeData = filter->getNode(hash))
|
||||
{
|
||||
@@ -205,7 +205,7 @@ SHAMap::checkFilter(SHAMapHash const& hash, SHAMapSyncFilter const* filter) cons
|
||||
// Get a node without throwing
|
||||
// Used on maps where missing nodes are expected
|
||||
intr_ptr::SharedPtr<SHAMapTreeNode>
|
||||
SHAMap::fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter const* filter) const
|
||||
SHAMap::fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter* filter) const
|
||||
{
|
||||
auto node = cacheLookup(hash);
|
||||
if (node)
|
||||
@@ -318,7 +318,7 @@ SHAMap::descend(
|
||||
SHAMapInnerNode* parent,
|
||||
SHAMapNodeID const& parentID,
|
||||
int branch,
|
||||
SHAMapSyncFilter const* filter) const
|
||||
SHAMapSyncFilter* filter) const
|
||||
{
|
||||
XRPL_ASSERT(parent->isInner(), "xrpl::SHAMap::descend : valid parent input");
|
||||
XRPL_ASSERT(
|
||||
@@ -347,7 +347,7 @@ SHAMapTreeNode*
|
||||
SHAMap::descendAsync(
|
||||
SHAMapInnerNode* parent,
|
||||
int branch,
|
||||
SHAMapSyncFilter const* filter,
|
||||
SHAMapSyncFilter* filter,
|
||||
bool& pending,
|
||||
descendCallback&& callback) const
|
||||
{
|
||||
|
||||
@@ -122,9 +122,7 @@ selectBranch(SHAMapNodeID const& id, uint256 const& hash)
|
||||
SHAMapNodeID
|
||||
SHAMapNodeID::createID(int depth, uint256 const& key)
|
||||
{
|
||||
XRPL_ASSERT(
|
||||
depth >= 0 && depth <= SHAMap::leafDepth,
|
||||
"xrpl::SHAMapNodeID::createID : valid branch input");
|
||||
XRPL_ASSERT((depth >= 0) && (depth < 65), "xrpl::SHAMapNodeID::createID : valid branch input");
|
||||
return SHAMapNodeID(depth, key & depthMask(depth));
|
||||
}
|
||||
|
||||
|
||||
@@ -392,7 +392,7 @@ SHAMap::getMissingNodes(int max, SHAMapSyncFilter* filter)
|
||||
bool
|
||||
SHAMap::getNodeFat(
|
||||
SHAMapNodeID const& wanted,
|
||||
std::vector<std::tuple<SHAMapNodeID, Blob, bool>>& data,
|
||||
std::vector<std::pair<SHAMapNodeID, Blob>>& data,
|
||||
bool fatLeaves,
|
||||
std::uint32_t depth) const
|
||||
{
|
||||
@@ -438,7 +438,7 @@ SHAMap::getNodeFat(
|
||||
// Add this node to the reply
|
||||
s.erase();
|
||||
node->serializeForWire(s);
|
||||
data.emplace_back(std::make_tuple(nodeID, s.getData(), node->isLeaf()));
|
||||
data.emplace_back(std::make_pair(nodeID, s.getData()));
|
||||
|
||||
if (node->isInner())
|
||||
{
|
||||
@@ -468,8 +468,7 @@ SHAMap::getNodeFat(
|
||||
// Just include this node
|
||||
s.erase();
|
||||
childNode->serializeForWire(s);
|
||||
data.emplace_back(
|
||||
std::make_tuple(childID, s.getData(), childNode->isLeaf()));
|
||||
data.emplace_back(std::make_pair(childID, s.getData()));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -487,18 +486,8 @@ SHAMap::serializeRoot(Serializer& s) const
|
||||
}
|
||||
|
||||
SHAMapAddNode
|
||||
SHAMap::addRootNode(
|
||||
SHAMapHash const& hash,
|
||||
intr_ptr::SharedPtr<SHAMapTreeNode> rootNode,
|
||||
SHAMapSyncFilter const* filter)
|
||||
SHAMap::addRootNode(SHAMapHash const& hash, Slice const& rootNode, SHAMapSyncFilter* filter)
|
||||
{
|
||||
XRPL_ASSERT(rootNode, "xrpl::SHAMap::addRootNode : non-null root node");
|
||||
if (!rootNode)
|
||||
{
|
||||
JLOG(journal_.error()) << "Null node received";
|
||||
return SHAMapAddNode::invalid();
|
||||
}
|
||||
|
||||
// we already have a root_ node
|
||||
if (root_->getHash().isNonZero())
|
||||
{
|
||||
@@ -508,16 +497,14 @@ SHAMap::addRootNode(
|
||||
}
|
||||
|
||||
XRPL_ASSERT(cowid_ >= 1, "xrpl::SHAMap::addRootNode : valid cowid");
|
||||
if (rootNode->getHash() != hash)
|
||||
{
|
||||
JLOG(journal_.warn()) << "Corrupt node received";
|
||||
auto node = SHAMapTreeNode::makeFromWire(rootNode);
|
||||
if (!node || node->getHash() != hash)
|
||||
return SHAMapAddNode::invalid();
|
||||
}
|
||||
|
||||
if (backed_)
|
||||
canonicalize(hash, rootNode);
|
||||
canonicalize(hash, node);
|
||||
|
||||
root_ = std::move(rootNode);
|
||||
root_ = node;
|
||||
|
||||
if (root_->isLeaf())
|
||||
clearSynching();
|
||||
@@ -534,23 +521,9 @@ SHAMap::addRootNode(
|
||||
}
|
||||
|
||||
SHAMapAddNode
|
||||
SHAMap::addKnownNode(
|
||||
SHAMapNodeID const& nodeID,
|
||||
intr_ptr::SharedPtr<SHAMapTreeNode> treeNode,
|
||||
SHAMapSyncFilter const* filter)
|
||||
SHAMap::addKnownNode(SHAMapNodeID const& node, Slice const& rawNode, SHAMapSyncFilter* filter)
|
||||
{
|
||||
XRPL_ASSERT(!nodeID.isRoot(), "xrpl::SHAMap::addKnownNode : valid node input");
|
||||
if (nodeID.isRoot())
|
||||
{
|
||||
JLOG(journal_.error()) << "Root node received";
|
||||
return SHAMapAddNode::invalid();
|
||||
}
|
||||
XRPL_ASSERT(treeNode, "xrpl::SHAMap::addKnownNode : non-null tree node");
|
||||
if (!treeNode)
|
||||
{
|
||||
JLOG(journal_.error()) << "Null node received";
|
||||
return SHAMapAddNode::invalid();
|
||||
}
|
||||
XRPL_ASSERT(!node.isRoot(), "xrpl::SHAMap::addKnownNode : valid node input");
|
||||
|
||||
if (!isSynching())
|
||||
{
|
||||
@@ -564,14 +537,14 @@ SHAMap::addKnownNode(
|
||||
|
||||
while (currNode->isInner() &&
|
||||
!safe_downcast<SHAMapInnerNode*>(currNode)->isFullBelow(generation) &&
|
||||
(currNodeID.getDepth() < nodeID.getDepth()))
|
||||
(currNodeID.getDepth() < node.getDepth()))
|
||||
{
|
||||
int const branch = selectBranch(currNodeID, nodeID.getNodeID());
|
||||
int const branch = selectBranch(currNodeID, node.getNodeID());
|
||||
XRPL_ASSERT(branch >= 0, "xrpl::SHAMap::addKnownNode : valid branch");
|
||||
auto inner = safe_downcast<SHAMapInnerNode*>(currNode);
|
||||
if (inner->isEmptyBranch(branch))
|
||||
{
|
||||
JLOG(journal_.warn()) << "Add known node for empty branch" << nodeID;
|
||||
JLOG(journal_.warn()) << "Add known node for empty branch" << node;
|
||||
return SHAMapAddNode::invalid();
|
||||
}
|
||||
|
||||
@@ -587,44 +560,67 @@ SHAMap::addKnownNode(
|
||||
if (currNode != nullptr)
|
||||
continue;
|
||||
|
||||
if (childHash != treeNode->getHash())
|
||||
auto newNode = SHAMapTreeNode::makeFromWire(rawNode);
|
||||
|
||||
if (!newNode || childHash != newNode->getHash())
|
||||
{
|
||||
JLOG(journal_.warn()) << "Corrupt node received";
|
||||
return SHAMapAddNode::invalid();
|
||||
}
|
||||
|
||||
// In rare cases, a node can still be corrupt even after hash
|
||||
// validation. For leaf nodes, we perform an additional check to
|
||||
// ensure the node's position in the tree is consistent with its
|
||||
// content to prevent inconsistencies that could
|
||||
// propagate further down the line.
|
||||
if (newNode->isLeaf())
|
||||
{
|
||||
auto const& actualKey =
|
||||
safe_downcast<SHAMapLeafNode const*>(newNode.get())->peekItem()->key();
|
||||
|
||||
// Validate that this leaf belongs at the target position
|
||||
auto const expectedNodeID = SHAMapNodeID::createID(node.getDepth(), actualKey);
|
||||
if (expectedNodeID.getNodeID() != node.getNodeID())
|
||||
{
|
||||
JLOG(journal_.debug())
|
||||
<< "Leaf node position mismatch: "
|
||||
<< "expected=" << expectedNodeID.getNodeID() << ", actual=" << node.getNodeID();
|
||||
return SHAMapAddNode::invalid();
|
||||
}
|
||||
}
|
||||
|
||||
// Inner nodes must be at a level strictly less than 64
|
||||
// but leaf nodes (while notionally at level 64) can be
|
||||
// at any depth up to and including 64:
|
||||
if ((currNodeID.getDepth() > leafDepth) ||
|
||||
(treeNode->isInner() && currNodeID.getDepth() == leafDepth))
|
||||
(newNode->isInner() && currNodeID.getDepth() == leafDepth))
|
||||
{
|
||||
// Map is provably invalid
|
||||
state_ = SHAMapState::Invalid;
|
||||
return SHAMapAddNode::useful();
|
||||
}
|
||||
|
||||
if (currNodeID != nodeID)
|
||||
if (currNodeID != node)
|
||||
{
|
||||
// Either this node is broken or we didn't request it (yet)
|
||||
JLOG(journal_.warn()) << "unable to hook node " << nodeID;
|
||||
JLOG(journal_.warn()) << "unable to hook node " << node;
|
||||
JLOG(journal_.info()) << " stuck at " << currNodeID;
|
||||
JLOG(journal_.info()) << "got depth=" << nodeID.getDepth()
|
||||
JLOG(journal_.info()) << "got depth=" << node.getDepth()
|
||||
<< ", walked to= " << currNodeID.getDepth();
|
||||
return SHAMapAddNode::useful();
|
||||
}
|
||||
|
||||
if (backed_)
|
||||
canonicalize(childHash, treeNode);
|
||||
canonicalize(childHash, newNode);
|
||||
|
||||
treeNode = prevNode->canonicalizeChild(branch, std::move(treeNode));
|
||||
newNode = prevNode->canonicalizeChild(branch, std::move(newNode));
|
||||
|
||||
if (filter != nullptr)
|
||||
{
|
||||
Serializer s;
|
||||
treeNode->serializeWithPrefix(s);
|
||||
newNode->serializeWithPrefix(s);
|
||||
filter->gotNode(
|
||||
false, childHash, ledgerSeq_, std::move(s.modData()), treeNode->getType());
|
||||
false, childHash, ledgerSeq_, std::move(s.modData()), newNode->getType());
|
||||
}
|
||||
|
||||
return SHAMapAddNode::useful();
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
#include <test/jtx/Env.h>
|
||||
|
||||
#include <xrpld/app/ledger/LedgerMaster.h>
|
||||
#include <xrpld/app/misc/SHAMapStore.h>
|
||||
|
||||
namespace xrpl {
|
||||
namespace test {
|
||||
@@ -97,6 +98,73 @@ class LedgerMaster_test : public beast::unit_test::suite
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
testCompleteLedgerRange(FeatureBitset features)
|
||||
{
|
||||
// Note that this test is intentionally very similar to
|
||||
// SHAMapStore_test::testLedgerGaps, but has a different
|
||||
// focus.
|
||||
|
||||
testcase("Complete Ledger operations");
|
||||
|
||||
using namespace test::jtx;
|
||||
|
||||
auto const deleteInterval = 8;
|
||||
|
||||
Env env{*this, envconfig([](auto cfg) {
|
||||
return online_delete(std::move(cfg), deleteInterval);
|
||||
})};
|
||||
|
||||
auto const alice = Account("alice");
|
||||
env.fund(XRP(1000), alice);
|
||||
env.close();
|
||||
|
||||
auto& lm = env.app().getLedgerMaster();
|
||||
LedgerIndex minSeq = 2;
|
||||
LedgerIndex maxSeq = env.closed()->header().seq;
|
||||
auto& store = env.app().getSHAMapStore();
|
||||
store.rendezvous();
|
||||
LedgerIndex lastRotated = store.getLastRotated();
|
||||
BEAST_EXPECTS(maxSeq == 3, to_string(maxSeq));
|
||||
BEAST_EXPECTS(lm.getCompleteLedgers() == "2-3", lm.getCompleteLedgers());
|
||||
BEAST_EXPECTS(lastRotated == 3, to_string(lastRotated));
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
|
||||
|
||||
// Close enough ledgers to rotate a few times
|
||||
for (int i = 0; i < 24; ++i)
|
||||
{
|
||||
for (int t = 0; t < 3; ++t)
|
||||
{
|
||||
env(noop(alice));
|
||||
}
|
||||
env.close();
|
||||
store.rendezvous();
|
||||
|
||||
++maxSeq;
|
||||
|
||||
if (maxSeq == lastRotated + deleteInterval)
|
||||
{
|
||||
minSeq = lastRotated;
|
||||
lastRotated = maxSeq;
|
||||
}
|
||||
BEAST_EXPECTS(
|
||||
env.closed()->header().seq == maxSeq, to_string(env.closed()->header().seq));
|
||||
BEAST_EXPECTS(store.getLastRotated() == lastRotated, to_string(store.getLastRotated()));
|
||||
std::stringstream expectedRange;
|
||||
expectedRange << minSeq << "-" << maxSeq;
|
||||
BEAST_EXPECTS(lm.getCompleteLedgers() == expectedRange.str(), lm.getCompleteLedgers());
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
|
||||
}
|
||||
}
|
||||
|
||||
public:
|
||||
void
|
||||
run() override
|
||||
@@ -110,6 +178,7 @@ public:
|
||||
testWithFeats(FeatureBitset features)
|
||||
{
|
||||
testTxnIdFromIndex(features);
|
||||
testCompleteLedgerRange(features);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -1,356 +0,0 @@
|
||||
#include <test/shamap/common.h>
|
||||
|
||||
#include <xrpld/app/ledger/detail/LedgerNodeHelpers.h>
|
||||
|
||||
#include <xrpl/beast/unit_test.h>
|
||||
#include <xrpl/protocol/messages.h>
|
||||
#include <xrpl/shamap/SHAMap.h>
|
||||
#include <xrpl/shamap/SHAMapAccountStateLeafNode.h>
|
||||
#include <xrpl/shamap/SHAMapInnerNode.h>
|
||||
#include <xrpl/shamap/SHAMapItem.h>
|
||||
#include <xrpl/shamap/SHAMapTreeNode.h>
|
||||
|
||||
#include <bit>
|
||||
|
||||
namespace xrpl {
|
||||
namespace tests {
|
||||
|
||||
class LedgerNodeHelpers_test : public beast::unit_test::suite
|
||||
{
|
||||
// Helper function to create a simple SHAMapItem for testing.
|
||||
static boost::intrusive_ptr<SHAMapItem>
|
||||
makeTestItem(std::uint32_t seed)
|
||||
{
|
||||
Serializer s;
|
||||
s.add32(seed);
|
||||
s.add32(seed + 1);
|
||||
s.add32(seed + 2);
|
||||
return make_shamapitem(s.getSHA512Half(), s.slice());
|
||||
}
|
||||
|
||||
// Helper function to serialize a tree node to wire format.
|
||||
static std::string
|
||||
serializeNode(intr_ptr::SharedPtr<SHAMapTreeNode> const& node)
|
||||
{
|
||||
Serializer s;
|
||||
node->serializeForWire(s);
|
||||
auto const slice = s.slice();
|
||||
return std::string(std::bit_cast<char const*>(slice.data()), slice.size());
|
||||
}
|
||||
|
||||
void
|
||||
testValidateLedgerNode()
|
||||
{
|
||||
// In the tests below the validity of the content of the node data and ID fields is not
|
||||
// checked - only that the fields have values when expected. The content of the fields is
|
||||
// verified in the other tests in this file.
|
||||
testcase("validateLedgerNode");
|
||||
|
||||
// Invalid: missing all fields.
|
||||
{
|
||||
protocol::TMLedgerNode const node;
|
||||
BEAST_EXPECT(!validateLedgerNode(node));
|
||||
}
|
||||
|
||||
// Invalid: missing `nodedata` field.
|
||||
{
|
||||
protocol::TMLedgerNode node;
|
||||
node.set_nodeid("test_nodeid");
|
||||
BEAST_EXPECT(!validateLedgerNode(node));
|
||||
}
|
||||
|
||||
// Invalid: missing `nodedata` field.
|
||||
{
|
||||
protocol::TMLedgerNode node;
|
||||
node.set_id("test_nodeid");
|
||||
BEAST_EXPECT(!validateLedgerNode(node));
|
||||
}
|
||||
|
||||
// Invalid: missing `nodedata` field.
|
||||
{
|
||||
protocol::TMLedgerNode node;
|
||||
node.set_depth(1);
|
||||
BEAST_EXPECT(!validateLedgerNode(node));
|
||||
}
|
||||
|
||||
// Valid: legacy `nodeid` field.
|
||||
{
|
||||
protocol::TMLedgerNode node;
|
||||
node.set_nodedata("test_data");
|
||||
node.set_nodeid("test_nodeid");
|
||||
BEAST_EXPECT(validateLedgerNode(node));
|
||||
}
|
||||
|
||||
// Invalid: has both legacy `nodeid` and new `id` fields.
|
||||
{
|
||||
protocol::TMLedgerNode node;
|
||||
node.set_nodedata("test_data");
|
||||
node.set_nodeid("test_nodeid");
|
||||
node.set_id("test_nodeid");
|
||||
BEAST_EXPECT(!validateLedgerNode(node));
|
||||
}
|
||||
|
||||
// Invalid: has both legacy `nodeid` and new `depth` fields.
|
||||
{
|
||||
protocol::TMLedgerNode node;
|
||||
node.set_nodedata("test_data");
|
||||
node.set_nodeid("test_nodeid");
|
||||
node.set_depth(5);
|
||||
BEAST_EXPECT(!validateLedgerNode(node));
|
||||
}
|
||||
|
||||
// Valid: new `id` field.
|
||||
{
|
||||
protocol::TMLedgerNode node;
|
||||
node.set_nodedata("test_data");
|
||||
node.set_id("test_id");
|
||||
BEAST_EXPECT(validateLedgerNode(node));
|
||||
}
|
||||
|
||||
// Valid: new `depth` field.
|
||||
{
|
||||
protocol::TMLedgerNode node;
|
||||
node.set_nodedata("test_data");
|
||||
node.set_depth(5);
|
||||
BEAST_EXPECT(validateLedgerNode(node));
|
||||
}
|
||||
|
||||
// Valid: `depth` at minimum depth.
|
||||
{
|
||||
protocol::TMLedgerNode node;
|
||||
node.set_nodedata("test_data");
|
||||
node.set_depth(0);
|
||||
BEAST_EXPECT(validateLedgerNode(node));
|
||||
}
|
||||
|
||||
// Valid: `depth` at arbitrary depth between minimum and maximum.
|
||||
{
|
||||
protocol::TMLedgerNode node;
|
||||
node.set_nodedata("test_data");
|
||||
node.set_depth(10);
|
||||
BEAST_EXPECT(validateLedgerNode(node));
|
||||
}
|
||||
|
||||
// Valid: `depth` at maximum depth.
|
||||
{
|
||||
protocol::TMLedgerNode node;
|
||||
node.set_nodedata("test_data");
|
||||
node.set_depth(SHAMap::leafDepth);
|
||||
BEAST_EXPECT(validateLedgerNode(node));
|
||||
}
|
||||
|
||||
// Invalid: `depth` is greater than maximum depth.
|
||||
{
|
||||
protocol::TMLedgerNode node;
|
||||
node.set_nodedata("test_data");
|
||||
node.set_depth(SHAMap::leafDepth + 1);
|
||||
BEAST_EXPECT(!validateLedgerNode(node));
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
testGetTreeNode()
|
||||
{
|
||||
testcase("getTreeNode");
|
||||
|
||||
// Valid: inner node. It must have at least one child for `serializeNode` to work.
|
||||
{
|
||||
auto const innerNode = intr_ptr::make_shared<SHAMapInnerNode>(1);
|
||||
auto const childNode = intr_ptr::make_shared<SHAMapInnerNode>(1);
|
||||
innerNode->setChild(0, childNode);
|
||||
auto const innerData = serializeNode(innerNode);
|
||||
auto const result = getTreeNode(innerData);
|
||||
BEAST_EXPECT(result.has_value());
|
||||
BEAST_EXPECT((*result)->isInner());
|
||||
}
|
||||
|
||||
// Valid: leaf node.
|
||||
{
|
||||
auto const leafItem = makeTestItem(12345);
|
||||
auto const leafNode = intr_ptr::make_shared<SHAMapAccountStateLeafNode>(leafItem, 1);
|
||||
auto const leafData = serializeNode(leafNode);
|
||||
auto result = getTreeNode(leafData);
|
||||
BEAST_EXPECT(result.has_value());
|
||||
BEAST_EXPECT((*result)->isLeaf());
|
||||
}
|
||||
|
||||
// Invalid: empty data.
|
||||
{
|
||||
auto const result = getTreeNode("");
|
||||
BEAST_EXPECT(!result.has_value());
|
||||
}
|
||||
|
||||
// Invalid: garbage data.
|
||||
{
|
||||
auto const result = getTreeNode("invalid");
|
||||
BEAST_EXPECT(!result.has_value());
|
||||
}
|
||||
|
||||
// Invalid: truncated data.
|
||||
{
|
||||
auto const leafItem = makeTestItem(54321);
|
||||
auto const leafNode = intr_ptr::make_shared<SHAMapAccountStateLeafNode>(leafItem, 1);
|
||||
// Truncate the data to trigger an exception in SHAMapTreeNode::makeAccountState when
|
||||
// the data is used to deserialize the node.
|
||||
uint256 const tag;
|
||||
auto const leafData = serializeNode(leafNode).substr(0, tag.bytes - 1);
|
||||
auto const result = getTreeNode(leafData);
|
||||
BEAST_EXPECT(!result.has_value());
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
testGetSHAMapNodeID()
|
||||
{
|
||||
testcase("getSHAMapNodeID");
|
||||
|
||||
{
|
||||
// Tests using inner nodes at various depths.
|
||||
auto const innerNode = intr_ptr::make_shared<SHAMapInnerNode>(1);
|
||||
auto const childNode = intr_ptr::make_shared<SHAMapInnerNode>(1);
|
||||
innerNode->setChild(0, childNode);
|
||||
auto const innerData = serializeNode(innerNode);
|
||||
|
||||
// Valid: legacy `nodeid` field at arbitrary depth.
|
||||
{
|
||||
auto const innerDepth = 3;
|
||||
auto const innerID = SHAMapNodeID::createID(innerDepth, uint256{});
|
||||
|
||||
protocol::TMLedgerNode node;
|
||||
node.set_nodedata(innerData);
|
||||
node.set_nodeid(innerID.getRawString());
|
||||
auto const result = getSHAMapNodeID(node, innerNode);
|
||||
BEAST_EXPECT(result.has_value());
|
||||
BEAST_EXPECT(*result == innerID);
|
||||
}
|
||||
|
||||
// Valid: new `id` field at minimum depth.
|
||||
{
|
||||
auto const innerDepth = 0;
|
||||
auto const innerID = SHAMapNodeID::createID(innerDepth, uint256{});
|
||||
|
||||
protocol::TMLedgerNode node;
|
||||
node.set_nodedata(innerData);
|
||||
node.set_id(innerID.getRawString());
|
||||
auto const result = getSHAMapNodeID(node, innerNode);
|
||||
BEAST_EXPECT(result.has_value());
|
||||
BEAST_EXPECT(*result == innerID);
|
||||
}
|
||||
|
||||
// Invalid: new `depth` field should not be used for inner nodes.
|
||||
{
|
||||
protocol::TMLedgerNode node;
|
||||
node.set_nodedata(innerData);
|
||||
node.set_depth(10);
|
||||
auto const result = getSHAMapNodeID(node, innerNode);
|
||||
BEAST_EXPECT(!result.has_value());
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
// Tests using leaf nodes at various depths.
|
||||
auto const leafItem = makeTestItem(12345);
|
||||
auto const leafNode = intr_ptr::make_shared<SHAMapAccountStateLeafNode>(leafItem, 1);
|
||||
auto const leafData = serializeNode(leafNode);
|
||||
auto const leafKey = leafItem->key();
|
||||
|
||||
// Valid: legacy `nodeid` field at arbitrary depth.
|
||||
{
|
||||
auto const leafDepth = 5;
|
||||
auto const leafID = SHAMapNodeID::createID(leafDepth, leafKey);
|
||||
|
||||
protocol::TMLedgerNode ledgerNode;
|
||||
ledgerNode.set_nodedata(leafData);
|
||||
ledgerNode.set_nodeid(leafID.getRawString());
|
||||
auto result = getSHAMapNodeID(ledgerNode, leafNode);
|
||||
BEAST_EXPECT(result.has_value());
|
||||
BEAST_EXPECT(*result == leafID);
|
||||
}
|
||||
|
||||
// Invalid: new `id` field should not be used for leaf nodes.
|
||||
{
|
||||
auto const leafDepth = 5;
|
||||
auto const leafID = SHAMapNodeID::createID(leafDepth, leafKey);
|
||||
|
||||
protocol::TMLedgerNode ledgerNode;
|
||||
ledgerNode.set_nodedata(leafData);
|
||||
ledgerNode.set_id(leafID.getRawString());
|
||||
auto result = getSHAMapNodeID(ledgerNode, leafNode);
|
||||
BEAST_EXPECT(!result.has_value());
|
||||
}
|
||||
|
||||
// Valid: new `depth` field at minimum depth.
|
||||
{
|
||||
auto const leafDepth = 0;
|
||||
auto const leafID = SHAMapNodeID::createID(leafDepth, leafKey);
|
||||
|
||||
protocol::TMLedgerNode node;
|
||||
node.set_nodedata(leafData);
|
||||
node.set_depth(leafDepth);
|
||||
auto result = getSHAMapNodeID(node, leafNode);
|
||||
BEAST_EXPECT(result.has_value());
|
||||
BEAST_EXPECT(*result == leafID);
|
||||
}
|
||||
|
||||
// Valid: new `depth` field at arbitrary depth between minimum and maximum.
|
||||
{
|
||||
auto const leafDepth = 10;
|
||||
auto const leafID = SHAMapNodeID::createID(leafDepth, leafKey);
|
||||
|
||||
protocol::TMLedgerNode ledgerNode;
|
||||
ledgerNode.set_nodedata(leafData);
|
||||
ledgerNode.set_depth(leafDepth);
|
||||
auto result = getSHAMapNodeID(ledgerNode, leafNode);
|
||||
BEAST_EXPECT(result.has_value());
|
||||
BEAST_EXPECT(*result == leafID);
|
||||
}
|
||||
|
||||
// Valid: new `depth` field at maximum depth.
|
||||
// Note that we do not test a depth greater than the maximum depth, because the proto
|
||||
// message is assumed to have been validated by the time the getSHAMapNodeID function is
|
||||
// called.
|
||||
{
|
||||
auto const leafDepth = SHAMap::leafDepth;
|
||||
auto const leafID = SHAMapNodeID::createID(leafDepth, leafKey);
|
||||
|
||||
protocol::TMLedgerNode node;
|
||||
node.set_nodedata(leafData);
|
||||
node.set_depth(leafDepth);
|
||||
auto result = getSHAMapNodeID(node, leafNode);
|
||||
BEAST_EXPECT(result.has_value());
|
||||
BEAST_EXPECT(*result == leafID);
|
||||
}
|
||||
|
||||
// Invalid: legacy `nodeid` field where the node ID is inconsistent with the key.
|
||||
{
|
||||
auto const otherItem = makeTestItem(54321);
|
||||
auto const otherNode =
|
||||
intr_ptr::make_shared<SHAMapAccountStateLeafNode>(otherItem, 1);
|
||||
auto const otherData = serializeNode(otherNode);
|
||||
auto const otherKey = otherItem->key();
|
||||
auto const otherDepth = 1;
|
||||
auto const otherID = SHAMapNodeID::createID(otherDepth, otherKey);
|
||||
|
||||
protocol::TMLedgerNode ledgerNode;
|
||||
ledgerNode.set_nodedata(otherData);
|
||||
ledgerNode.set_nodeid(otherID.getRawString());
|
||||
auto result = getSHAMapNodeID(ledgerNode, leafNode);
|
||||
BEAST_EXPECT(!result.has_value());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public:
|
||||
void
|
||||
run() override
|
||||
{
|
||||
testValidateLedgerNode();
|
||||
testGetTreeNode();
|
||||
testGetSHAMapNodeID();
|
||||
}
|
||||
};
|
||||
|
||||
BEAST_DEFINE_TESTSUITE(LedgerNodeHelpers, app, xrpl);
|
||||
|
||||
} // namespace tests
|
||||
} // namespace xrpl
|
||||
@@ -1,6 +1,7 @@
|
||||
#include <test/jtx.h>
|
||||
#include <test/jtx/envconfig.h>
|
||||
|
||||
#include <xrpld/app/ledger/LedgerMaster.h>
|
||||
#include <xrpld/app/main/Application.h>
|
||||
#include <xrpld/app/main/NodeStoreScheduler.h>
|
||||
#include <xrpld/app/misc/SHAMapStore.h>
|
||||
@@ -10,6 +11,8 @@
|
||||
#include <xrpl/nodestore/detail/DatabaseRotatingImp.h>
|
||||
#include <xrpl/protocol/jss.h>
|
||||
|
||||
#include <thread>
|
||||
|
||||
namespace xrpl {
|
||||
namespace test {
|
||||
|
||||
@@ -20,10 +23,8 @@ class SHAMapStore_test : public beast::unit_test::suite
|
||||
static auto
|
||||
onlineDelete(std::unique_ptr<Config> cfg)
|
||||
{
|
||||
cfg->LEDGER_HISTORY = deleteInterval;
|
||||
auto& section = cfg->section(ConfigSection::nodeDatabase());
|
||||
section.set("online_delete", std::to_string(deleteInterval));
|
||||
return cfg;
|
||||
using namespace jtx;
|
||||
return online_delete(std::move(cfg), deleteInterval);
|
||||
}
|
||||
|
||||
static auto
|
||||
@@ -563,6 +564,158 @@ public:
|
||||
BEAST_EXPECT(dbr->getName() == "3");
|
||||
}
|
||||
|
||||
void
|
||||
testLedgerGaps()
|
||||
{
|
||||
// Note that this test is intentionally very similar to
|
||||
// LedgerMaster_test::testCompleteLedgerRange, but has a different
|
||||
// focus.
|
||||
|
||||
testcase("Wait for ledger gaps to fill in");
|
||||
|
||||
using namespace test::jtx;
|
||||
|
||||
Env env{*this, envconfig(onlineDelete)};
|
||||
|
||||
std::map<LedgerIndex, uint256> hashes;
|
||||
|
||||
auto failureMessage = [&](char const* label, auto expected, auto actual) {
|
||||
std::stringstream ss;
|
||||
ss << label << ": Expected: " << expected << ", Got: " << actual;
|
||||
return ss.str();
|
||||
};
|
||||
|
||||
auto const alice = Account("alice");
|
||||
env.fund(XRP(1000), alice);
|
||||
env.close();
|
||||
|
||||
auto& lm = env.app().getLedgerMaster();
|
||||
LedgerIndex minSeq = 2;
|
||||
LedgerIndex maxSeq = env.closed()->header().seq;
|
||||
auto& store = env.app().getSHAMapStore();
|
||||
store.rendezvous();
|
||||
LedgerIndex lastRotated = store.getLastRotated();
|
||||
BEAST_EXPECTS(maxSeq == 3, to_string(maxSeq));
|
||||
BEAST_EXPECTS(lm.getCompleteLedgers() == "2-3", lm.getCompleteLedgers());
|
||||
BEAST_EXPECTS(lastRotated == 3, to_string(lastRotated));
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
|
||||
|
||||
// Close enough ledgers to rotate a few times
|
||||
while (maxSeq < 20)
|
||||
{
|
||||
for (int t = 0; t < 3; ++t)
|
||||
{
|
||||
env(noop(alice));
|
||||
}
|
||||
env.close();
|
||||
store.rendezvous();
|
||||
|
||||
++maxSeq;
|
||||
|
||||
if (maxSeq + 1 == lastRotated + deleteInterval)
|
||||
{
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
// The next ledger will trigger a rotation. Delete the
|
||||
// current ledger from LedgerMaster.
|
||||
std::this_thread::sleep_for(100ms);
|
||||
LedgerIndex const deleteSeq = maxSeq;
|
||||
while (!lm.haveLedger(deleteSeq))
|
||||
{
|
||||
std::this_thread::sleep_for(100ms);
|
||||
}
|
||||
lm.clearLedger(deleteSeq);
|
||||
|
||||
auto expectedRange = [](auto minSeq, auto deleteSeq, auto maxSeq) {
|
||||
std::stringstream expectedRange;
|
||||
expectedRange << minSeq << "-" << (deleteSeq - 1);
|
||||
if (deleteSeq + 1 == maxSeq)
|
||||
expectedRange << "," << maxSeq;
|
||||
else if (deleteSeq < maxSeq)
|
||||
expectedRange << "," << (deleteSeq + 1) << "-" << maxSeq;
|
||||
return expectedRange.str();
|
||||
};
|
||||
BEAST_EXPECTS(
|
||||
lm.getCompleteLedgers() == expectedRange(minSeq, deleteSeq, maxSeq),
|
||||
failureMessage(
|
||||
"Complete ledgers",
|
||||
expectedRange(minSeq, deleteSeq, maxSeq),
|
||||
lm.getCompleteLedgers()));
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 1);
|
||||
|
||||
// Close another ledger, which will trigger a rotation, but the
|
||||
// rotation will be stuck until the missing ledger is filled in.
|
||||
env.close();
|
||||
// DO NOT CALL rendezvous()! You'll end up with a deadlock.
|
||||
++maxSeq;
|
||||
|
||||
// Nothing has changed
|
||||
BEAST_EXPECTS(
|
||||
store.getLastRotated() == lastRotated,
|
||||
failureMessage("lastRotated", lastRotated, store.getLastRotated()));
|
||||
BEAST_EXPECTS(
|
||||
lm.getCompleteLedgers() == expectedRange(minSeq, deleteSeq, maxSeq),
|
||||
failureMessage(
|
||||
"Complete ledgers",
|
||||
expectedRange(minSeq, deleteSeq, maxSeq),
|
||||
lm.getCompleteLedgers()));
|
||||
|
||||
// Close 5 more ledgers, waiting one second in between to
|
||||
// simulate the ledger making progress while online delete waits
|
||||
// for the missing ledger to be filled in.
|
||||
// This ensures the healthWait check has time to run and
|
||||
// detect the gap.
|
||||
for (int l = 0; l < 5; ++l)
|
||||
{
|
||||
env.close();
|
||||
// DO NOT CALL rendezvous()! You'll end up with a deadlock.
|
||||
++maxSeq;
|
||||
// Nothing has changed
|
||||
BEAST_EXPECTS(
|
||||
store.getLastRotated() == lastRotated,
|
||||
failureMessage("lastRotated", lastRotated, store.getLastRotated()));
|
||||
BEAST_EXPECTS(
|
||||
lm.getCompleteLedgers() == expectedRange(minSeq, deleteSeq, maxSeq),
|
||||
failureMessage(
|
||||
"Complete Ledgers",
|
||||
expectedRange(minSeq, deleteSeq, maxSeq),
|
||||
lm.getCompleteLedgers()));
|
||||
std::this_thread::sleep_for(1s);
|
||||
}
|
||||
|
||||
// Put the missing ledger back in LedgerMaster
|
||||
lm.setLedgerRangePresent(deleteSeq, deleteSeq);
|
||||
|
||||
// Wait for the rotation to finish
|
||||
store.rendezvous();
|
||||
|
||||
minSeq = lastRotated;
|
||||
lastRotated = deleteSeq + 1;
|
||||
}
|
||||
BEAST_EXPECT(maxSeq != lastRotated + deleteInterval);
|
||||
BEAST_EXPECTS(
|
||||
env.closed()->header().seq == maxSeq,
|
||||
failureMessage("maxSeq", maxSeq, env.closed()->header().seq));
|
||||
BEAST_EXPECTS(
|
||||
store.getLastRotated() == lastRotated,
|
||||
failureMessage("lastRotated", lastRotated, store.getLastRotated()));
|
||||
std::stringstream expectedRange;
|
||||
expectedRange << minSeq << "-" << maxSeq;
|
||||
BEAST_EXPECTS(
|
||||
lm.getCompleteLedgers() == expectedRange.str(),
|
||||
failureMessage("CompleteLedgers", expectedRange.str(), lm.getCompleteLedgers()));
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
run() override
|
||||
{
|
||||
@@ -570,6 +723,7 @@ public:
|
||||
testAutomatic();
|
||||
testCanDelete();
|
||||
testRotate();
|
||||
testLedgerGaps();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -57,6 +57,17 @@ envconfig(F&& modfunc, Args&&... args)
|
||||
return modfunc(envconfig(), std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
/// @brief adjust config to enable online_delete
|
||||
///
|
||||
/// @param cfg config instance to be modified
|
||||
///
|
||||
/// @param deleteInterval how many new ledgers should be available before
|
||||
/// rotating. Defaults to 8, because the standalone minimum is 8.
|
||||
///
|
||||
/// @return unique_ptr to Config instance
|
||||
std::unique_ptr<Config>
|
||||
online_delete(std::unique_ptr<Config> cfg, std::uint32_t deleteInterval = 8);
|
||||
|
||||
/// @brief adjust config so no admin ports are enabled
|
||||
///
|
||||
/// this is intended for use with envconfig, as in
|
||||
|
||||
@@ -53,6 +53,15 @@ setupConfigForUnitTests(Config& cfg)
|
||||
|
||||
namespace jtx {
|
||||
|
||||
std::unique_ptr<Config>
|
||||
online_delete(std::unique_ptr<Config> cfg, std::uint32_t deleteInterval)
|
||||
{
|
||||
cfg->LEDGER_HISTORY = deleteInterval;
|
||||
auto& section = cfg->section(ConfigSection::nodeDatabase());
|
||||
section.set("online_delete", std::to_string(deleteInterval));
|
||||
return cfg;
|
||||
}
|
||||
|
||||
std::unique_ptr<Config>
|
||||
no_admin(std::unique_ptr<Config> cfg)
|
||||
{
|
||||
|
||||
@@ -60,8 +60,8 @@ public:
|
||||
negotiateProtocolVersion("RTXP/1.2, XRPL/2.0, XRPL/2.1") == make_protocol(2, 1));
|
||||
BEAST_EXPECT(negotiateProtocolVersion("XRPL/2.2") == make_protocol(2, 2));
|
||||
BEAST_EXPECT(
|
||||
negotiateProtocolVersion("RTXP/1.2, XRPL/2.3, XRPL/2.4, XRPL/999.999") ==
|
||||
make_protocol(2, 3));
|
||||
negotiateProtocolVersion("RTXP/1.2, XRPL/2.2, XRPL/2.3, XRPL/999.999") ==
|
||||
make_protocol(2, 2));
|
||||
BEAST_EXPECT(negotiateProtocolVersion("XRPL/999.999, WebSocket/1.0") == std::nullopt);
|
||||
BEAST_EXPECT(negotiateProtocolVersion("") == std::nullopt);
|
||||
}
|
||||
|
||||
@@ -99,17 +99,14 @@ public:
|
||||
destination.setSynching();
|
||||
|
||||
{
|
||||
std::vector<std::tuple<SHAMapNodeID, Blob, bool>> a;
|
||||
std::vector<std::pair<SHAMapNodeID, Blob>> a;
|
||||
|
||||
BEAST_EXPECT(source.getNodeFat(SHAMapNodeID(), a, rand_bool(eng_), rand_int(eng_, 2)));
|
||||
|
||||
unexpected(a.empty(), "NodeSize");
|
||||
|
||||
auto node = SHAMapTreeNode::makeFromWire(makeSlice(std::get<1>(a[0])));
|
||||
if (!node)
|
||||
fail("", __FILE__, __LINE__);
|
||||
BEAST_EXPECT(
|
||||
destination.addRootNode(source.getHash(), std::move(node), nullptr).isGood());
|
||||
BEAST_EXPECT(destination.addRootNode(source.getHash(), makeSlice(a[0].second), nullptr)
|
||||
.isGood());
|
||||
}
|
||||
|
||||
do
|
||||
@@ -123,7 +120,7 @@ public:
|
||||
break;
|
||||
|
||||
// get as many nodes as possible based on this information
|
||||
std::vector<std::tuple<SHAMapNodeID, Blob, bool>> b;
|
||||
std::vector<std::pair<SHAMapNodeID, Blob>> b;
|
||||
|
||||
for (auto& it : nodesMissing)
|
||||
{
|
||||
@@ -145,10 +142,7 @@ public:
|
||||
// Don't use BEAST_EXPECT here b/c it will be called a
|
||||
// non-deterministic number of times and the number of tests run
|
||||
// should be deterministic
|
||||
auto node = SHAMapTreeNode::makeFromWire(makeSlice(std::get<1>(b[i])));
|
||||
if (!node)
|
||||
fail("", __FILE__, __LINE__);
|
||||
if (!destination.addKnownNode(std::get<0>(b[i]), std::move(node), nullptr)
|
||||
if (!destination.addKnownNode(b[i].first, makeSlice(b[i].second), nullptr)
|
||||
.isUseful())
|
||||
fail("", __FILE__, __LINE__);
|
||||
}
|
||||
|
||||
@@ -9,7 +9,6 @@
|
||||
|
||||
#include <mutex>
|
||||
#include <set>
|
||||
#include <string_view>
|
||||
#include <utility>
|
||||
|
||||
namespace xrpl {
|
||||
@@ -132,16 +131,16 @@ private:
|
||||
processData(std::shared_ptr<Peer> peer, protocol::TMLedgerData& data);
|
||||
|
||||
bool
|
||||
takeHeader(std::string_view data);
|
||||
takeHeader(std::string const& data);
|
||||
|
||||
void
|
||||
receiveNode(protocol::TMLedgerData& packet, SHAMapAddNode& san);
|
||||
receiveNode(protocol::TMLedgerData& packet, SHAMapAddNode&);
|
||||
|
||||
bool
|
||||
takeTxRootNode(std::string_view data, SHAMapAddNode& san);
|
||||
takeTxRootNode(Slice const& data, SHAMapAddNode&);
|
||||
|
||||
bool
|
||||
takeAsRootNode(std::string_view data, SHAMapAddNode& san);
|
||||
takeAsRootNode(Slice const& data, SHAMapAddNode&);
|
||||
|
||||
std::vector<uint256>
|
||||
neededTxHashes(int max, SHAMapSyncFilter* filter) const;
|
||||
|
||||
@@ -105,7 +105,10 @@ public:
|
||||
failedSave(std::uint32_t seq, uint256 const& hash);
|
||||
|
||||
std::string
|
||||
getCompleteLedgers();
|
||||
getCompleteLedgers() const;
|
||||
|
||||
std::size_t
|
||||
missingFromCompleteLedgerRange(LedgerIndex first, LedgerIndex last) const;
|
||||
|
||||
/** Apply held transactions to the open ledger
|
||||
This is normally called as we close the ledger.
|
||||
@@ -322,7 +325,7 @@ private:
|
||||
// A set of transactions to replay during the next close
|
||||
std::unique_ptr<LedgerReplay> replayData;
|
||||
|
||||
std::recursive_mutex mCompleteLock;
|
||||
std::recursive_mutex mutable mCompleteLock;
|
||||
RangeSet<std::uint32_t> mCompleteLedgers;
|
||||
|
||||
// Publish thread is running.
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
#include <xrpld/app/ledger/InboundLedgers.h>
|
||||
#include <xrpld/app/ledger/LedgerMaster.h>
|
||||
#include <xrpld/app/ledger/TransactionStateSF.h>
|
||||
#include <xrpld/app/ledger/detail/LedgerNodeHelpers.h>
|
||||
#include <xrpld/app/main/Application.h>
|
||||
#include <xrpld/overlay/Overlay.h>
|
||||
|
||||
@@ -763,7 +762,7 @@ InboundLedger::filterNodes(
|
||||
*/
|
||||
// data must not have hash prefix
|
||||
bool
|
||||
InboundLedger::takeHeader(std::string_view data)
|
||||
InboundLedger::takeHeader(std::string const& data)
|
||||
{
|
||||
// Return value: true=normal, false=bad data
|
||||
JLOG(journal_.trace()) << "got header acquiring ledger " << hash_;
|
||||
@@ -851,31 +850,20 @@ InboundLedger::receiveNode(protocol::TMLedgerData& packet, SHAMapAddNode& san)
|
||||
{
|
||||
auto const f = filter.get();
|
||||
|
||||
for (auto const& ledger_node : packet.nodes())
|
||||
for (auto const& node : packet.nodes())
|
||||
{
|
||||
auto treeNode = getTreeNode(ledger_node.nodedata());
|
||||
if (!treeNode)
|
||||
{
|
||||
JLOG(journal_.warn()) << "Got invalid node data";
|
||||
san.incInvalid();
|
||||
return;
|
||||
}
|
||||
auto const nodeID = deserializeSHAMapNodeID(node.nodeid());
|
||||
|
||||
auto const nodeID = getSHAMapNodeID(ledger_node, *treeNode);
|
||||
if (!nodeID)
|
||||
{
|
||||
JLOG(journal_.warn()) << "Got invalid node id";
|
||||
san.incInvalid();
|
||||
return;
|
||||
}
|
||||
throw std::runtime_error("data does not properly deserialize");
|
||||
|
||||
if (nodeID->isRoot())
|
||||
{
|
||||
san += map.addRootNode(rootHash, std::move(*treeNode), f);
|
||||
san += map.addRootNode(rootHash, makeSlice(node.nodedata()), f);
|
||||
}
|
||||
else
|
||||
{
|
||||
san += map.addKnownNode(*nodeID, std::move(*treeNode), f);
|
||||
san += map.addKnownNode(*nodeID, makeSlice(node.nodedata()), f);
|
||||
}
|
||||
|
||||
if (!san.isGood())
|
||||
@@ -915,7 +903,7 @@ InboundLedger::receiveNode(protocol::TMLedgerData& packet, SHAMapAddNode& san)
|
||||
Call with a lock
|
||||
*/
|
||||
bool
|
||||
InboundLedger::takeAsRootNode(std::string_view data, SHAMapAddNode& san)
|
||||
InboundLedger::takeAsRootNode(Slice const& data, SHAMapAddNode& san)
|
||||
{
|
||||
if (failed_ || mHaveState)
|
||||
{
|
||||
@@ -931,17 +919,9 @@ InboundLedger::takeAsRootNode(std::string_view data, SHAMapAddNode& san)
|
||||
// LCOV_EXCL_STOP
|
||||
}
|
||||
|
||||
auto treeNode = getTreeNode(data);
|
||||
if (!treeNode)
|
||||
{
|
||||
JLOG(journal_.warn()) << "Got invalid node data";
|
||||
san.incInvalid();
|
||||
return false;
|
||||
}
|
||||
|
||||
AccountStateSF filter(mLedger->stateMap().family().db(), app_.getLedgerMaster());
|
||||
san += mLedger->stateMap().addRootNode(
|
||||
SHAMapHash{mLedger->header().accountHash}, std::move(*treeNode), &filter);
|
||||
san +=
|
||||
mLedger->stateMap().addRootNode(SHAMapHash{mLedger->header().accountHash}, data, &filter);
|
||||
return san.isGood();
|
||||
}
|
||||
|
||||
@@ -949,7 +929,7 @@ InboundLedger::takeAsRootNode(std::string_view data, SHAMapAddNode& san)
|
||||
Call with a lock
|
||||
*/
|
||||
bool
|
||||
InboundLedger::takeTxRootNode(std::string_view data, SHAMapAddNode& san)
|
||||
InboundLedger::takeTxRootNode(Slice const& data, SHAMapAddNode& san)
|
||||
{
|
||||
if (failed_ || mHaveTransactions)
|
||||
{
|
||||
@@ -965,17 +945,8 @@ InboundLedger::takeTxRootNode(std::string_view data, SHAMapAddNode& san)
|
||||
// LCOV_EXCL_STOP
|
||||
}
|
||||
|
||||
auto treeNode = getTreeNode(data);
|
||||
if (!treeNode)
|
||||
{
|
||||
JLOG(journal_.warn()) << "Got invalid node data";
|
||||
san.incInvalid();
|
||||
return false;
|
||||
}
|
||||
|
||||
TransactionStateSF filter(mLedger->txMap().family().db(), app_.getLedgerMaster());
|
||||
san += mLedger->txMap().addRootNode(
|
||||
SHAMapHash{mLedger->header().txHash}, std::move(*treeNode), &filter);
|
||||
san += mLedger->txMap().addRootNode(SHAMapHash{mLedger->header().txHash}, data, &filter);
|
||||
return san.isGood();
|
||||
}
|
||||
|
||||
@@ -1072,13 +1043,13 @@ InboundLedger::processData(std::shared_ptr<Peer> peer, protocol::TMLedgerData& p
|
||||
}
|
||||
|
||||
if (!mHaveState && (packet.nodes().size() > 1) &&
|
||||
!takeAsRootNode(packet.nodes(1).nodedata(), san))
|
||||
!takeAsRootNode(makeSlice(packet.nodes(1).nodedata()), san))
|
||||
{
|
||||
JLOG(journal_.warn()) << "Included AS root invalid";
|
||||
}
|
||||
|
||||
if (!mHaveTransactions && (packet.nodes().size() > 2) &&
|
||||
!takeTxRootNode(packet.nodes(2).nodedata(), san))
|
||||
!takeTxRootNode(makeSlice(packet.nodes(2).nodedata()), san))
|
||||
{
|
||||
JLOG(journal_.warn()) << "Included TX root invalid";
|
||||
}
|
||||
@@ -1109,13 +1080,13 @@ InboundLedger::processData(std::shared_ptr<Peer> peer, protocol::TMLedgerData& p
|
||||
|
||||
ScopedLockType const sl(mtx_);
|
||||
|
||||
// Verify nodes are complete
|
||||
for (auto const& ledger_node : packet.nodes())
|
||||
// Verify node IDs and data are complete
|
||||
for (auto const& node : packet.nodes())
|
||||
{
|
||||
if (!validateLedgerNode(ledger_node))
|
||||
if (!node.has_nodeid() || !node.has_nodedata())
|
||||
{
|
||||
JLOG(journal_.warn()) << "Got malformed ledger node";
|
||||
peer->charge(Resource::feeMalformedRequest, "ledger_node");
|
||||
JLOG(journal_.warn()) << "Got bad node";
|
||||
peer->charge(Resource::feeMalformedRequest, "ledger_data bad node");
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
#include <xrpld/app/ledger/InboundLedgers.h>
|
||||
#include <xrpld/app/ledger/LedgerMaster.h>
|
||||
#include <xrpld/app/ledger/detail/LedgerNodeHelpers.h>
|
||||
#include <xrpld/app/main/Application.h>
|
||||
|
||||
#include <xrpl/basics/DecayingSample.h>
|
||||
@@ -224,21 +223,23 @@ public:
|
||||
Serializer s;
|
||||
try
|
||||
{
|
||||
for (auto const& ledger_node : packet_ptr->nodes())
|
||||
for (int i = 0; i < packet_ptr->nodes().size(); ++i)
|
||||
{
|
||||
if (!validateLedgerNode(ledger_node))
|
||||
auto const& node = packet_ptr->nodes(i);
|
||||
|
||||
if (!node.has_nodeid() || !node.has_nodedata())
|
||||
return;
|
||||
|
||||
auto const treeNode = getTreeNode(ledger_node.nodedata());
|
||||
if (!treeNode)
|
||||
auto newNode = SHAMapTreeNode::makeFromWire(makeSlice(node.nodedata()));
|
||||
|
||||
if (!newNode)
|
||||
return;
|
||||
auto const tn = *treeNode;
|
||||
|
||||
s.erase();
|
||||
tn->serializeWithPrefix(s);
|
||||
newNode->serializeWithPrefix(s);
|
||||
|
||||
app_.getLedgerMaster().addFetchPack(
|
||||
tn->getHash().as_uint256(), std::make_shared<Blob>(s.begin(), s.end()));
|
||||
newNode->getHash().as_uint256(), std::make_shared<Blob>(s.begin(), s.end()));
|
||||
}
|
||||
}
|
||||
catch (std::exception const&) // NOLINT(bugprone-empty-catch)
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
#include <xrpld/app/ledger/InboundLedgers.h>
|
||||
#include <xrpld/app/ledger/InboundTransactions.h>
|
||||
#include <xrpld/app/ledger/detail/LedgerNodeHelpers.h>
|
||||
#include <xrpld/app/ledger/detail/TransactionAcquire.h>
|
||||
#include <xrpld/app/main/Application.h>
|
||||
|
||||
@@ -132,35 +131,26 @@ public:
|
||||
return;
|
||||
}
|
||||
|
||||
std::vector<std::pair<SHAMapNodeID, intr_ptr::SharedPtr<SHAMapTreeNode>>> data;
|
||||
std::vector<std::pair<SHAMapNodeID, Slice>> data;
|
||||
data.reserve(packet.nodes().size());
|
||||
|
||||
for (auto const& ledger_node : packet.nodes())
|
||||
for (auto const& node : packet.nodes())
|
||||
{
|
||||
if (!validateLedgerNode(ledger_node))
|
||||
if (!node.has_nodeid() || !node.has_nodedata())
|
||||
{
|
||||
JLOG(j_.warn()) << "Got malformed ledger node";
|
||||
peer->charge(Resource::feeMalformedRequest, "ledger_node");
|
||||
peer->charge(Resource::feeMalformedRequest, "ledger_data");
|
||||
return;
|
||||
}
|
||||
|
||||
auto treeNode = getTreeNode(ledger_node.nodedata());
|
||||
if (!treeNode)
|
||||
auto const id = deserializeSHAMapNodeID(node.nodeid());
|
||||
|
||||
if (!id)
|
||||
{
|
||||
JLOG(j_.warn()) << "Got invalid node data";
|
||||
peer->charge(Resource::feeInvalidData, "node_data");
|
||||
peer->charge(Resource::feeInvalidData, "ledger_data");
|
||||
return;
|
||||
}
|
||||
|
||||
auto const nodeID = getSHAMapNodeID(ledger_node, *treeNode);
|
||||
if (!nodeID)
|
||||
{
|
||||
JLOG(j_.warn()) << "Got invalid node id";
|
||||
peer->charge(Resource::feeInvalidData, "node_id");
|
||||
return;
|
||||
}
|
||||
|
||||
data.emplace_back(std::make_pair(*nodeID, std::move(*treeNode)));
|
||||
data.emplace_back(std::make_pair(*id, makeSlice(node.nodedata())));
|
||||
}
|
||||
|
||||
if (!ta->takeNodes(data, peer).isUseful())
|
||||
|
||||
@@ -1529,12 +1529,34 @@ LedgerMaster::getPublishedLedger()
|
||||
}
|
||||
|
||||
std::string
|
||||
LedgerMaster::getCompleteLedgers()
|
||||
LedgerMaster::getCompleteLedgers() const
|
||||
{
|
||||
std::lock_guard const sl(mCompleteLock);
|
||||
return to_string(mCompleteLedgers);
|
||||
}
|
||||
|
||||
std::size_t
|
||||
LedgerMaster::missingFromCompleteLedgerRange(LedgerIndex first, LedgerIndex last) const
|
||||
{
|
||||
// Make a copy of the range to avoid holding the lock
|
||||
auto const range = [&] {
|
||||
std::lock_guard sl(mCompleteLock);
|
||||
return mCompleteLedgers;
|
||||
}();
|
||||
|
||||
std::size_t missing = 0;
|
||||
|
||||
for (LedgerIndex idx = first; idx <= last; ++idx)
|
||||
{
|
||||
if (!boost::icl::contains(range, idx))
|
||||
{
|
||||
++missing;
|
||||
}
|
||||
}
|
||||
|
||||
return missing;
|
||||
}
|
||||
|
||||
std::optional<NetClock::time_point>
|
||||
LedgerMaster::getCloseTimeBySeq(LedgerIndex ledgerIndex)
|
||||
{
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
#include <xrpld/app/ledger/detail/LedgerNodeHelpers.h>
|
||||
|
||||
#include <xrpl/basics/IntrusivePointer.h>
|
||||
#include <xrpl/basics/Slice.h>
|
||||
#include <xrpl/basics/safe_cast.h>
|
||||
#include <xrpl/beast/utility/instrumentation.h>
|
||||
#include <xrpl/protocol/messages.h>
|
||||
#include <xrpl/shamap/SHAMap.h>
|
||||
#include <xrpl/shamap/SHAMapLeafNode.h>
|
||||
#include <xrpl/shamap/SHAMapNodeID.h>
|
||||
#include <xrpl/shamap/SHAMapTreeNode.h>
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
namespace xrpl {
|
||||
|
||||
bool
|
||||
validateLedgerNode(protocol::TMLedgerNode const& ledger_node)
|
||||
{
|
||||
if (!ledger_node.has_nodedata())
|
||||
return false;
|
||||
|
||||
if (ledger_node.has_nodeid())
|
||||
return !ledger_node.has_id() && !ledger_node.has_depth();
|
||||
|
||||
return ledger_node.has_id() ||
|
||||
(ledger_node.has_depth() && ledger_node.depth() <= SHAMap::leafDepth);
|
||||
}
|
||||
|
||||
std::optional<intr_ptr::SharedPtr<SHAMapTreeNode>>
|
||||
getTreeNode(std::string_view data)
|
||||
{
|
||||
auto const slice = makeSlice(data);
|
||||
try
|
||||
{
|
||||
auto treeNode = SHAMapTreeNode::makeFromWire(slice);
|
||||
if (!treeNode)
|
||||
return std::nullopt;
|
||||
return treeNode;
|
||||
}
|
||||
catch (std::exception const&)
|
||||
{
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<SHAMapNodeID>
|
||||
getSHAMapNodeID(
|
||||
protocol::TMLedgerNode const& ledger_node,
|
||||
intr_ptr::SharedPtr<SHAMapTreeNode> const& treeNode)
|
||||
{
|
||||
if (ledger_node.has_id() || ledger_node.has_depth())
|
||||
{
|
||||
if (treeNode->isInner())
|
||||
{
|
||||
if (!ledger_node.has_id())
|
||||
return std::nullopt;
|
||||
|
||||
return deserializeSHAMapNodeID(ledger_node.id());
|
||||
}
|
||||
|
||||
if (treeNode->isLeaf())
|
||||
{
|
||||
if (!ledger_node.has_depth())
|
||||
return std::nullopt;
|
||||
|
||||
auto const key =
|
||||
safe_downcast<SHAMapLeafNode const*>(treeNode.get())->peekItem()->key();
|
||||
return SHAMapNodeID::createID(ledger_node.depth(), key);
|
||||
}
|
||||
|
||||
UNREACHABLE("xrpl::getSHAMapNodeID : tree node is neither inner nor leaf");
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
if (!ledger_node.has_nodeid())
|
||||
return std::nullopt;
|
||||
|
||||
auto nodeID = deserializeSHAMapNodeID(ledger_node.nodeid());
|
||||
if (!nodeID.has_value())
|
||||
return std::nullopt;
|
||||
|
||||
if (treeNode->isLeaf())
|
||||
{
|
||||
auto const key = safe_downcast<SHAMapLeafNode const*>(treeNode.get())->peekItem()->key();
|
||||
auto const expected_id = SHAMapNodeID::createID(static_cast<int>(nodeID->getDepth()), key);
|
||||
if (nodeID->getNodeID() != expected_id.getNodeID())
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
return nodeID;
|
||||
}
|
||||
|
||||
} // namespace xrpl
|
||||
@@ -1,75 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <xrpl/basics/IntrusivePointer.h>
|
||||
#include <xrpl/shamap/SHAMapNodeID.h>
|
||||
#include <xrpl/shamap/SHAMapTreeNode.h>
|
||||
|
||||
#include <optional>
|
||||
#include <string_view>
|
||||
|
||||
namespace protocol {
|
||||
class TMLedgerNode;
|
||||
} // namespace protocol
|
||||
|
||||
namespace xrpl {
|
||||
|
||||
/**
|
||||
* @brief Validates a ledger node proto message.
|
||||
*
|
||||
* This function checks whether a ledger node has the expected fields (for non-ledger base data):
|
||||
* - The node must have `nodedata`.
|
||||
* - If the legacy `nodeid` field is present then the new `id` and `depth` fields must not be
|
||||
* present.
|
||||
* - If the new `id` or `depth` fields are present (it is a oneof field, so only one of the two can
|
||||
* be set) then the legacy `nodeid` must not be present.
|
||||
* - If the `depth` field is present then it must be between 0 and SHAMap::leafDepth (inclusive).
|
||||
*
|
||||
* @param ledger_node The ledger node to validate.
|
||||
* @return true if the ledger node has the expected fields, false otherwise.
|
||||
*/
|
||||
[[nodiscard]] bool
|
||||
validateLedgerNode(protocol::TMLedgerNode const& ledger_node);
|
||||
|
||||
/**
|
||||
* @brief Deserializes a SHAMapTreeNode from wire format data.
|
||||
*
|
||||
* This function attempts to create a SHAMapTreeNode from the provided data string. If the data is
|
||||
* malformed or deserialization fails, the function returns std::nullopt instead of throwing an
|
||||
* exception.
|
||||
*
|
||||
* @param data The serialized node data in wire format.
|
||||
* @return An optional containing the deserialized tree node if successful, or std::nullopt if
|
||||
* deserialization fails.
|
||||
*/
|
||||
[[nodiscard]] std::optional<intr_ptr::SharedPtr<SHAMapTreeNode>>
|
||||
getTreeNode(std::string_view data);
|
||||
|
||||
/**
|
||||
* @brief Extracts or reconstructs the SHAMapNodeID from a ledger node proto message.
|
||||
*
|
||||
* This function retrieves the SHAMapNodeID for a tree node, with behavior that depends on which
|
||||
* field is set and the node type (inner vs. leaf).
|
||||
*
|
||||
* When the legacy `nodeid` field is set in the message:
|
||||
* - For all nodes: Deserializes the node ID from the field.
|
||||
* - For leaf nodes: Validates that the node ID is consistent with the leaf's key.
|
||||
*
|
||||
* When the new `id` or `depth` field is set in the message:
|
||||
* - For inner nodes: Deserializes the node ID from the `id` field. Note that root nodes are also
|
||||
* inner nodes.
|
||||
* - For leaf nodes: Reconstructs the node ID using both the depth from the `depth` field and the
|
||||
* key from the leaf node's item.
|
||||
*
|
||||
* @param ledger_node The validated protocol message containing the ledger node data.
|
||||
* @param treeNode The deserialized tree node (inner or leaf node).
|
||||
* @return An optional containing the node ID if extraction/reconstruction succeeds, or std::nullopt
|
||||
* if the required fields are missing or validation fails.
|
||||
* @note This function expects that the caller has already validated the ledger node by calling the
|
||||
* `validateLedgerNode` function and obtained a valid tree node by calling `getTreeNode`.
|
||||
*/
|
||||
[[nodiscard]] std::optional<SHAMapNodeID>
|
||||
getSHAMapNodeID(
|
||||
protocol::TMLedgerNode const& ledger_node,
|
||||
intr_ptr::SharedPtr<SHAMapTreeNode> const& treeNode);
|
||||
|
||||
} // namespace xrpl
|
||||
@@ -158,7 +158,7 @@ TransactionAcquire::trigger(std::shared_ptr<Peer> const& peer)
|
||||
|
||||
SHAMapAddNode
|
||||
TransactionAcquire::takeNodes(
|
||||
std::vector<std::pair<SHAMapNodeID, intr_ptr::SharedPtr<SHAMapTreeNode>>> const& data,
|
||||
std::vector<std::pair<SHAMapNodeID, Slice>> const& data,
|
||||
std::shared_ptr<Peer> const& peer)
|
||||
{
|
||||
ScopedLockType const sl(mtx_);
|
||||
@@ -182,7 +182,7 @@ TransactionAcquire::takeNodes(
|
||||
|
||||
ConsensusTransSetSF sf(app_, app_.getTempNodeCache());
|
||||
|
||||
for (auto& d : data)
|
||||
for (auto const& d : data)
|
||||
{
|
||||
if (d.first.isRoot())
|
||||
{
|
||||
|
||||
@@ -20,8 +20,8 @@ public:
|
||||
|
||||
SHAMapAddNode
|
||||
takeNodes(
|
||||
std::vector<std::pair<SHAMapNodeID, intr_ptr::SharedPtr<SHAMapTreeNode>>> const& data,
|
||||
std::shared_ptr<Peer> const& peer);
|
||||
std::vector<std::pair<SHAMapNodeID, Slice>> const& data,
|
||||
std::shared_ptr<Peer> const&);
|
||||
|
||||
void
|
||||
init(int startPeers);
|
||||
|
||||
@@ -263,6 +263,17 @@ SHAMapStoreImp::run()
|
||||
bool const readyToRotate = validatedSeq >= lastRotated + deleteInterval_ &&
|
||||
canDelete_ >= lastRotated - 1 && healthWait() == keepGoing;
|
||||
|
||||
JLOG(journal_.debug()) << "run: Setting lastGoodValidatedLedger_ to " << validatedSeq;
|
||||
|
||||
{
|
||||
// Note that this is set after the healthWait() check, so that we
|
||||
// don't start the rotation until the validated ledger is fully
|
||||
// processed. It is not guaranteed to be done at this point. It also
|
||||
// allows the testLedgerGaps unit test to work.
|
||||
std::unique_lock<std::mutex> lock(mutex_);
|
||||
lastGoodValidatedLedger_ = validatedSeq;
|
||||
}
|
||||
|
||||
// will delete up to (not including) lastRotated
|
||||
if (readyToRotate)
|
||||
{
|
||||
@@ -270,7 +281,8 @@ SHAMapStoreImp::run()
|
||||
<< lastRotated << " deleteInterval " << deleteInterval_
|
||||
<< " canDelete_ " << canDelete_ << " state "
|
||||
<< app_.getOPs().strOperatingMode(false) << " age "
|
||||
<< ledgerMaster_->getValidatedLedgerAge().count() << 's';
|
||||
<< ledgerMaster_->getValidatedLedgerAge().count()
|
||||
<< "s. Complete ledgers: " << ledgerMaster_->getCompleteLedgers();
|
||||
|
||||
clearPrior(lastRotated);
|
||||
if (healthWait() == stopping)
|
||||
@@ -330,7 +342,9 @@ SHAMapStoreImp::run()
|
||||
clearCaches(validatedSeq);
|
||||
});
|
||||
|
||||
JLOG(journal_.warn()) << "finished rotation " << validatedSeq;
|
||||
JLOG(journal_.warn()) << "finished rotation. validatedSeq: " << validatedSeq
|
||||
<< ", lastRotated: " << lastRotated
|
||||
<< ". Complete ledgers: " << ledgerMaster_->getCompleteLedgers();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -570,22 +584,40 @@ SHAMapStoreImp::clearPrior(LedgerIndex lastRotated)
|
||||
SHAMapStoreImp::HealthResult
|
||||
SHAMapStoreImp::healthWait()
|
||||
{
|
||||
auto index = ledgerMaster_->getValidLedgerIndex();
|
||||
auto age = ledgerMaster_->getValidatedLedgerAge();
|
||||
OperatingMode mode = netOPs_->getOperatingMode();
|
||||
std::unique_lock lock(mutex_);
|
||||
while (!stop_ && (mode != OperatingMode::FULL || age > ageThreshold_))
|
||||
|
||||
auto numMissing =
|
||||
ledgerMaster_->missingFromCompleteLedgerRange(lastGoodValidatedLedger_, index);
|
||||
while (!stop_ && (mode != OperatingMode::FULL || age > ageThreshold_ || numMissing > 0))
|
||||
{
|
||||
// this value shouldn't change, so grab it while we have the
|
||||
// lock
|
||||
auto const lastGood = lastGoodValidatedLedger_;
|
||||
|
||||
lock.unlock();
|
||||
JLOG(journal_.warn()) << "Waiting " << recoveryWaitTime_.count()
|
||||
<< "s for node to stabilize. state: "
|
||||
<< app_.getOPs().strOperatingMode(mode, false) << ". age "
|
||||
<< age.count() << 's';
|
||||
|
||||
auto const stream =
|
||||
mode != OperatingMode::FULL || age > ageThreshold_ ? journal_.warn() : journal_.info();
|
||||
JLOG(stream) << "Waiting " << recoveryWaitTime_.count()
|
||||
<< "s for node to stabilize. state: "
|
||||
<< app_.getOPs().strOperatingMode(mode, false) << ". age " << age.count()
|
||||
<< "s. Missing ledgers: " << numMissing << ". Expect: " << lastGood << "-"
|
||||
<< index << ". Complete ledgers: " << ledgerMaster_->getCompleteLedgers();
|
||||
std::this_thread::sleep_for(recoveryWaitTime_);
|
||||
index = ledgerMaster_->getValidLedgerIndex();
|
||||
age = ledgerMaster_->getValidatedLedgerAge();
|
||||
mode = netOPs_->getOperatingMode();
|
||||
numMissing = ledgerMaster_->missingFromCompleteLedgerRange(lastGood, index);
|
||||
|
||||
lock.lock();
|
||||
}
|
||||
|
||||
JLOG(journal_.debug()) << "healthWait: Setting lastGoodValidatedLedger_ to " << index;
|
||||
lastGoodValidatedLedger_ = index;
|
||||
|
||||
return stop_ ? stopping : keepGoing;
|
||||
}
|
||||
|
||||
|
||||
@@ -70,6 +70,11 @@ private:
|
||||
std::thread thread_;
|
||||
bool stop_ = false;
|
||||
bool healthy_ = true;
|
||||
// Used to prevent ledger gaps from forming during online deletion. Keeps
|
||||
// track of the last validated ledger that was processed without gaps. There
|
||||
// are no guarantees about gaps while online delete is not running. For
|
||||
// that, use advisory_delete and check for gaps externally.
|
||||
LedgerIndex lastGoodValidatedLedger_ = 0;
|
||||
mutable std::condition_variable cond_;
|
||||
mutable std::condition_variable rendezvous_;
|
||||
mutable std::mutex mutex_;
|
||||
@@ -83,11 +88,11 @@ private:
|
||||
std::uint32_t deleteBatch_ = 100;
|
||||
std::chrono::milliseconds backOff_{100};
|
||||
std::chrono::seconds ageThreshold_{60};
|
||||
/// If the node is out of sync during an online_delete healthWait()
|
||||
/// call, sleep the thread for this time, and continue checking until
|
||||
/// recovery.
|
||||
/// If the node is out of sync, or any recent ledgers are not
|
||||
/// available during an online_delete healthWait() call, sleep
|
||||
/// the thread for this time, and continue checking until recovery.
|
||||
/// See also: "recovery_wait_seconds" in xrpld-example.cfg
|
||||
std::chrono::seconds recoveryWaitTime_{5};
|
||||
std::chrono::seconds recoveryWaitTime_{1};
|
||||
|
||||
// these do not exist upon SHAMapStore creation, but do exist
|
||||
// as of run() or before
|
||||
@@ -204,6 +209,8 @@ private:
|
||||
enum HealthResult { stopping, keepGoing };
|
||||
[[nodiscard]] HealthResult
|
||||
healthWait();
|
||||
bool
|
||||
hasCompleteRange(LedgerIndex first, LedgerIndex last);
|
||||
|
||||
public:
|
||||
void
|
||||
|
||||
@@ -17,7 +17,6 @@ enum class ProtocolFeature {
|
||||
ValidatorListPropagation,
|
||||
ValidatorList2Propagation,
|
||||
LedgerReplay,
|
||||
LedgerNodeDepth,
|
||||
};
|
||||
|
||||
/** Represents a peer connection in the overlay. */
|
||||
|
||||
@@ -30,7 +30,6 @@
|
||||
#include <mutex>
|
||||
#include <numeric>
|
||||
#include <sstream>
|
||||
#include <tuple>
|
||||
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
@@ -499,8 +498,6 @@ PeerImp::supportsFeature(ProtocolFeature f) const
|
||||
return protocol_ >= make_protocol(2, 1);
|
||||
case ProtocolFeature::ValidatorList2Propagation:
|
||||
return protocol_ >= make_protocol(2, 2);
|
||||
case ProtocolFeature::LedgerNodeDepth:
|
||||
return protocol_ >= make_protocol(2, 3);
|
||||
case ProtocolFeature::LedgerReplay:
|
||||
return ledgerReplayEnabled_;
|
||||
}
|
||||
@@ -3391,19 +3388,13 @@ PeerImp::processLedgerRequest(std::shared_ptr<protocol::TMGetLedger> const& m)
|
||||
std::uint32_t const defaultDepth = isHighLatency() ? 2 : 1;
|
||||
auto const queryDepth{m->has_querydepth() ? m->querydepth() : defaultDepth};
|
||||
|
||||
std::vector<std::tuple<SHAMapNodeID, Blob, bool>> data;
|
||||
auto const useLedgerNodeDepth = supportsFeature(ProtocolFeature::LedgerNodeDepth);
|
||||
std::vector<std::pair<SHAMapNodeID, Blob>> data;
|
||||
|
||||
for (int i = 0;
|
||||
i < m->nodeids_size() && ledgerData.nodes_size() < Tuning::softMaxReplyNodes;
|
||||
++i)
|
||||
{
|
||||
auto const shaMapNodeId{deserializeSHAMapNodeID(m->nodeids(i))};
|
||||
if (!shaMapNodeId.has_value())
|
||||
{
|
||||
JLOG(p_journal_.error()) << "processLedgerRequest: Invalid SHAMap node ID";
|
||||
return;
|
||||
}
|
||||
|
||||
data.clear();
|
||||
data.reserve(Tuning::softMaxReplyNodes);
|
||||
@@ -3419,28 +3410,9 @@ PeerImp::processLedgerRequest(std::shared_ptr<protocol::TMGetLedger> const& m)
|
||||
{
|
||||
if (ledgerData.nodes_size() >= Tuning::hardMaxReplyNodes)
|
||||
break;
|
||||
|
||||
protocol::TMLedgerNode* node{ledgerData.add_nodes()};
|
||||
|
||||
auto const& nodeData = std::get<1>(d);
|
||||
node->set_nodedata(nodeData.data(), nodeData.size());
|
||||
|
||||
// When the LedgerNodeDepth protocol feature is not supported by the peer,
|
||||
// we always set the `nodeid` field. However, when it is supported then we
|
||||
// set the `id` field for inner nodes and the `depth` field for leaf nodes.
|
||||
auto const& nodeID = std::get<0>(d);
|
||||
if (!useLedgerNodeDepth)
|
||||
{
|
||||
node->set_nodeid(nodeID.getRawString());
|
||||
}
|
||||
else if (std::get<2>(d))
|
||||
{
|
||||
node->set_depth(nodeID.getDepth());
|
||||
}
|
||||
else
|
||||
{
|
||||
node->set_id(nodeID.getRawString());
|
||||
}
|
||||
node->set_nodeid(d.first.getRawString());
|
||||
node->set_nodedata(d.second.data(), d.second.size());
|
||||
}
|
||||
}
|
||||
else
|
||||
|
||||
@@ -20,7 +20,6 @@ namespace xrpl {
|
||||
constexpr ProtocolVersion const supportedProtocolList[]{
|
||||
{2, 1},
|
||||
{2, 2},
|
||||
{2, 3},
|
||||
};
|
||||
|
||||
// This ugly construct ensures that supportedProtocolList is sorted in strictly
|
||||
|
||||
Reference in New Issue
Block a user