diff --git a/CMakeLists.txt b/CMakeLists.txt index 97cbe87839..3c369d980b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,9 @@ if (POLICY CMP0074) endif () project (rippled) -set (CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_EXTENSIONS OFF) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) # make GIT_COMMIT_HASH define available to all sources find_package(Git) diff --git a/src/ripple/proto/org/xrpl/rpc/v1/get_ledger.proto b/src/ripple/proto/org/xrpl/rpc/v1/get_ledger.proto index c3147b1f1e..0df9ca5ceb 100644 --- a/src/ripple/proto/org/xrpl/rpc/v1/get_ledger.proto +++ b/src/ripple/proto/org/xrpl/rpc/v1/get_ledger.proto @@ -32,6 +32,10 @@ message GetLedgerRequest // coming from a secure_gateway host, then the client is not subject to // resource controls string user = 6; + + // For every object in the diff, get the object's predecessor and successor + // in the state map. Only used if get_objects is also true. + bool get_object_neighbors = 7; } message GetLedgerResponse @@ -58,6 +62,18 @@ message GetLedgerResponse // True if request was exempt from resource controls bool is_unlimited = 7; + + // True if the response contains the state map diff + bool objects_included = 8; + + // True if the response contains key of objects adjacent to objects in state + // map diff + bool object_neighbors_included = 9; + + + // Successor information for book directories modified as part of this + // ledger + repeated BookSuccessor book_successors = 10; } message TransactionHashList diff --git a/src/ripple/proto/org/xrpl/rpc/v1/ledger.proto b/src/ripple/proto/org/xrpl/rpc/v1/ledger.proto index 70b0a35626..78e4211f5f 100644 --- a/src/ripple/proto/org/xrpl/rpc/v1/ledger.proto +++ b/src/ripple/proto/org/xrpl/rpc/v1/ledger.proto @@ -54,7 +54,14 @@ message RawLedgerObject DELETED = 3; } + // Whether the object was created, modified or deleted ModificationType mod_type = 3; + + // Key of the object preceding this object in the desired ledger + bytes predecessor = 4; + + // Key of the object succeeding this object in the desired ledger + bytes successor = 5; } message RawLedgerObjects @@ -62,3 +69,17 @@ message RawLedgerObjects repeated RawLedgerObject objects = 1; } +// Successor information for book directories. The book base is (usually) not +// an actual object, yet we need to be able to ask for the successor to the +// book base. +message BookSuccessor { + + // Base of the book in question + bytes book_base = 1; + + // First book directory in the book. An empty value here means the entire + // book is deleted + bytes first_book = 2; + +}; + diff --git a/src/ripple/rpc/handlers/LedgerHandler.cpp b/src/ripple/rpc/handlers/LedgerHandler.cpp index c358f27616..b2e4cb8dd1 100644 --- a/src/ripple/rpc/handlers/LedgerHandler.cpp +++ b/src/ripple/rpc/handlers/LedgerHandler.cpp @@ -105,6 +105,7 @@ LedgerHandler::check() std::pair doLedgerGrpc(RPC::GRPCContext& context) { + auto begin = std::chrono::system_clock::now(); org::xrpl::rpc::v1::GetLedgerRequest& request = context.params; org::xrpl::rpc::v1::GetLedgerResponse response; grpc::Status status = grpc::Status::OK; @@ -212,13 +213,101 @@ doLedgerGrpc(RPC::GRPCContext& context) obj->set_mod_type(org::xrpl::rpc::v1::RawLedgerObject::DELETED); else obj->set_mod_type(org::xrpl::rpc::v1::RawLedgerObject::CREATED); + auto const blob = inDesired ? inDesired->slice() : inBase->slice(); + auto const objectType = + static_cast(blob[1] << 8 | blob[2]); + + if (request.get_object_neighbors()) + { + if (!(inBase && inDesired)) + { + auto lb = desired->stateMap().lower_bound(k); + auto ub = desired->stateMap().upper_bound(k); + if (lb != desired->stateMap().end()) + obj->set_predecessor( + lb->key().data(), lb->key().size()); + if (ub != desired->stateMap().end()) + obj->set_successor(ub->key().data(), ub->key().size()); + if (objectType == ltDIR_NODE) + { + auto sle = std::make_shared(SerialIter{blob}, k); + if (!sle->isFieldPresent(sfOwner)) + { + auto bookBase = keylet::quality({ltDIR_NODE, k}, 0); + if (!inBase && inDesired) + { + auto firstBook = + desired->stateMap().upper_bound( + bookBase.key); + if (firstBook != desired->stateMap().end() && + firstBook->key() < + getQualityNext(bookBase.key) && + firstBook->key() == k) + { + auto succ = response.add_book_successors(); + succ->set_book_base( + bookBase.key.data(), + bookBase.key.size()); + succ->set_first_book( + firstBook->key().data(), + firstBook->key().size()); + } + } + if (inBase && !inDesired) + { + auto oldFirstBook = + base->stateMap().upper_bound(bookBase.key); + if (oldFirstBook != base->stateMap().end() && + oldFirstBook->key() < + getQualityNext(bookBase.key) && + oldFirstBook->key() == k) + { + auto succ = response.add_book_successors(); + succ->set_book_base( + bookBase.key.data(), + bookBase.key.size()); + auto newFirstBook = + desired->stateMap().upper_bound( + bookBase.key); + + if (newFirstBook != + desired->stateMap().end() && + newFirstBook->key() < + getQualityNext(bookBase.key)) + { + succ->set_first_book( + newFirstBook->key().data(), + newFirstBook->key().size()); + } + } + } + } + } + } + } } + response.set_objects_included(true); + response.set_object_neighbors_included(request.get_object_neighbors()); response.set_skiplist_included(true); } response.set_validated( RPC::isValidated(context.ledgerMaster, *ledger, context.app)); + auto end = std::chrono::system_clock::now(); + auto duration = + std::chrono::duration_cast(end - begin) + .count() * + 1.0; + JLOG(context.j.warn()) + << __func__ << " - Extract time = " << duration + << " - num objects = " << response.ledger_objects().objects_size() + << " - num txns = " << response.transactions_list().transactions_size() + << " - ms per obj " + << duration / response.ledger_objects().objects_size() + << " - ms per txn " + << duration / response.transactions_list().transactions_size(); + return {response, status}; } } // namespace ripple diff --git a/src/ripple/shamap/SHAMap.h b/src/ripple/shamap/SHAMap.h index 76e27bf1ea..b913bd5b1d 100644 --- a/src/ripple/shamap/SHAMap.h +++ b/src/ripple/shamap/SHAMap.h @@ -210,9 +210,17 @@ public: peekItem(uint256 const& id, SHAMapHash& hash) const; // traverse functions + + // finds the object in the tree with the smallest object id greater than the + // input id const_iterator upper_bound(uint256 const& id) const; + // finds the object in the tree with the greatest object id smaller than the + // input id + const_iterator + lower_bound(uint256 const& id) const; + /** Visit every node in this SHAMap @param function called with every node visited. @@ -400,12 +408,31 @@ private: std::shared_ptr writeNode(NodeObjectType t, std::shared_ptr node) const; + // returns the first item at or below this node SHAMapLeafNode* firstBelow( std::shared_ptr, SharedPtrNodeStack& stack, int branch = 0) const; + // returns the last item at or below this node + SHAMapLeafNode* + lastBelow( + std::shared_ptr node, + SharedPtrNodeStack& stack, + int branch = branchFactor) const; + + // helper function for firstBelow and lastBelow + SHAMapLeafNode* + belowHelper( + std::shared_ptr node, + SharedPtrNodeStack& stack, + int branch, + std::tuple< + int, + std::function, + std::function> const& loopParams) const; + // Simple descent // Get a child of the specified node SHAMapTreeNode* diff --git a/src/ripple/shamap/impl/SHAMap.cpp b/src/ripple/shamap/impl/SHAMap.cpp index 2bd1c85f32..27547aaec8 100644 --- a/src/ripple/shamap/impl/SHAMap.cpp +++ b/src/ripple/shamap/impl/SHAMap.cpp @@ -437,12 +437,14 @@ SHAMap::unshareNode(std::shared_ptr node, SHAMapNodeID const& nodeID) } SHAMapLeafNode* -SHAMap::firstBelow( +SHAMap::belowHelper( std::shared_ptr node, SharedPtrNodeStack& stack, - int branch) const + int branch, + std::tuple, std::function> const& + loopParams) const { - // Return the first item at or below this node + auto& [init, cmp, incr] = loopParams; if (node->isLeaf()) { auto n = std::static_pointer_cast(node); @@ -454,7 +456,7 @@ SHAMap::firstBelow( stack.push({inner, SHAMapNodeID{}}); else stack.push({inner, stack.top().second.getChildNodeID(branch)}); - for (int i = 0; i < branchFactor;) + for (int i = init; cmp(i);) { if (!inner->isEmptyBranch(i)) { @@ -468,14 +470,37 @@ SHAMap::firstBelow( } inner = std::static_pointer_cast(node); stack.push({inner, stack.top().second.getChildNodeID(branch)}); - i = 0; // scan all 16 branches of this new node + i = init; // descend and reset loop } else - ++i; // scan next branch + incr(i); // scan next branch } return nullptr; } +SHAMapLeafNode* +SHAMap::lastBelow( + std::shared_ptr node, + SharedPtrNodeStack& stack, + int branch) const +{ + auto init = branchFactor - 1; + auto cmp = [](int i) { return i >= 0; }; + auto incr = [](int& i) { --i; }; + return belowHelper(node, stack, branch, {init, cmp, incr}); +} +SHAMapLeafNode* +SHAMap::firstBelow( + std::shared_ptr node, + SharedPtrNodeStack& stack, + int branch) const +{ + auto init = 0; + auto cmp = [](int i) { return i <= branchFactor; }; + auto incr = [](int& i) { ++i; }; + + return belowHelper(node, stack, branch, {init, cmp, incr}); +} static const std::shared_ptr no_item; std::shared_ptr const& @@ -619,6 +644,43 @@ SHAMap::upper_bound(uint256 const& id) const } return end(); } +SHAMap::const_iterator +SHAMap::lower_bound(uint256 const& id) const +{ + SharedPtrNodeStack stack; + walkTowardsKey(id, &stack); + while (!stack.empty()) + { + auto [node, nodeID] = stack.top(); + if (node->isLeaf()) + { + auto leaf = static_cast(node.get()); + if (leaf->peekItem()->key() < id) + return const_iterator( + this, leaf->peekItem().get(), std::move(stack)); + } + else + { + auto inner = std::static_pointer_cast(node); + for (int branch = selectBranch(nodeID, id) - 1; branch >= 0; + --branch) + { + if (!inner->isEmptyBranch(branch)) + { + node = descendThrow(inner, branch); + auto leaf = lastBelow(node, stack, branch); + if (!leaf) + Throw(type_, id); + return const_iterator( + this, leaf->peekItem().get(), std::move(stack)); + } + } + } + stack.pop(); + } + // TODO: what to return here? + return end(); +} bool SHAMap::hasItem(uint256 const& id) const diff --git a/src/test/ledger/View_test.cpp b/src/test/ledger/View_test.cpp index 3d86ad227a..45c8f007e0 100644 --- a/src/test/ledger/View_test.cpp +++ b/src/test/ledger/View_test.cpp @@ -384,6 +384,273 @@ class View_test : public beast::unit_test::suite return std::vector({uint256(args)...}); } + void + testUpperAndLowerBound() + { + using namespace jtx; + Env env(*this); + Config config; + std::shared_ptr const genesis = std::make_shared( + create_genesis, + config, + std::vector{}, + env.app().getNodeFamily()); + auto const ledger = std::make_shared( + *genesis, env.app().timeKeeper().closeTime()); + + auto setup = [&ledger](std::vector const& vec) { + wipe(*ledger); + for (auto x : vec) + { + ledger->rawInsert(sle(x)); + } + }; + { + setup({1, 2, 3}); + BEAST_EXPECT(sles(*ledger) == list(1, 2, 3)); + auto e = ledger->stateMap().end(); + auto b1 = ledger->stateMap().begin(); + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(1)) == e); + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(2)) == b1); + ++b1; + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(3)) == b1); + ++b1; + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(4)) == b1); + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(5)) == b1); + b1 = ledger->stateMap().begin(); + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(0)) == b1); + ++b1; + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(1)) == b1); + ++b1; + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(2)) == b1); + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(3)) == e); + } + + { + setup({2, 4, 6}); + BEAST_EXPECT(sles(*ledger) == list(2, 4, 6)); + auto e = ledger->stateMap().end(); + auto b1 = ledger->stateMap().begin(); + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(1)) == e); + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(2)) == e); + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(3)) == b1); + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(4)) == b1); + ++b1; + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(5)) == b1); + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(6)) == b1); + ++b1; + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(7)) == b1); + b1 = ledger->stateMap().begin(); + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(1)) == b1); + ++b1; + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(2)) == b1); + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(3)) == b1); + ++b1; + + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(4)) == b1); + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(5)) == b1); + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(6)) == e); + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(7)) == e); + } + { + setup({2, 3, 5, 6, 10, 15}); + BEAST_EXPECT(sles(*ledger) == list(2, 3, 5, 6, 10, 15)); + auto e = ledger->stateMap().end(); + auto b = ledger->stateMap().begin(); + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(1)) == e); + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(2)) == e); + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(3)) == b); + ++b; + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(4)) == b); + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(5)) == b); + ++b; + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(6)) == b); + ++b; + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(7)) == b); + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(8)) == b); + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(9)) == b); + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(10)) == b); + ++b; + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(11)) == b); + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(12)) == b); + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(13)) == b); + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(14)) == b); + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(15)) == b); + ++b; + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(16)) == b); + b = ledger->stateMap().begin(); + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(0)) == b); + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(1)) == b); + ++b; + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(2)) == b); + ++b; + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(3)) == b); + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(4)) == b); + ++b; + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(5)) == b); + ++b; + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(6)) == b); + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(7)) == b); + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(8)) == b); + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(9)) == b); + ++b; + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(10)) == b); + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(11)) == b); + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(12)) == b); + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(13)) == b); + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(14)) == b); + ++b; + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(15)) == e); + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(16)) == e); + } + { + // some full trees, some empty trees, etc + setup({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16, 20, 25, 30, 32, 33, 34, 35, 36, 37, + 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 66, 100}); + BEAST_EXPECT( + sles(*ledger) == + list( + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 20, + 25, + 30, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 66, + 100)); + auto b = ledger->stateMap().begin(); + auto e = ledger->stateMap().end(); + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(0)) == e); + BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(1)) == b); + BEAST_EXPECT( + ledger->stateMap().lower_bound(uint256(5))->key() == + uint256(4)); + BEAST_EXPECT( + ledger->stateMap().lower_bound(uint256(15))->key() == + uint256(14)); + BEAST_EXPECT( + ledger->stateMap().lower_bound(uint256(16))->key() == + uint256(15)); + BEAST_EXPECT( + ledger->stateMap().lower_bound(uint256(19))->key() == + uint256(16)); + BEAST_EXPECT( + ledger->stateMap().lower_bound(uint256(20))->key() == + uint256(16)); + BEAST_EXPECT( + ledger->stateMap().lower_bound(uint256(24))->key() == + uint256(20)); + BEAST_EXPECT( + ledger->stateMap().lower_bound(uint256(31))->key() == + uint256(30)); + BEAST_EXPECT( + ledger->stateMap().lower_bound(uint256(32))->key() == + uint256(30)); + BEAST_EXPECT( + ledger->stateMap().lower_bound(uint256(40))->key() == + uint256(39)); + BEAST_EXPECT( + ledger->stateMap().lower_bound(uint256(47))->key() == + uint256(46)); + BEAST_EXPECT( + ledger->stateMap().lower_bound(uint256(48))->key() == + uint256(47)); + BEAST_EXPECT( + ledger->stateMap().lower_bound(uint256(64))->key() == + uint256(48)); + + BEAST_EXPECT( + ledger->stateMap().lower_bound(uint256(90))->key() == + uint256(66)); + BEAST_EXPECT( + ledger->stateMap().lower_bound(uint256(96))->key() == + uint256(66)); + BEAST_EXPECT( + ledger->stateMap().lower_bound(uint256(100))->key() == + uint256(66)); + + BEAST_EXPECT( + ledger->stateMap().upper_bound(uint256(0))->key() == + uint256(1)); + BEAST_EXPECT( + ledger->stateMap().upper_bound(uint256(5))->key() == + uint256(6)); + BEAST_EXPECT( + ledger->stateMap().upper_bound(uint256(15))->key() == + uint256(16)); + BEAST_EXPECT( + ledger->stateMap().upper_bound(uint256(16))->key() == + uint256(20)); + BEAST_EXPECT( + ledger->stateMap().upper_bound(uint256(18))->key() == + uint256(20)); + BEAST_EXPECT( + ledger->stateMap().upper_bound(uint256(20))->key() == + uint256(25)); + BEAST_EXPECT( + ledger->stateMap().upper_bound(uint256(31))->key() == + uint256(32)); + BEAST_EXPECT( + ledger->stateMap().upper_bound(uint256(32))->key() == + uint256(33)); + BEAST_EXPECT( + ledger->stateMap().upper_bound(uint256(47))->key() == + uint256(48)); + BEAST_EXPECT( + ledger->stateMap().upper_bound(uint256(48))->key() == + uint256(66)); + BEAST_EXPECT( + ledger->stateMap().upper_bound(uint256(53))->key() == + uint256(66)); + BEAST_EXPECT( + ledger->stateMap().upper_bound(uint256(66))->key() == + uint256(100)); + BEAST_EXPECT( + ledger->stateMap().upper_bound(uint256(70))->key() == + uint256(100)); + BEAST_EXPECT( + ledger->stateMap().upper_bound(uint256(85))->key() == + uint256(100)); + BEAST_EXPECT( + ledger->stateMap().upper_bound(uint256(98))->key() == + uint256(100)); + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(100)) == e); + BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(155)) == e); + } + } + void testSles() { @@ -811,6 +1078,7 @@ class View_test : public beast::unit_test::suite testStacked(); testContext(); testSles(); + testUpperAndLowerBound(); testFlags(); testTransferRate(); testAreCompatible(); diff --git a/src/test/rpc/ReportingETL_test.cpp b/src/test/rpc/ReportingETL_test.cpp index 8678e2b9be..36b2f9b0b6 100644 --- a/src/test/rpc/ReportingETL_test.cpp +++ b/src/test/rpc/ReportingETL_test.cpp @@ -70,20 +70,22 @@ class ReportingETL_test : public beast::unit_test::suite auto sequence, bool transactions, bool expand, - bool get_objects) { + bool get_objects, + bool get_object_neighbors) { GrpcLedgerClient grpcClient{grpcPort}; grpcClient.request.mutable_ledger()->set_sequence(sequence); grpcClient.request.set_transactions(transactions); grpcClient.request.set_expand(expand); grpcClient.request.set_get_objects(get_objects); + grpcClient.request.set_get_object_neighbors(get_object_neighbors); grpcClient.GetLedger(); return std::make_pair(grpcClient.status, grpcClient.reply); }; { - auto [status, reply] = grpcLedger(3, false, false, false); + auto [status, reply] = grpcLedger(3, false, false, false, false); BEAST_EXPECT(status.ok()); BEAST_EXPECT(reply.validated()); @@ -119,7 +121,7 @@ class ReportingETL_test : public beast::unit_test::suite addRaw(ledger->info(), s, true); { - auto [status, reply] = grpcLedger(4, true, false, false); + auto [status, reply] = grpcLedger(4, true, false, false, false); BEAST_EXPECT(status.ok()); BEAST_EXPECT(reply.validated()); BEAST_EXPECT(reply.has_hashes_list()); @@ -145,7 +147,7 @@ class ReportingETL_test : public beast::unit_test::suite } { - auto [status, reply] = grpcLedger(4, true, true, false); + auto [status, reply] = grpcLedger(4, true, true, false, false); BEAST_EXPECT(status.ok()); BEAST_EXPECT(reply.validated()); @@ -209,7 +211,7 @@ class ReportingETL_test : public beast::unit_test::suite } { - auto [status, reply] = grpcLedger(4, true, true, true); + auto [status, reply] = grpcLedger(4, true, true, true, false); BEAST_EXPECT(status.ok()); BEAST_EXPECT(reply.validated()); @@ -295,6 +297,112 @@ class ReportingETL_test : public beast::unit_test::suite ++idx; } } + { + auto [status, reply] = grpcLedger(4, true, true, true, true); + + BEAST_EXPECT(status.ok()); + BEAST_EXPECT(reply.validated()); + BEAST_EXPECT(!reply.has_hashes_list()); + BEAST_EXPECT(reply.object_neighbors_included()); + + BEAST_EXPECT(reply.has_transactions_list()); + BEAST_EXPECT(reply.transactions_list().transactions_size() == 4); + + BEAST_EXPECT( + makeSlice(reply.transactions_list() + .transactions(0) + .transaction_blob()) == + transactions[0]->getSerializer().slice()); + + BEAST_EXPECT( + makeSlice(reply.transactions_list() + .transactions(0) + .metadata_blob()) == + metas[0]->getSerializer().slice()); + + BEAST_EXPECT( + makeSlice(reply.transactions_list() + .transactions(1) + .transaction_blob()) == + transactions[1]->getSerializer().slice()); + + BEAST_EXPECT( + makeSlice(reply.transactions_list() + .transactions(1) + .metadata_blob()) == + metas[1]->getSerializer().slice()); + + BEAST_EXPECT( + makeSlice(reply.transactions_list() + .transactions(2) + .transaction_blob()) == + transactions[2]->getSerializer().slice()); + + BEAST_EXPECT( + makeSlice(reply.transactions_list() + .transactions(2) + .metadata_blob()) == + metas[2]->getSerializer().slice()); + + BEAST_EXPECT( + makeSlice(reply.transactions_list() + .transactions(3) + .transaction_blob()) == + transactions[3]->getSerializer().slice()); + + BEAST_EXPECT( + makeSlice(reply.transactions_list() + .transactions(3) + .metadata_blob()) == + metas[3]->getSerializer().slice()); + BEAST_EXPECT(reply.skiplist_included()); + + BEAST_EXPECT(s.slice() == makeSlice(reply.ledger_header())); + + auto parent = env.app().getLedgerMaster().getLedgerBySeq(3); + + SHAMap::Delta differences; + + int maxDifferences = std::numeric_limits::max(); + + bool res = parent->stateMap().compare( + ledger->stateMap(), differences, maxDifferences); + BEAST_EXPECT(res); + + size_t idx = 0; + + for (auto& [k, v] : differences) + { + auto obj = reply.ledger_objects().objects(idx); + BEAST_EXPECT(k == uint256::fromVoid(obj.key().data())); + if (v.second) + { + BEAST_EXPECT(v.second->slice() == makeSlice(obj.data())); + } + else + BEAST_EXPECT(obj.data().size() == 0); + + if (!(v.first && v.second)) + { + auto succ = ledger->stateMap().upper_bound(k); + auto pred = ledger->stateMap().lower_bound(k); + + if (succ != ledger->stateMap().end()) + BEAST_EXPECT( + succ->key() == + uint256::fromVoid(obj.successor().data())); + else + BEAST_EXPECT(obj.successor().size() == 0); + if (pred != ledger->stateMap().end()) + BEAST_EXPECT( + pred->key() == + uint256::fromVoid(obj.predecessor().data())); + else + BEAST_EXPECT(obj.predecessor().size() == 0); + } + ++idx; + } + } // Delete an account @@ -312,7 +420,7 @@ class ReportingETL_test : public beast::unit_test::suite { auto [status, reply] = - grpcLedger(env.closed()->seq(), true, true, true); + grpcLedger(env.closed()->seq(), true, true, true, true); BEAST_EXPECT(status.ok()); BEAST_EXPECT(reply.validated()); @@ -333,16 +441,34 @@ class ReportingETL_test : public beast::unit_test::suite size_t idx = 0; for (auto& [k, v] : differences) { - BEAST_EXPECT( - k == - uint256::fromVoid( - reply.ledger_objects().objects(idx).key().data())); + auto obj = reply.ledger_objects().objects(idx); + BEAST_EXPECT(k == uint256::fromVoid(obj.key().data())); if (v.second) { BEAST_EXPECT( v.second->slice() == makeSlice(reply.ledger_objects().objects(idx).data())); } + else + BEAST_EXPECT(obj.data().size() == 0); + if (!(v.first && v.second)) + { + auto succ = base->stateMap().upper_bound(k); + auto pred = base->stateMap().lower_bound(k); + + if (succ != base->stateMap().end()) + BEAST_EXPECT( + succ->key() == + uint256::fromVoid(obj.successor().data())); + else + BEAST_EXPECT(obj.successor().size() == 0); + if (pred != base->stateMap().end()) + BEAST_EXPECT( + pred->key() == + uint256::fromVoid(obj.predecessor().data())); + else + BEAST_EXPECT(obj.predecessor().size() == 0); + } ++idx; }