Add successor information to clio ETL messages

* Allow clio to ask for object successors and predecessors from rippled
* Add lower_bound and last_below to SHAMap

CMakeLists.txt

@@ -5,7 +5,9 @@ if (POLICY CMP0074)
 endif ()
 
 project (rippled)
-set (CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 # make GIT_COMMIT_HASH define available to all sources
 find_package(Git)

src/ripple/proto/org/xrpl/rpc/v1/get_ledger.proto

@@ -32,6 +32,10 @@ message GetLedgerRequest
     // coming from a secure_gateway host, then the client is not subject to
     // resource controls
     string user = 6;
+
+    // For every object in the diff, get the object's predecessor and successor
+    // in the state map. Only used if get_objects is also true.
+    bool get_object_neighbors = 7;
 }
 
 message GetLedgerResponse
@@ -58,6 +62,18 @@ message GetLedgerResponse
 
     // True if request was exempt from resource controls
     bool is_unlimited = 7;
+
+    // True if the response contains the state map diff
+    bool objects_included = 8;
+
+    // True if the response contains key of objects adjacent to objects in state
+    // map diff
+    bool object_neighbors_included = 9;
+
+
+    // Successor information for book directories modified as part of this
+    // ledger
+    repeated BookSuccessor book_successors = 10;
 }
 
 message TransactionHashList

src/ripple/proto/org/xrpl/rpc/v1/ledger.proto

@@ -54,7 +54,14 @@ message RawLedgerObject
         DELETED = 3;
     }
 
+    // Whether the object was created, modified or deleted
     ModificationType mod_type = 3;
+
+    // Key of the object preceding this object in the desired ledger
+    bytes predecessor = 4;
+
+    // Key of the object succeeding this object in the desired ledger
+    bytes successor = 5;
 }
 
 message RawLedgerObjects
@@ -62,3 +69,17 @@ message RawLedgerObjects
     repeated RawLedgerObject objects = 1;
 }
 
+// Successor information for book directories. The book base is (usually) not
+// an actual object, yet we need to be able to ask for the successor to the
+// book base.
+message BookSuccessor {
+
+    // Base of the book in question
+    bytes book_base = 1;
+
+    // First book directory in the book. An empty value here means the entire
+    // book is deleted
+    bytes first_book = 2;
+
+};
+

src/ripple/rpc/handlers/LedgerHandler.cpp

@@ -105,6 +105,7 @@ LedgerHandler::check()
 std::pair<org::xrpl::rpc::v1::GetLedgerResponse, grpc::Status>
 doLedgerGrpc(RPC::GRPCContext<org::xrpl::rpc::v1::GetLedgerRequest>& context)
 {
+    auto begin = std::chrono::system_clock::now();
     org::xrpl::rpc::v1::GetLedgerRequest& request = context.params;
     org::xrpl::rpc::v1::GetLedgerResponse response;
     grpc::Status status = grpc::Status::OK;
@@ -212,13 +213,101 @@ doLedgerGrpc(RPC::GRPCContext<org::xrpl::rpc::v1::GetLedgerRequest>& context)
                 obj->set_mod_type(org::xrpl::rpc::v1::RawLedgerObject::DELETED);
             else
                 obj->set_mod_type(org::xrpl::rpc::v1::RawLedgerObject::CREATED);
+            auto const blob = inDesired ? inDesired->slice() : inBase->slice();
+            auto const objectType =
+                static_cast<LedgerEntryType>(blob[1] << 8 | blob[2]);
+
+            if (request.get_object_neighbors())
+            {
+                if (!(inBase && inDesired))
+                {
+                    auto lb = desired->stateMap().lower_bound(k);
+                    auto ub = desired->stateMap().upper_bound(k);
+                    if (lb != desired->stateMap().end())
+                        obj->set_predecessor(
+                            lb->key().data(), lb->key().size());
+                    if (ub != desired->stateMap().end())
+                        obj->set_successor(ub->key().data(), ub->key().size());
+                    if (objectType == ltDIR_NODE)
+                    {
+                        auto sle = std::make_shared<SLE>(SerialIter{blob}, k);
+                        if (!sle->isFieldPresent(sfOwner))
+                        {
+                            auto bookBase = keylet::quality({ltDIR_NODE, k}, 0);
+                            if (!inBase && inDesired)
+                            {
+                                auto firstBook =
+                                    desired->stateMap().upper_bound(
+                                        bookBase.key);
+                                if (firstBook != desired->stateMap().end() &&
+                                    firstBook->key() <
+                                        getQualityNext(bookBase.key) &&
+                                    firstBook->key() == k)
+                                {
+                                    auto succ = response.add_book_successors();
+                                    succ->set_book_base(
+                                        bookBase.key.data(),
+                                        bookBase.key.size());
+                                    succ->set_first_book(
+                                        firstBook->key().data(),
+                                        firstBook->key().size());
+                                }
+                            }
+                            if (inBase && !inDesired)
+                            {
+                                auto oldFirstBook =
+                                    base->stateMap().upper_bound(bookBase.key);
+                                if (oldFirstBook != base->stateMap().end() &&
+                                    oldFirstBook->key() <
+                                        getQualityNext(bookBase.key) &&
+                                    oldFirstBook->key() == k)
+                                {
+                                    auto succ = response.add_book_successors();
+                                    succ->set_book_base(
+                                        bookBase.key.data(),
+                                        bookBase.key.size());
+                                    auto newFirstBook =
+                                        desired->stateMap().upper_bound(
+                                            bookBase.key);
+
+                                    if (newFirstBook !=
+                                            desired->stateMap().end() &&
+                                        newFirstBook->key() <
+                                            getQualityNext(bookBase.key))
+                                    {
+                                        succ->set_first_book(
+                                            newFirstBook->key().data(),
+                                            newFirstBook->key().size());
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
         }
+        response.set_objects_included(true);
+        response.set_object_neighbors_included(request.get_object_neighbors());
         response.set_skiplist_included(true);
     }
 
     response.set_validated(
         RPC::isValidated(context.ledgerMaster, *ledger, context.app));
 
+    auto end = std::chrono::system_clock::now();
+    auto duration =
+        std::chrono::duration_cast<std::chrono::milliseconds>(end - begin)
+            .count() *
+        1.0;
+    JLOG(context.j.warn())
+        << __func__ << " - Extract time = " << duration
+        << " - num objects = " << response.ledger_objects().objects_size()
+        << " - num txns = " << response.transactions_list().transactions_size()
+        << " - ms per obj "
+        << duration / response.ledger_objects().objects_size()
+        << " - ms per txn "
+        << duration / response.transactions_list().transactions_size();
+
     return {response, status};
 }
 }  // namespace ripple

src/ripple/shamap/SHAMap.h

@@ -210,9 +210,17 @@ public:
     peekItem(uint256 const& id, SHAMapHash& hash) const;
 
     // traverse functions
+
+    // finds the object in the tree with the smallest object id greater than the
+    // input id
     const_iterator
     upper_bound(uint256 const& id) const;
 
+    // finds the object in the tree with the greatest object id smaller than the
+    // input id
+    const_iterator
+    lower_bound(uint256 const& id) const;
+
     /**  Visit every node in this SHAMap
 
          @param function called with every node visited.
@@ -400,12 +408,31 @@ private:
     std::shared_ptr<SHAMapTreeNode>
     writeNode(NodeObjectType t, std::shared_ptr<SHAMapTreeNode> node) const;
 
+    // returns the first item at or below this node
     SHAMapLeafNode*
     firstBelow(
         std::shared_ptr<SHAMapTreeNode>,
         SharedPtrNodeStack& stack,
         int branch = 0) const;
 
+    // returns the last item at or below this node
+    SHAMapLeafNode*
+    lastBelow(
+        std::shared_ptr<SHAMapTreeNode> node,
+        SharedPtrNodeStack& stack,
+        int branch = branchFactor) const;
+
+    // helper function for firstBelow and lastBelow
+    SHAMapLeafNode*
+    belowHelper(
+        std::shared_ptr<SHAMapTreeNode> node,
+        SharedPtrNodeStack& stack,
+        int branch,
+        std::tuple<
+            int,
+            std::function<bool(int)>,
+            std::function<void(int&)>> const& loopParams) const;
+
     // Simple descent
     // Get a child of the specified node
     SHAMapTreeNode*

src/ripple/shamap/impl/SHAMap.cpp

@@ -437,12 +437,14 @@ SHAMap::unshareNode(std::shared_ptr<Node> node, SHAMapNodeID const& nodeID)
 }
 
 SHAMapLeafNode*
-SHAMap::firstBelow(
+SHAMap::belowHelper(
     std::shared_ptr<SHAMapTreeNode> node,
     SharedPtrNodeStack& stack,
-    int branch) const
+    int branch,
+    std::tuple<int, std::function<bool(int)>, std::function<void(int&)>> const&
+        loopParams) const
 {
-    // Return the first item at or below this node
+    auto& [init, cmp, incr] = loopParams;
     if (node->isLeaf())
     {
         auto n = std::static_pointer_cast<SHAMapLeafNode>(node);
@@ -454,7 +456,7 @@ SHAMap::firstBelow(
         stack.push({inner, SHAMapNodeID{}});
     else
         stack.push({inner, stack.top().second.getChildNodeID(branch)});
-    for (int i = 0; i < branchFactor;)
+    for (int i = init; cmp(i);)
     {
         if (!inner->isEmptyBranch(i))
         {
@@ -468,14 +470,37 @@ SHAMap::firstBelow(
             }
             inner = std::static_pointer_cast<SHAMapInnerNode>(node);
             stack.push({inner, stack.top().second.getChildNodeID(branch)});
-            i = 0;  // scan all 16 branches of this new node
+            i = init;  // descend and reset loop
         }
         else
-            ++i;  // scan next branch
+            incr(i);  // scan next branch
     }
     return nullptr;
 }
+SHAMapLeafNode*
+SHAMap::lastBelow(
+    std::shared_ptr<SHAMapTreeNode> node,
+    SharedPtrNodeStack& stack,
+    int branch) const
+{
+    auto init = branchFactor - 1;
+    auto cmp = [](int i) { return i >= 0; };
+    auto incr = [](int& i) { --i; };
 
+    return belowHelper(node, stack, branch, {init, cmp, incr});
+}
+SHAMapLeafNode*
+SHAMap::firstBelow(
+    std::shared_ptr<SHAMapTreeNode> node,
+    SharedPtrNodeStack& stack,
+    int branch) const
+{
+    auto init = 0;
+    auto cmp = [](int i) { return i <= branchFactor; };
+    auto incr = [](int& i) { ++i; };
+
+    return belowHelper(node, stack, branch, {init, cmp, incr});
+}
 static const std::shared_ptr<SHAMapItem const> no_item;
 
 std::shared_ptr<SHAMapItem const> const&
@@ -619,6 +644,43 @@ SHAMap::upper_bound(uint256 const& id) const
     }
     return end();
 }
+SHAMap::const_iterator
+SHAMap::lower_bound(uint256 const& id) const
+{
+    SharedPtrNodeStack stack;
+    walkTowardsKey(id, &stack);
+    while (!stack.empty())
+    {
+        auto [node, nodeID] = stack.top();
+        if (node->isLeaf())
+        {
+            auto leaf = static_cast<SHAMapLeafNode*>(node.get());
+            if (leaf->peekItem()->key() < id)
+                return const_iterator(
+                    this, leaf->peekItem().get(), std::move(stack));
+        }
+        else
+        {
+            auto inner = std::static_pointer_cast<SHAMapInnerNode>(node);
+            for (int branch = selectBranch(nodeID, id) - 1; branch >= 0;
+                 --branch)
+            {
+                if (!inner->isEmptyBranch(branch))
+                {
+                    node = descendThrow(inner, branch);
+                    auto leaf = lastBelow(node, stack, branch);
+                    if (!leaf)
+                        Throw<SHAMapMissingNode>(type_, id);
+                    return const_iterator(
+                        this, leaf->peekItem().get(), std::move(stack));
+                }
+            }
+        }
+        stack.pop();
+    }
+    // TODO: what to return here?
+    return end();
+}
 
 bool
 SHAMap::hasItem(uint256 const& id) const

src/test/ledger/View_test.cpp

@@ -384,6 +384,273 @@ class View_test : public beast::unit_test::suite
         return std::vector<uint256>({uint256(args)...});
     }
 
+    void
+    testUpperAndLowerBound()
+    {
+        using namespace jtx;
+        Env env(*this);
+        Config config;
+        std::shared_ptr<Ledger const> const genesis = std::make_shared<Ledger>(
+            create_genesis,
+            config,
+            std::vector<uint256>{},
+            env.app().getNodeFamily());
+        auto const ledger = std::make_shared<Ledger>(
+            *genesis, env.app().timeKeeper().closeTime());
+
+        auto setup = [&ledger](std::vector<int> const& vec) {
+            wipe(*ledger);
+            for (auto x : vec)
+            {
+                ledger->rawInsert(sle(x));
+            }
+        };
+        {
+            setup({1, 2, 3});
+            BEAST_EXPECT(sles(*ledger) == list(1, 2, 3));
+            auto e = ledger->stateMap().end();
+            auto b1 = ledger->stateMap().begin();
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(1)) == e);
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(2)) == b1);
+            ++b1;
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(3)) == b1);
+            ++b1;
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(4)) == b1);
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(5)) == b1);
+            b1 = ledger->stateMap().begin();
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(0)) == b1);
+            ++b1;
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(1)) == b1);
+            ++b1;
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(2)) == b1);
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(3)) == e);
+        }
+
+        {
+            setup({2, 4, 6});
+            BEAST_EXPECT(sles(*ledger) == list(2, 4, 6));
+            auto e = ledger->stateMap().end();
+            auto b1 = ledger->stateMap().begin();
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(1)) == e);
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(2)) == e);
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(3)) == b1);
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(4)) == b1);
+            ++b1;
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(5)) == b1);
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(6)) == b1);
+            ++b1;
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(7)) == b1);
+            b1 = ledger->stateMap().begin();
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(1)) == b1);
+            ++b1;
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(2)) == b1);
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(3)) == b1);
+            ++b1;
+
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(4)) == b1);
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(5)) == b1);
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(6)) == e);
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(7)) == e);
+        }
+        {
+            setup({2, 3, 5, 6, 10, 15});
+            BEAST_EXPECT(sles(*ledger) == list(2, 3, 5, 6, 10, 15));
+            auto e = ledger->stateMap().end();
+            auto b = ledger->stateMap().begin();
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(1)) == e);
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(2)) == e);
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(3)) == b);
+            ++b;
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(4)) == b);
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(5)) == b);
+            ++b;
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(6)) == b);
+            ++b;
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(7)) == b);
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(8)) == b);
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(9)) == b);
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(10)) == b);
+            ++b;
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(11)) == b);
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(12)) == b);
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(13)) == b);
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(14)) == b);
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(15)) == b);
+            ++b;
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(16)) == b);
+            b = ledger->stateMap().begin();
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(0)) == b);
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(1)) == b);
+            ++b;
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(2)) == b);
+            ++b;
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(3)) == b);
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(4)) == b);
+            ++b;
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(5)) == b);
+            ++b;
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(6)) == b);
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(7)) == b);
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(8)) == b);
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(9)) == b);
+            ++b;
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(10)) == b);
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(11)) == b);
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(12)) == b);
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(13)) == b);
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(14)) == b);
+            ++b;
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(15)) == e);
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(16)) == e);
+        }
+        {
+            // some full trees, some empty trees, etc
+            setup({0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+                   13, 14, 15, 16, 20, 25, 30, 32, 33, 34, 35, 36, 37,
+                   38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 66, 100});
+            BEAST_EXPECT(
+                sles(*ledger) ==
+                list(
+                    0,
+                    1,
+                    2,
+                    3,
+                    4,
+                    5,
+                    6,
+                    7,
+                    8,
+                    9,
+                    10,
+                    11,
+                    12,
+                    13,
+                    14,
+                    15,
+                    16,
+                    20,
+                    25,
+                    30,
+                    32,
+                    33,
+                    34,
+                    35,
+                    36,
+                    37,
+                    38,
+                    39,
+                    40,
+                    41,
+                    42,
+                    43,
+                    44,
+                    45,
+                    46,
+                    47,
+                    48,
+                    66,
+                    100));
+            auto b = ledger->stateMap().begin();
+            auto e = ledger->stateMap().end();
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(0)) == e);
+            BEAST_EXPECT(ledger->stateMap().lower_bound(uint256(1)) == b);
+            BEAST_EXPECT(
+                ledger->stateMap().lower_bound(uint256(5))->key() ==
+                uint256(4));
+            BEAST_EXPECT(
+                ledger->stateMap().lower_bound(uint256(15))->key() ==
+                uint256(14));
+            BEAST_EXPECT(
+                ledger->stateMap().lower_bound(uint256(16))->key() ==
+                uint256(15));
+            BEAST_EXPECT(
+                ledger->stateMap().lower_bound(uint256(19))->key() ==
+                uint256(16));
+            BEAST_EXPECT(
+                ledger->stateMap().lower_bound(uint256(20))->key() ==
+                uint256(16));
+            BEAST_EXPECT(
+                ledger->stateMap().lower_bound(uint256(24))->key() ==
+                uint256(20));
+            BEAST_EXPECT(
+                ledger->stateMap().lower_bound(uint256(31))->key() ==
+                uint256(30));
+            BEAST_EXPECT(
+                ledger->stateMap().lower_bound(uint256(32))->key() ==
+                uint256(30));
+            BEAST_EXPECT(
+                ledger->stateMap().lower_bound(uint256(40))->key() ==
+                uint256(39));
+            BEAST_EXPECT(
+                ledger->stateMap().lower_bound(uint256(47))->key() ==
+                uint256(46));
+            BEAST_EXPECT(
+                ledger->stateMap().lower_bound(uint256(48))->key() ==
+                uint256(47));
+            BEAST_EXPECT(
+                ledger->stateMap().lower_bound(uint256(64))->key() ==
+                uint256(48));
+
+            BEAST_EXPECT(
+                ledger->stateMap().lower_bound(uint256(90))->key() ==
+                uint256(66));
+            BEAST_EXPECT(
+                ledger->stateMap().lower_bound(uint256(96))->key() ==
+                uint256(66));
+            BEAST_EXPECT(
+                ledger->stateMap().lower_bound(uint256(100))->key() ==
+                uint256(66));
+
+            BEAST_EXPECT(
+                ledger->stateMap().upper_bound(uint256(0))->key() ==
+                uint256(1));
+            BEAST_EXPECT(
+                ledger->stateMap().upper_bound(uint256(5))->key() ==
+                uint256(6));
+            BEAST_EXPECT(
+                ledger->stateMap().upper_bound(uint256(15))->key() ==
+                uint256(16));
+            BEAST_EXPECT(
+                ledger->stateMap().upper_bound(uint256(16))->key() ==
+                uint256(20));
+            BEAST_EXPECT(
+                ledger->stateMap().upper_bound(uint256(18))->key() ==
+                uint256(20));
+            BEAST_EXPECT(
+                ledger->stateMap().upper_bound(uint256(20))->key() ==
+                uint256(25));
+            BEAST_EXPECT(
+                ledger->stateMap().upper_bound(uint256(31))->key() ==
+                uint256(32));
+            BEAST_EXPECT(
+                ledger->stateMap().upper_bound(uint256(32))->key() ==
+                uint256(33));
+            BEAST_EXPECT(
+                ledger->stateMap().upper_bound(uint256(47))->key() ==
+                uint256(48));
+            BEAST_EXPECT(
+                ledger->stateMap().upper_bound(uint256(48))->key() ==
+                uint256(66));
+            BEAST_EXPECT(
+                ledger->stateMap().upper_bound(uint256(53))->key() ==
+                uint256(66));
+            BEAST_EXPECT(
+                ledger->stateMap().upper_bound(uint256(66))->key() ==
+                uint256(100));
+            BEAST_EXPECT(
+                ledger->stateMap().upper_bound(uint256(70))->key() ==
+                uint256(100));
+            BEAST_EXPECT(
+                ledger->stateMap().upper_bound(uint256(85))->key() ==
+                uint256(100));
+            BEAST_EXPECT(
+                ledger->stateMap().upper_bound(uint256(98))->key() ==
+                uint256(100));
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(100)) == e);
+            BEAST_EXPECT(ledger->stateMap().upper_bound(uint256(155)) == e);
+        }
+    }
+
     void
     testSles()
     {
@@ -811,6 +1078,7 @@ class View_test : public beast::unit_test::suite
         testStacked();
         testContext();
         testSles();
+        testUpperAndLowerBound();
         testFlags();
         testTransferRate();
         testAreCompatible();

src/test/rpc/ReportingETL_test.cpp

@@ -70,20 +70,22 @@ class ReportingETL_test : public beast::unit_test::suite
                               auto sequence,
                               bool transactions,
                               bool expand,
-                              bool get_objects) {
+                              bool get_objects,
+                              bool get_object_neighbors) {
             GrpcLedgerClient grpcClient{grpcPort};
 
             grpcClient.request.mutable_ledger()->set_sequence(sequence);
             grpcClient.request.set_transactions(transactions);
             grpcClient.request.set_expand(expand);
             grpcClient.request.set_get_objects(get_objects);
+            grpcClient.request.set_get_object_neighbors(get_object_neighbors);
 
             grpcClient.GetLedger();
             return std::make_pair(grpcClient.status, grpcClient.reply);
         };
 
         {
-            auto [status, reply] = grpcLedger(3, false, false, false);
+            auto [status, reply] = grpcLedger(3, false, false, false, false);
 
             BEAST_EXPECT(status.ok());
             BEAST_EXPECT(reply.validated());
@@ -119,7 +121,7 @@ class ReportingETL_test : public beast::unit_test::suite
         addRaw(ledger->info(), s, true);
 
         {
-            auto [status, reply] = grpcLedger(4, true, false, false);
+            auto [status, reply] = grpcLedger(4, true, false, false, false);
             BEAST_EXPECT(status.ok());
             BEAST_EXPECT(reply.validated());
             BEAST_EXPECT(reply.has_hashes_list());
@@ -145,7 +147,7 @@ class ReportingETL_test : public beast::unit_test::suite
         }
 
         {
-            auto [status, reply] = grpcLedger(4, true, true, false);
+            auto [status, reply] = grpcLedger(4, true, true, false, false);
 
             BEAST_EXPECT(status.ok());
             BEAST_EXPECT(reply.validated());
@@ -209,7 +211,7 @@ class ReportingETL_test : public beast::unit_test::suite
         }
 
         {
-            auto [status, reply] = grpcLedger(4, true, true, true);
+            auto [status, reply] = grpcLedger(4, true, true, true, false);
 
             BEAST_EXPECT(status.ok());
             BEAST_EXPECT(reply.validated());
@@ -295,6 +297,112 @@ class ReportingETL_test : public beast::unit_test::suite
                 ++idx;
             }
         }
+        {
+            auto [status, reply] = grpcLedger(4, true, true, true, true);
+
+            BEAST_EXPECT(status.ok());
+            BEAST_EXPECT(reply.validated());
+            BEAST_EXPECT(!reply.has_hashes_list());
+            BEAST_EXPECT(reply.object_neighbors_included());
+
+            BEAST_EXPECT(reply.has_transactions_list());
+            BEAST_EXPECT(reply.transactions_list().transactions_size() == 4);
+
+            BEAST_EXPECT(
+                makeSlice(reply.transactions_list()
+                              .transactions(0)
+                              .transaction_blob()) ==
+                transactions[0]->getSerializer().slice());
+
+            BEAST_EXPECT(
+                makeSlice(reply.transactions_list()
+                              .transactions(0)
+                              .metadata_blob()) ==
+                metas[0]->getSerializer().slice());
+
+            BEAST_EXPECT(
+                makeSlice(reply.transactions_list()
+                              .transactions(1)
+                              .transaction_blob()) ==
+                transactions[1]->getSerializer().slice());
+
+            BEAST_EXPECT(
+                makeSlice(reply.transactions_list()
+                              .transactions(1)
+                              .metadata_blob()) ==
+                metas[1]->getSerializer().slice());
+
+            BEAST_EXPECT(
+                makeSlice(reply.transactions_list()
+                              .transactions(2)
+                              .transaction_blob()) ==
+                transactions[2]->getSerializer().slice());
+
+            BEAST_EXPECT(
+                makeSlice(reply.transactions_list()
+                              .transactions(2)
+                              .metadata_blob()) ==
+                metas[2]->getSerializer().slice());
+
+            BEAST_EXPECT(
+                makeSlice(reply.transactions_list()
+                              .transactions(3)
+                              .transaction_blob()) ==
+                transactions[3]->getSerializer().slice());
+
+            BEAST_EXPECT(
+                makeSlice(reply.transactions_list()
+                              .transactions(3)
+                              .metadata_blob()) ==
+                metas[3]->getSerializer().slice());
+            BEAST_EXPECT(reply.skiplist_included());
+
+            BEAST_EXPECT(s.slice() == makeSlice(reply.ledger_header()));
+
+            auto parent = env.app().getLedgerMaster().getLedgerBySeq(3);
+
+            SHAMap::Delta differences;
+
+            int maxDifferences = std::numeric_limits<int>::max();
+
+            bool res = parent->stateMap().compare(
+                ledger->stateMap(), differences, maxDifferences);
+            BEAST_EXPECT(res);
+
+            size_t idx = 0;
+
+            for (auto& [k, v] : differences)
+            {
+                auto obj = reply.ledger_objects().objects(idx);
+                BEAST_EXPECT(k == uint256::fromVoid(obj.key().data()));
+                if (v.second)
+                {
+                    BEAST_EXPECT(v.second->slice() == makeSlice(obj.data()));
+                }
+                else
+                    BEAST_EXPECT(obj.data().size() == 0);
+
+                if (!(v.first && v.second))
+                {
+                    auto succ = ledger->stateMap().upper_bound(k);
+                    auto pred = ledger->stateMap().lower_bound(k);
+
+                    if (succ != ledger->stateMap().end())
+                        BEAST_EXPECT(
+                            succ->key() ==
+                            uint256::fromVoid(obj.successor().data()));
+                    else
+                        BEAST_EXPECT(obj.successor().size() == 0);
+                    if (pred != ledger->stateMap().end())
+                        BEAST_EXPECT(
+                            pred->key() ==
+                            uint256::fromVoid(obj.predecessor().data()));
+                    else
+                        BEAST_EXPECT(obj.predecessor().size() == 0);
+                }
+                ++idx;
+            }
+        }
 
         // Delete an account
 
@@ -312,7 +420,7 @@ class ReportingETL_test : public beast::unit_test::suite
 
         {
             auto [status, reply] =
-                grpcLedger(env.closed()->seq(), true, true, true);
+                grpcLedger(env.closed()->seq(), true, true, true, true);
 
             BEAST_EXPECT(status.ok());
             BEAST_EXPECT(reply.validated());
@@ -333,16 +441,34 @@ class ReportingETL_test : public beast::unit_test::suite
             size_t idx = 0;
             for (auto& [k, v] : differences)
             {
-                BEAST_EXPECT(
-                    k ==
-                    uint256::fromVoid(
-                        reply.ledger_objects().objects(idx).key().data()));
+                auto obj = reply.ledger_objects().objects(idx);
+                BEAST_EXPECT(k == uint256::fromVoid(obj.key().data()));
                 if (v.second)
                 {
                     BEAST_EXPECT(
                         v.second->slice() ==
                         makeSlice(reply.ledger_objects().objects(idx).data()));
                 }
+                else
+                    BEAST_EXPECT(obj.data().size() == 0);
+                if (!(v.first && v.second))
+                {
+                    auto succ = base->stateMap().upper_bound(k);
+                    auto pred = base->stateMap().lower_bound(k);
+
+                    if (succ != base->stateMap().end())
+                        BEAST_EXPECT(
+                            succ->key() ==
+                            uint256::fromVoid(obj.successor().data()));
+                    else
+                        BEAST_EXPECT(obj.successor().size() == 0);
+                    if (pred != base->stateMap().end())
+                        BEAST_EXPECT(
+                            pred->key() ==
+                            uint256::fromVoid(obj.predecessor().data()));
+                    else
+                        BEAST_EXPECT(obj.predecessor().size() == 0);
+                }
 
                 ++idx;
             }