From 7c6979ae576fe7a5a140cd86ee65e0c3e3d12920 Mon Sep 17 00:00:00 2001
From: CJ Cobb <ccobb@ripple.com>
Date: Thu, 10 Feb 2022 16:33:14 +0000
Subject: [PATCH] optimized ledger_data

---
 src/backend/BackendInterface.cpp |  5 +-
 src/backend/BackendInterface.h   |  1 +
 src/rpc/handlers/LedgerData.cpp  | 81 ++++++++++++++++++++++++--------
 3 files changed, 66 insertions(+), 21 deletions(-)

diff --git a/src/backend/BackendInterface.cpp b/src/backend/BackendInterface.cpp
index 5b816c01..f3e82beb 100644
--- a/src/backend/BackendInterface.cpp
+++ b/src/backend/BackendInterface.cpp
@@ -244,6 +244,7 @@ BackendInterface::fetchLedgerPage(
     std::optional<ripple::uint256> const& cursor,
     std::uint32_t const ledgerSequence,
     std::uint32_t const limit,
+    bool outOfOrder,
     boost::asio::yield_context& yield) const
 {
     LedgerPage page;
@@ -254,8 +255,8 @@ BackendInterface::fetchLedgerPage(
         ripple::uint256 const& curCursor = keys.size() ? keys.back()
             : cursor                                   ? *cursor
                                                        : firstKey;
-        auto succ = fetchSuccessorKey(curCursor, ledgerSequence, yield);
-
+        uint32_t seq = outOfOrder ? range->maxSequence : ledgerSequence;
+        auto succ = fetchSuccessorKey(curCursor, seq, yield);
         if (!succ)
             break;
         BOOST_LOG_TRIVIAL(trace) << ripple::strHex(*succ);
diff --git a/src/backend/BackendInterface.h b/src/backend/BackendInterface.h
index 37302e9a..a675491c 100644
--- a/src/backend/BackendInterface.h
+++ b/src/backend/BackendInterface.h
@@ -217,6 +217,7 @@ public:
         std::optional<ripple::uint256> const& cursor,
         std::uint32_t const ledgerSequence,
         std::uint32_t const limit,
+        bool outOfOrder,
         boost::asio::yield_context& yield) const;
 
     // Fetches the successor to key/index
diff --git a/src/rpc/handlers/LedgerData.cpp b/src/rpc/handlers/LedgerData.cpp
index 15021d76..2b878e9c 100644
--- a/src/rpc/handlers/LedgerData.cpp
+++ b/src/rpc/handlers/LedgerData.cpp
@@ -45,12 +45,30 @@ doLedgerData(Context const& context)
 
         limit = boost::json::value_to<int>(request.at("limit"));
     }
+    bool outOfOrder = false;
+    if (request.contains("out_of_order"))
+    {
+        if (!request.at("out_of_order").is_bool())
+            return Status{Error::rpcINVALID_PARAMS, "binaryFlagNotBool"};
+        outOfOrder = request.at("out_of_order").as_bool();
+    }
 
     std::optional<ripple::uint256> cursor;
+    std::optional<uint32_t> diffCursor;
     if (request.contains("marker"))
     {
         if (!request.at("marker").is_string())
-            return Status{Error::rpcINVALID_PARAMS, "markerNotString"};
+        {
+            if (outOfOrder)
+            {
+                if (!request.at("marker").is_int64())
+                    return Status{
+                        Error::rpcINVALID_PARAMS, "markerNotStringOrInt"};
+                diffCursor = value_to<uint32_t>(request.at("marker"));
+            }
+            else
+                return Status{Error::rpcINVALID_PARAMS, "markerNotString"};
+        }
 
         BOOST_LOG_TRIVIAL(debug) << __func__ << " : parsing marker";
 
@@ -64,15 +82,8 @@ doLedgerData(Context const& context)
         return *status;
 
     auto lgrInfo = std::get<ripple::LedgerInfo>(v);
-
-    Backend::LedgerPage page;
-    auto start = std::chrono::system_clock::now();
-    page = context.backend->fetchLedgerPage(
-        cursor, lgrInfo.seq, limit, context.yield);
-
-    auto end = std::chrono::system_clock::now();
-
     boost::json::object header;
+    // no cursor means this is the first call, so we return header info
     if (!cursor)
     {
         if (binary)
@@ -104,23 +115,55 @@ doLedgerData(Context const& context)
             response["ledger"] = header;
         }
     }
-
     response["ledger_hash"] = ripple::strHex(lgrInfo.hash);
     response["ledger_index"] = lgrInfo.seq;
 
-    boost::json::array objects;
-    std::vector<Backend::LedgerObject>& results = page.objects;
-    std::optional<ripple::uint256> const& returnedCursor = page.cursor;
-
-    if (returnedCursor)
+    auto start = std::chrono::system_clock::now();
+    std::vector<Backend::LedgerObject> results;
+    if (diffCursor)
     {
-        response["marker"] = ripple::strHex(*returnedCursor);
-        BOOST_LOG_TRIVIAL(debug)
-            << __func__ << " cursor = " << ripple::strHex(*returnedCursor);
+        assert(outOfOrder);
+        auto diff =
+            context.backend->fetchLedgerDiff(*diffCursor, context.yield);
+        std::vector<ripple::uint256> keys;
+        for (auto&& [key, object] : diff)
+        {
+            if (!object.size())
+            {
+                keys.push_back(std::move(key));
+            }
+        }
+        auto objs = context.backend->fetchLedgerObjects(
+            keys, lgrInfo.seq, context.yield);
+        for (size_t i = 0; i < objs.size(); ++i)
+        {
+            auto&& obj = objs[i];
+            results.push_back({std::move(keys[i]), std::move(obj)});
+        }
+        if (*diffCursor > lgrInfo.seq)
+            response["marker"] = *diffCursor - 1;
     }
+    else
+    {
+        auto page = context.backend->fetchLedgerPage(
+            cursor, lgrInfo.seq, limit, outOfOrder, context.yield);
+        results = std::move(page.objects);
+        if (page.cursor)
+            response["marker"] = ripple::strHex(*(page.cursor));
+        else if (outOfOrder)
+            response["marker"] =
+                context.backend->fetchLedgerRange()->maxSequence;
+    }
+    auto end = std::chrono::system_clock::now();
+
+    auto time =
+        std::chrono::duration_cast<std::chrono::microseconds>(end - start)
+            .count();
 
     BOOST_LOG_TRIVIAL(debug)
-        << __func__ << " number of results = " << results.size();
+        << __func__ << " number of results = " << results.size()
+        << " fetched in " << time << "microseconds";
+    boost::json::array objects;
     for (auto const& [key, object] : results)
     {
         ripple::STLedgerEntry sle{