diff --git a/src/backend/BackendInterface.cpp b/src/backend/BackendInterface.cpp index f3e82beb..f22bab8f 100644 --- a/src/backend/BackendInterface.cpp +++ b/src/backend/BackendInterface.cpp @@ -250,7 +250,8 @@ BackendInterface::fetchLedgerPage( LedgerPage page; std::vector keys; - while (keys.size() < limit) + bool reachedEnd = false; + while (keys.size() < limit && !reachedEnd) { ripple::uint256 const& curCursor = keys.size() ? keys.back() : cursor ? *cursor @@ -258,15 +259,17 @@ BackendInterface::fetchLedgerPage( uint32_t seq = outOfOrder ? range->maxSequence : ledgerSequence; auto succ = fetchSuccessorKey(curCursor, seq, yield); if (!succ) - break; - BOOST_LOG_TRIVIAL(trace) << ripple::strHex(*succ); - keys.push_back(std::move(*succ)); + reachedEnd = true; + else + keys.push_back(std::move(*succ)); } auto objects = fetchLedgerObjects(keys, ledgerSequence, yield); for (size_t i = 0; i < objects.size(); ++i) { - if (!objects[i].size()) + if (objects[i].size()) + page.objects.push_back({std::move(keys[i]), std::move(objects[i])}); + else if (!outOfOrder) { BOOST_LOG_TRIVIAL(error) << __func__ << " incorrect successor table. key = " @@ -277,12 +280,11 @@ BackendInterface::fetchLedgerPage( msg << " - " << ripple::strHex(keys[j]); } BOOST_LOG_TRIVIAL(error) << __func__ << msg.str(); + assert(false); } - assert(objects[i].size()); - page.objects.push_back({std::move(keys[i]), std::move(objects[i])}); } - if (page.objects.size() >= limit) - page.cursor = page.objects.back().key; + if (!reachedEnd) + page.cursor = keys.back(); return page; } diff --git a/src/backend/CassandraBackend.cpp b/src/backend/CassandraBackend.cpp index 0782b19d..69835d12 100644 --- a/src/backend/CassandraBackend.cpp +++ b/src/backend/CassandraBackend.cpp @@ -758,8 +758,9 @@ CassandraBackend::doOnlineDelete( std::optional cursor; while (true) { - auto [objects, curCursor] = retryOnTimeout( - [&]() { return fetchLedgerPage(cursor, minLedger, 256, yield); }); + auto [objects, curCursor] = retryOnTimeout([&]() { + return fetchLedgerPage(cursor, minLedger, 256, false, yield); + }); for (auto& obj : objects) { diff --git a/src/backend/PostgresBackend.cpp b/src/backend/PostgresBackend.cpp index 630ffaa0..722b231a 100644 --- a/src/backend/PostgresBackend.cpp +++ b/src/backend/PostgresBackend.cpp @@ -812,8 +812,9 @@ PostgresBackend::doOnlineDelete( std::optional cursor; while (true) { - auto [objects, curCursor] = retryOnTimeout( - [&]() { return fetchLedgerPage(cursor, minLedger, 256, yield); }); + auto [objects, curCursor] = retryOnTimeout([&]() { + return fetchLedgerPage(cursor, minLedger, 256, false, yield); + }); BOOST_LOG_TRIVIAL(debug) << __func__ << " fetched a page"; std::stringstream objectsBuffer; diff --git a/src/rpc/handlers/LedgerData.cpp b/src/rpc/handlers/LedgerData.cpp index 064bc011..d4f44b82 100644 --- a/src/rpc/handlers/LedgerData.cpp +++ b/src/rpc/handlers/LedgerData.cpp @@ -69,12 +69,14 @@ doLedgerData(Context const& context) else return Status{Error::rpcINVALID_PARAMS, "markerNotString"}; } + else + { + BOOST_LOG_TRIVIAL(debug) << __func__ << " : parsing marker"; - BOOST_LOG_TRIVIAL(debug) << __func__ << " : parsing marker"; - - cursor = ripple::uint256{}; - if (!cursor->parseHex(request.at("marker").as_string().c_str())) - return Status{Error::rpcINVALID_PARAMS, "markerMalformed"}; + cursor = ripple::uint256{}; + if (!cursor->parseHex(request.at("marker").as_string().c_str())) + return Status{Error::rpcINVALID_PARAMS, "markerMalformed"}; + } } auto v = ledgerInfoFromRequest(context); @@ -163,8 +165,9 @@ doLedgerData(Context const& context) BOOST_LOG_TRIVIAL(debug) << __func__ << " number of results = " << results.size() - << " fetched in " << time << "microseconds"; + << " fetched in " << time << " microseconds"; boost::json::array objects; + objects.reserve(results.size()); for (auto const& [key, object] : results) { ripple::STLedgerEntry sle{ @@ -174,12 +177,19 @@ doLedgerData(Context const& context) boost::json::object entry; entry["data"] = ripple::serializeHex(sle); entry["index"] = ripple::to_string(sle.key()); - objects.push_back(entry); + objects.push_back(std::move(entry)); } else objects.push_back(toJson(sle)); } - response["state"] = objects; + response["state"] = std::move(objects); + auto end2 = std::chrono::system_clock::now(); + + time = std::chrono::duration_cast(end2 - end) + .count(); + BOOST_LOG_TRIVIAL(debug) + << __func__ << " number of results = " << results.size() + << " serialized in " << time << " microseconds"; return response; } diff --git a/test.py b/test.py index ffe161fe..3c940634 100755 --- a/test.py +++ b/test.py @@ -503,7 +503,7 @@ async def ledger_data_full(ip, port, ledger, binary, limit, typ=None, count=-1): else: - await ws.send(json.dumps({"command":"ledger_data","ledger_index":int(ledger),"cursor":marker, "marker":marker,"binary":bool(binary), "limit":int(limit)})) + await ws.send(json.dumps({"command":"ledger_data","ledger_index":int(ledger),"cursor":marker, "marker":marker,"binary":bool(binary), "limit":int(limit),"out_of_order":True})) res = json.loads(await ws.recv()) diff --git a/unittests/main.cpp b/unittests/main.cpp index c9b1bd4f..de434c0f 100644 --- a/unittests/main.cpp +++ b/unittests/main.cpp @@ -804,7 +804,7 @@ TEST(BackendTest, Basic) { uint32_t limit = 10; page = backend->fetchLedgerPage( - page.cursor, seq, limit, yield); + page.cursor, seq, limit, false, yield); std::cout << "fetched a page " << page.objects.size() << std::endl; if (page.cursor) @@ -2186,7 +2186,7 @@ TEST(Backend, cacheIntegration) { uint32_t limit = 10; page = backend->fetchLedgerPage( - page.cursor, seq, limit, yield); + page.cursor, seq, limit, false, yield); std::cout << "fetched a page " << page.objects.size() << std::endl; if (page.cursor)