Make Clio RPCs more consistent with rippled (#110)

* parse ledger_index as number

* allow websocket to use "command" or "method"

* mark all non-forwarded responses as validated

* dont mark forwarded errors as successful

* reduce forwarding Websocket expiration 30->3 seconds

* fix merge conflict in test.py

* adds ledger_current and ledger_closed

* deserialize `taker_gets_funded` into amount json

* limit account RPCs by number of objects traversed

* assign result correctly

src/etl/ETLSource.cpp

@@ -1055,7 +1055,7 @@ ETLSourceImpl<Derived>::forwardToRippled(
         if (ec)
             return {};
 
-        ws->next_layer().expires_after(std::chrono::seconds(30));
+        ws->next_layer().expires_after(std::chrono::seconds(3));
 
         BOOST_LOG_TRIVIAL(debug) << "Connecting websocket";
         // Make the connection on the IP address we get from a lookup

src/rpc/Counters.cpp

@@ -1,4 +1,5 @@
 #include <rpc/Counters.h>
+#include <rpc/RPC.h>
 
 namespace RPC {
 
@@ -19,6 +20,9 @@ Counters::initializeCounter(std::string const& method)
 void
 Counters::rpcErrored(std::string const& method)
 {
+    if (!validHandler(method))
+        return;
+
     initializeCounter(method);
 
     std::shared_lock lk(mutex_);
@@ -32,6 +36,9 @@ Counters::rpcComplete(
     std::string const& method,
     std::chrono::microseconds const& rpcDuration)
 {
+    if (!validHandler(method))
+        return;
+
     initializeCounter(method);
 
     std::shared_lock lk(mutex_);
@@ -44,6 +51,9 @@ Counters::rpcComplete(
 void
 Counters::rpcForwarded(std::string const& method)
 {
+    if (!validHandler(method))
+        return;
+
     initializeCounter(method);
 
     std::shared_lock lk(mutex_);

src/rpc/RPC.cpp

@@ -18,10 +18,16 @@ make_WsContext(
     Counters& counters,
     std::string const& clientIp)
 {
-    if (!request.contains("command"))
+    boost::json::value commandValue = nullptr;
+    if (!request.contains("command") && request.contains("method"))
+        commandValue = request.at("method");
+    else if (request.contains("command") && !request.contains("method"))
+        commandValue = request.at("command");
+
+    if (!commandValue.is_string())
         return {};
 
-    std::string command = request.at("command").as_string().c_str();
+    std::string command = commandValue.as_string().c_str();
 
     return Context{
         yc,
@@ -142,10 +148,17 @@ static std::unordered_set<std::string> forwardCommands{
     "submit",
     "submit_multisigned",
     "fee",
-    "path_find",
+    "ledger_closed",
+    "ledger_current",
     "ripple_path_find",
     "manifest"};
 
+bool
+validHandler(std::string const& method)
+{
+    return handlerTable.contains(method) || forwardCommands.contains(method);
+}
+
 bool
 shouldForwardToRippled(Context const& ctx)
 {
@@ -203,7 +216,12 @@ buildResponse(Context const& ctx)
 
     try
     {
-        return method(ctx);
+        auto v = method(ctx);
+
+        if (auto object = std::get_if<boost::json::object>(&v))
+            (*object)["validated"] = true;
+
+        return v;
     }
     catch (InvalidParamsError const& err)
     {

src/rpc/RPC.h

@@ -75,6 +75,24 @@ struct Context
 };
 using Error = ripple::error_code_i;
 
+struct AccountCursor
+{
+    ripple::uint256 index;
+    std::uint32_t hint;
+
+    std::string
+    toString()
+    {
+        return ripple::strHex(index) + "," + std::to_string(hint);
+    }
+
+    bool
+    isNonZero()
+    {
+        return index.isNonZero() || hint != 0;
+    }
+};
+
 struct Status
 {
     Error error = Error::rpcSUCCESS;
@@ -169,6 +187,9 @@ make_HttpContext(
 Result
 buildResponse(Context const& ctx);
 
+bool
+validHandler(std::string const& method);
+
 }  // namespace RPC
 
 #endif  // REPORTING_RPC_H_INCLUDED

src/rpc/RPCHelpers.cpp

@@ -67,6 +67,81 @@ getRequiredUInt(boost::json::object const& request, std::string const& field)
         throw InvalidParamsError("Missing field " + field);
 }
 
+bool
+isOwnedByAccount(ripple::SLE const& sle, ripple::AccountID const& accountID)
+{
+    if (sle.getType() == ripple::ltRIPPLE_STATE)
+    {
+        return (sle.getFieldAmount(ripple::sfLowLimit).getIssuer() ==
+                accountID) ||
+            (sle.getFieldAmount(ripple::sfHighLimit).getIssuer() == accountID);
+    }
+    else if (sle.isFieldPresent(ripple::sfAccount))
+    {
+        return sle.getAccountID(ripple::sfAccount) == accountID;
+    }
+    else if (sle.getType() == ripple::ltSIGNER_LIST)
+    {
+        ripple::Keylet const accountSignerList =
+            ripple::keylet::signers(accountID);
+        return sle.key() == accountSignerList.key;
+    }
+
+    return false;
+}
+
+std::optional<AccountCursor>
+parseAccountCursor(
+    BackendInterface const& backend,
+    std::uint32_t seq,
+    std::optional<std::string> jsonCursor,
+    ripple::AccountID const& accountID,
+    boost::asio::yield_context& yield)
+{
+    ripple::uint256 cursorIndex = beast::zero;
+    std::uint64_t startHint = 0;
+
+    if (!jsonCursor)
+        return AccountCursor({cursorIndex, startHint});
+
+    // Cursor is composed of a comma separated index and start hint. The
+    // former will be read as hex, and the latter using boost lexical cast.
+    std::stringstream cursor(*jsonCursor);
+    std::string value;
+    if (!std::getline(cursor, value, ','))
+        return {};
+
+    if (!cursorIndex.parseHex(value))
+        return {};
+
+    if (!std::getline(cursor, value, ','))
+        return {};
+
+    try
+    {
+        startHint = boost::lexical_cast<std::uint64_t>(value);
+    }
+    catch (boost::bad_lexical_cast&)
+    {
+        return {};
+    }
+
+    // We then must check if the object pointed to by the marker is actually
+    // owned by the account in the request.
+    auto const ownedNode = backend.fetchLedgerObject(cursorIndex, seq, yield);
+
+    if (!ownedNode)
+        return {};
+
+    ripple::SerialIter it{ownedNode->data(), ownedNode->size()};
+    ripple::SLE sle{it, cursorIndex};
+
+    if (!isOwnedByAccount(sle, accountID))
+        return {};
+
+    return AccountCursor({cursorIndex, startHint});
+}
+
 std::optional<std::string>
 getString(boost::json::object const& request, std::string const& field)
 {
@@ -341,18 +416,28 @@ toJson(ripple::LedgerInfo const& lgrInfo)
     return header;
 }
 
+std::optional<std::uint32_t>
+parseStringAsUInt(std::string const& value)
+{
+    std::optional<std::uint32_t> index = {};
+    try
+    {
+        index = boost::lexical_cast<std::uint32_t>(value);
+    }
+    catch (boost::bad_lexical_cast const&)
+    {
+    }
+
+    return index;
+}
+
 std::variant<Status, ripple::LedgerInfo>
 ledgerInfoFromRequest(Context const& ctx)
 {
-    auto indexValue = ctx.params.contains("ledger_index")
-        ? ctx.params.at("ledger_index")
-        : nullptr;
-
     auto hashValue = ctx.params.contains("ledger_hash")
         ? ctx.params.at("ledger_hash")
         : nullptr;
 
-    std::optional<ripple::LedgerInfo> lgrInfo;
     if (!hashValue.is_null())
     {
         if (!hashValue.is_string())
@@ -362,26 +447,38 @@ ledgerInfoFromRequest(Context const& ctx)
         if (!ledgerHash.parseHex(hashValue.as_string().c_str()))
             return Status{Error::rpcINVALID_PARAMS, "ledgerHashMalformed"};
 
-        lgrInfo = ctx.backend->fetchLedgerByHash(ledgerHash, ctx.yield);
+        auto lgrInfo = ctx.backend->fetchLedgerByHash(ledgerHash, ctx.yield);
     }
-    else if (!indexValue.is_null())
-    {
-        std::uint32_t ledgerSequence;
-        if (indexValue.is_string() && indexValue.as_string() == "validated")
-            ledgerSequence = ctx.range.maxSequence;
-        else if (!indexValue.is_string() && indexValue.is_int64())
-            ledgerSequence = indexValue.as_int64();
-        else
-            return Status{Error::rpcINVALID_PARAMS, "ledgerIndexMalformed"};
 
-        lgrInfo = ctx.backend->fetchLedgerBySequence(ledgerSequence, ctx.yield);
+    auto indexValue = ctx.params.contains("ledger_index")
+        ? ctx.params.at("ledger_index")
+        : nullptr;
+
+    std::optional<std::uint32_t> ledgerSequence = {};
+    if (!indexValue.is_null())
+    {
+        if (indexValue.is_string())
+        {
+            boost::json::string const& stringIndex = indexValue.as_string();
+            if (stringIndex == "validated")
+                ledgerSequence = ctx.range.maxSequence;
+            else
+                ledgerSequence = parseStringAsUInt(stringIndex.c_str());
+        }
+        else if (indexValue.is_int64())
+            ledgerSequence = indexValue.as_int64();
     }
     else
     {
-        lgrInfo = ctx.backend->fetchLedgerBySequence(
-            ctx.range.maxSequence, ctx.yield);
+        ledgerSequence = ctx.range.maxSequence;
     }
 
+    if (!ledgerSequence)
+        return Status{Error::rpcLGR_NOT_FOUND, "ledgerIndexMalformed"};
+
+    auto lgrInfo =
+        ctx.backend->fetchLedgerBySequence(*ledgerSequence, ctx.yield);
+
     if (!lgrInfo)
         return Status{Error::rpcLGR_NOT_FOUND, "ledgerNotFound"};
 
@@ -406,50 +503,155 @@ ledgerInfoToBlob(ripple::LedgerInfo const& info, bool includeHash)
     return s.peekData();
 }
 
-std::optional<ripple::uint256>
+std::uint64_t
+getStartHint(ripple::SLE const& sle, ripple::AccountID const& accountID)
+{
+    if (sle.getType() == ripple::ltRIPPLE_STATE)
+    {
+        if (sle.getFieldAmount(ripple::sfLowLimit).getIssuer() == accountID)
+            return sle.getFieldU64(ripple::sfLowNode);
+        else if (
+            sle.getFieldAmount(ripple::sfHighLimit).getIssuer() == accountID)
+            return sle.getFieldU64(ripple::sfHighNode);
+    }
+
+    if (!sle.isFieldPresent(ripple::sfOwnerNode))
+        return 0;
+
+    return sle.getFieldU64(ripple::sfOwnerNode);
+}
+
+std::variant<Status, AccountCursor>
 traverseOwnedNodes(
     BackendInterface const& backend,
     ripple::AccountID const& accountID,
     std::uint32_t sequence,
-    ripple::uint256 const& cursor,
+    std::uint32_t limit,
+    std::optional<std::string> jsonCursor,
     boost::asio::yield_context& yield,
-    std::function<bool(ripple::SLE)> atOwnedNode)
+    std::function<void(ripple::SLE)> atOwnedNode)
 {
-    if (!backend.fetchLedgerObject(
-            ripple::keylet::account(accountID).key, sequence, yield))
-        throw AccountNotFoundError(ripple::toBase58(accountID));
+    auto parsedCursor =
+        parseAccountCursor(backend, sequence, jsonCursor, accountID, yield);
+
+    if (!parsedCursor)
+        return Status(ripple::rpcINVALID_PARAMS, "Malformed cursor");
+
+    auto cursor = AccountCursor({beast::zero, 0});
+
+    auto [hexCursor, startHint] = *parsedCursor;
+
     auto const rootIndex = ripple::keylet::ownerDir(accountID);
     auto currentIndex = rootIndex;
 
     std::vector<ripple::uint256> keys;
-    std::optional<ripple::uint256> nextCursor = {};
+    keys.reserve(limit);
 
     auto start = std::chrono::system_clock::now();
+
+    // If startAfter is not zero try jumping to that page using the hint
+    if (hexCursor.isNonZero())
+    {
+        auto const hintIndex = ripple::keylet::page(rootIndex, startHint);
+        auto hintDir =
+            backend.fetchLedgerObject(hintIndex.key, sequence, yield);
+
+        if (hintDir)
+        {
+            ripple::SerialIter it{hintDir->data(), hintDir->size()};
+            ripple::SLE sle{it, hintIndex.key};
+
+            for (auto const& key : sle.getFieldV256(ripple::sfIndexes))
+            {
+                if (key == hexCursor)
+                {
+                    // We found the hint, we can start here
+                    currentIndex = hintIndex;
+                    break;
+                }
+            }
+        }
+
+        bool found = false;
         for (;;)
         {
-        auto ownedNode =
+            auto const ownerDir =
                 backend.fetchLedgerObject(currentIndex.key, sequence, yield);
 
-        if (!ownedNode)
+            if (!ownerDir)
+                return Status(
+                    ripple::rpcINVALID_PARAMS, "Owner directory not found");
+
+            ripple::SerialIter it{ownerDir->data(), ownerDir->size()};
+            ripple::SLE sle{it, currentIndex.key};
+
+            for (auto const& key : sle.getFieldV256(ripple::sfIndexes))
+            {
+                if (!found)
+                {
+                    if (key == hexCursor)
+                        found = true;
+                }
+                else
+                {
+                    keys.push_back(key);
+
+                    if (--limit == 0)
                     {
                         break;
                     }
-
-        ripple::SerialIter it{ownedNode->data(), ownedNode->size()};
-        ripple::SLE dir{it, currentIndex.key};
-
-        for (auto const& key : dir.getFieldV256(ripple::sfIndexes))
-        {
-            if (key >= cursor)
-                keys.push_back(key);
+                }
+            }
+
+            auto const uNodeNext = sle.getFieldU64(ripple::sfIndexNext);
+
+            if (limit == 0)
+            {
+                cursor = AccountCursor({keys.back(), uNodeNext});
+                break;
             }
 
-        auto const uNodeNext = dir.getFieldU64(ripple::sfIndexNext);
             if (uNodeNext == 0)
                 break;
 
             currentIndex = ripple::keylet::page(rootIndex, uNodeNext);
         }
+    }
+    else
+    {
+        for (;;)
+        {
+            auto const ownerDir =
+                backend.fetchLedgerObject(currentIndex.key, sequence, yield);
+
+            if (!ownerDir)
+                return Status(ripple::rpcACT_NOT_FOUND);
+
+            ripple::SerialIter it{ownerDir->data(), ownerDir->size()};
+            ripple::SLE sle{it, currentIndex.key};
+
+            for (auto const& key : sle.getFieldV256(ripple::sfIndexes))
+            {
+                keys.push_back(key);
+
+                if (--limit == 0)
+                    break;
+            }
+
+            auto const uNodeNext = sle.getFieldU64(ripple::sfIndexNext);
+
+            if (limit == 0)
+            {
+                cursor = AccountCursor({keys.back(), uNodeNext});
+                break;
+            }
+
+            if (uNodeNext == 0)
+                break;
+
+            currentIndex = ripple::keylet::page(rootIndex, uNodeNext);
+        }
+    }
     auto end = std::chrono::system_clock::now();
 
     BOOST_LOG_TRIVIAL(debug) << "Time loading owned directories: "
@@ -466,14 +668,14 @@ traverseOwnedNodes(
     {
         ripple::SerialIter it{objects[i].data(), objects[i].size()};
         ripple::SLE sle(it, keys[i]);
-        if (!atOwnedNode(sle))
-        {
-            nextCursor = keys[i + 1];
-            break;
-        }
+
+        atOwnedNode(sle);
     }
 
-    return nextCursor;
+    if (limit == 0)
+        return cursor;
+
+    return AccountCursor({beast::zero, 0});
 }
 
 std::optional<ripple::Seed>
@@ -934,7 +1136,8 @@ postProcessOrderBook(
             else
             {
                 saTakerGetsFunded = saOwnerFundsLimit;
-                offerJson["taker_gets_funded"] = saTakerGetsFunded.getText();
+                offerJson["taker_gets_funded"] = toBoostJson(
+                    saTakerGetsFunded.getJson(ripple::JsonOptions::none));
                 offerJson["taker_pays_funded"] = toBoostJson(
                     std::min(
                         saTakerPays,

src/rpc/RPCHelpers.h

@@ -20,6 +20,20 @@ accountFromStringStrict(std::string const& account);
 std::optional<ripple::AccountID>
 accountFromSeed(std::string const& account);
 
+bool
+isOwnedByAccount(ripple::SLE const& sle, ripple::AccountID const& accountID);
+
+std::uint64_t
+getStartHint(ripple::SLE const& sle, ripple::AccountID const& accountID);
+
+std::optional<AccountCursor>
+parseAccountCursor(
+    BackendInterface const& backend,
+    std::uint32_t seq,
+    std::optional<std::string> jsonCursor,
+    ripple::AccountID const& accountID,
+    boost::asio::yield_context& yield);
+
 // TODO this function should probably be in a different file and namespace
 std::pair<
     std::shared_ptr<ripple::STTx const>,
@@ -69,14 +83,15 @@ generatePubLedgerMessage(
 std::variant<Status, ripple::LedgerInfo>
 ledgerInfoFromRequest(Context const& ctx);
 
-std::optional<ripple::uint256>
+std::variant<Status, AccountCursor>
 traverseOwnedNodes(
     BackendInterface const& backend,
     ripple::AccountID const& accountID,
     std::uint32_t sequence,
-    ripple::uint256 const& cursor,
+    std::uint32_t limit,
+    std::optional<std::string> jsonCursor,
     boost::asio::yield_context& yield,
-    std::function<bool(ripple::SLE)> atOwnedNode);
+    std::function<void(ripple::SLE)> atOwnedNode);
 
 std::variant<Status, std::pair<ripple::PublicKey, ripple::SecretKey>>
 keypairFromRequst(boost::json::object const& request);

src/rpc/handlers/AccountChannels.cpp

@@ -90,14 +90,13 @@ doAccountChannels(Context const& context)
             return Status{Error::rpcINVALID_PARAMS, "limitNotPositive"};
     }
 
-    ripple::uint256 marker;
+    std::optional<std::string> marker = {};
     if (request.contains("marker"))
     {
         if (!request.at("marker").is_string())
             return Status{Error::rpcINVALID_PARAMS, "markerNotString"};
 
-        if (!marker.parseHex(request.at("marker").as_string().c_str()))
-            return Status{Error::rpcINVALID_PARAMS, "malformedCursor"};
+        marker = request.at("marker").as_string().c_str();
     }
 
     response["account"] = ripple::to_string(*accountID);
@@ -121,18 +120,25 @@ doAccountChannels(Context const& context)
         return true;
     };
 
-    auto nextCursor = traverseOwnedNodes(
+    auto next = traverseOwnedNodes(
         *context.backend,
         *accountID,
         lgrInfo.seq,
+        limit,
         marker,
         context.yield,
         addToResponse);
 
     response["ledger_hash"] = ripple::strHex(lgrInfo.hash);
     response["ledger_index"] = lgrInfo.seq;
-    if (nextCursor)
-        response["marker"] = ripple::strHex(*nextCursor);
+
+    if (auto status = std::get_if<RPC::Status>(&next))
+        return *status;
+
+    auto nextCursor = std::get<RPC::AccountCursor>(next);
+
+    if (nextCursor.isNonZero())
+        response["marker"] = nextCursor.toString();
 
     return response;
 }

src/rpc/handlers/AccountCurrencies.cpp

@@ -63,7 +63,8 @@ doAccountCurrencies(Context const& context)
         *context.backend,
         *accountID,
         lgrInfo.seq,
-        beast::zero,
+        std::numeric_limits<std::uint32_t>::max(),
+        {},
         context.yield,
         addToResponse);
 

src/rpc/handlers/AccountLines.cpp

@@ -135,14 +135,13 @@ doAccountLines(Context const& context)
             return Status{Error::rpcINVALID_PARAMS, "limitNotPositive"};
     }
 
-    ripple::uint256 cursor;
-    if (request.contains("cursor"))
+    std::optional<std::string> cursor = {};
+    if (request.contains("marker"))
     {
-        if (!request.at("cursor").is_string())
-            return Status{Error::rpcINVALID_PARAMS, "cursorNotString"};
+        if (!request.at("marker").is_string())
+            return Status{Error::rpcINVALID_PARAMS, "markerNotString"};
 
-        if (!cursor.parseHex(request.at("cursor").as_string().c_str()))
-            return Status{Error::rpcINVALID_PARAMS, "malformedCursor"};
+        cursor = request.at("marker").as_string().c_str();
     }
 
     response["account"] = ripple::to_string(*accountID);
@@ -151,30 +150,29 @@ doAccountLines(Context const& context)
     response["lines"] = boost::json::value(boost::json::array_kind);
     boost::json::array& jsonLines = response.at("lines").as_array();
 
-    auto const addToResponse = [&](ripple::SLE const& sle) {
+    auto const addToResponse = [&](ripple::SLE const& sle) -> void {
         if (sle.getType() == ripple::ltRIPPLE_STATE)
         {
-            if (limit-- == 0)
-            {
-                return false;
-            }
-
             addLine(jsonLines, sle, *accountID, peerAccount);
         }
-
-        return true;
     };
 
-    auto nextCursor = traverseOwnedNodes(
+    auto next = traverseOwnedNodes(
         *context.backend,
         *accountID,
         lgrInfo.seq,
+        limit,
         cursor,
         context.yield,
         addToResponse);
 
-    if (nextCursor)
-        response["marker"] = ripple::strHex(*nextCursor);
+    if (auto status = std::get_if<RPC::Status>(&next))
+        return *status;
+
+    auto nextCursor = std::get<RPC::AccountCursor>(next);
+
+    if (nextCursor.isNonZero())
+        response["marker"] = nextCursor.toString();
 
     return response;
 }

src/rpc/handlers/AccountObjects.cpp

@@ -60,14 +60,13 @@ doAccountObjects(Context const& context)
             return Status{Error::rpcINVALID_PARAMS, "limitNotPositive"};
     }
 
-    ripple::uint256 cursor;
+    std::optional<std::string> cursor = {};
     if (request.contains("marker"))
     {
         if (!request.at("marker").is_string())
             return Status{Error::rpcINVALID_PARAMS, "markerNotString"};
 
-        if (!cursor.parseHex(request.at("marker").as_string().c_str()))
-            return Status{Error::rpcINVALID_PARAMS, "malformedCursor"};
+        cursor = request.at("marker").as_string().c_str();
     }
 
     std::optional<ripple::LedgerEntryType> objectType = {};
@@ -90,21 +89,15 @@ doAccountObjects(Context const& context)
     auto const addToResponse = [&](ripple::SLE const& sle) {
         if (!objectType || objectType == sle.getType())
         {
-            if (limit-- == 0)
-            {
-                return false;
-            }
-
             jsonObjects.push_back(toJson(sle));
         }
-
-        return true;
     };
 
-    auto nextCursor = traverseOwnedNodes(
+    auto next = traverseOwnedNodes(
         *context.backend,
         *accountID,
         lgrInfo.seq,
+        limit,
         cursor,
         context.yield,
         addToResponse);
@@ -112,8 +105,13 @@ doAccountObjects(Context const& context)
     response["ledger_hash"] = ripple::strHex(lgrInfo.hash);
     response["ledger_index"] = lgrInfo.seq;
 
-    if (nextCursor)
-        response["marker"] = ripple::strHex(*nextCursor);
+    if (auto status = std::get_if<RPC::Status>(&next))
+        return *status;
+
+    auto nextCursor = std::get<RPC::AccountCursor>(next);
+
+    if (nextCursor.isNonZero())
+        response["marker"] = nextCursor.toString();
 
     return response;
 }

src/rpc/handlers/AccountOffers.cpp

@@ -97,14 +97,13 @@ doAccountOffers(Context const& context)
             return Status{Error::rpcINVALID_PARAMS, "limitNotPositive"};
     }
 
-    ripple::uint256 cursor;
-    if (request.contains("cursor"))
+    std::optional<std::string> cursor = {};
+    if (request.contains("marker"))
     {
-        if (!request.at("cursor").is_string())
-            return Status{Error::rpcINVALID_PARAMS, "cursorNotString"};
+        if (!request.at("marker").is_string())
+            return Status{Error::rpcINVALID_PARAMS, "markerNotString"};
 
-        if (!cursor.parseHex(request.at("cursor").as_string().c_str()))
-            return Status{Error::rpcINVALID_PARAMS, "malformedCursor"};
+        cursor = request.at("marker").as_string().c_str();
     }
 
     response["account"] = ripple::to_string(*accountID);
@@ -127,16 +126,22 @@ doAccountOffers(Context const& context)
         return true;
     };
 
-    auto nextCursor = traverseOwnedNodes(
+    auto next = traverseOwnedNodes(
         *context.backend,
         *accountID,
         lgrInfo.seq,
+        limit,
         cursor,
         context.yield,
         addToResponse);
 
-    if (nextCursor)
-        response["marker"] = ripple::strHex(*nextCursor);
+    if (auto status = std::get_if<RPC::Status>(&next))
+        return *status;
+
+    auto nextCursor = std::get<RPC::AccountCursor>(next);
+
+    if (nextCursor.isNonZero())
+        response["marker"] = nextCursor.toString();
 
     return response;
 }

src/rpc/handlers/AccountTx.cpp

@@ -219,8 +219,6 @@ doAccountTx(Context const& context)
             obj["date"] = txnPlusMeta.date;
         }
 
-        obj["validated"] = true;
-
         txns.push_back(obj);
         if (!minReturnedIndex || txnPlusMeta.ledgerSequence < *minReturnedIndex)
             minReturnedIndex = txnPlusMeta.ledgerSequence;

src/rpc/handlers/GatewayBalances.cpp

@@ -150,7 +150,8 @@ doGatewayBalances(Context const& context)
         *context.backend,
         *accountID,
         lgrInfo.seq,
-        beast::zero,
+        std::numeric_limits<std::uint32_t>::max(),
+        {},
         context.yield,
         addToResponse);
 

src/rpc/handlers/NoRippleCheck.cpp

@@ -90,6 +90,7 @@ doNoRippleCheck(Context const& context)
         *context.backend,
         *accountID,
         lgrInfo.seq,
+        std::numeric_limits<std::uint32_t>::max(),
         {},
         context.yield,
         [roleGateway,

src/webserver/HttpBase.h

@@ -358,10 +358,14 @@ handle_request(
         }
         else
         {
+            // This can still technically be an error. Clio counts forwarded
+            // requests as successful.
+
             counters.rpcComplete(context->method, us);
             result = std::get<boost::json::object>(v);
+
+            if (!result.contains("error"))
                 result["status"] = "success";
-            result["validated"] = true;
 
             responseStr = boost::json::serialize(response);
         }

src/webserver/WsBase.h

@@ -246,7 +246,6 @@ public:
                 auto id = request.contains("id") ? request.at("id") : nullptr;
 
                 response = getDefaultWsResponse(id);
-                boost::json::object& result = response["result"].as_object();
 
                 auto start = std::chrono::system_clock::now();
                 auto v = RPC::buildResponse(*context);
@@ -262,6 +261,7 @@ public:
 
                     if (!id.is_null())
                         error["id"] = id;
+
                     error["request"] = request;
                     response = error;
                 }
@@ -269,7 +269,7 @@ public:
                 {
                     counters_.rpcComplete(context->method, us);
 
-                    result = std::get<boost::json::object>(v);
+                    response["result"] = std::get<boost::json::object>(v);
                 }
             }
             catch (Backend::DatabaseTimeout const& t)

test.py

@@ -1,5 +1,6 @@
 #!/usr/bin/python3
 
+from ast import parse
 import websockets
 import asyncio
 import json
@@ -486,7 +487,7 @@ def writeLedgerData(data,filename):
             f.write('\n')
 
 
-async def ledger_data_full(ip, port, ledger, binary, limit, typ=None, count=-1):
+async def ledger_data_full(ip, port, ledger, binary, limit, typ=None, count=-1, marker = None):
     address = 'ws://' + str(ip) + ':' + str(port)
     try:
         blobs = []
@@ -494,7 +495,6 @@ async def ledger_data_full(ip, port, ledger, binary, limit, typ=None, count=-1):
         async with websockets.connect(address,max_size=1000000000) as ws:
             if int(limit) < 2048:
                 limit = 2048
-            marker = None
             while True:
                 res = {}
                 if marker is None:
@@ -974,7 +974,8 @@ parser = argparse.ArgumentParser(description='test script for xrpl-reporting')
 parser.add_argument('action', choices=["account_info", "tx", "txs","account_tx", "account_tx_full","ledger_data", "ledger_data_full", "book_offers","ledger","ledger_range","ledger_entry", "ledgers", "ledger_entries","account_txs","account_infos","account_txs_full","book_offerses","ledger_diff","perf","fee","server_info", "gaps","subscribe","verify_subscribe","call"])
 
 parser.add_argument('--ip', default='127.0.0.1')
-parser.add_argument('--port', default='51233')
+parser.add_argument('--port', default='8080')
+parser.add_argument('--marker')
 parser.add_argument('--hash')
 parser.add_argument('--account')
 parser.add_argument('--ledger')
@@ -993,6 +994,7 @@ parser.add_argument('--transactions',default=False)
 parser.add_argument('--minLedger',default=-1)
 parser.add_argument('--maxLedger',default=-1)
 parser.add_argument('--filename',default=None)
+parser.add_argument('--ledgerIndex', default=-1)
 parser.add_argument('--index')
 parser.add_argument('--numPages',default=3)
 parser.add_argument('--base')
@@ -1260,7 +1262,7 @@ def run(args):
                 args.filename = str(args.port) + "." + str(args.ledger)
 
         res = asyncio.get_event_loop().run_until_complete(
-                ledger_data_full(args.ip, args.port, args.ledger, bool(args.binary), args.limit,args.type, int(args.count)))
+                ledger_data_full(args.ip, args.port, args.ledger, bool(args.binary), args.limit,args.type, int(args.count), args.marker))
         print(len(res[0]))
         if args.verify:
             writeLedgerData(res,args.filename)