add cursor and limit to book offers

handlers/AccountTx.cpp

@@ -172,9 +172,10 @@ doAccountTx(
         cursor = {*ledgerSequence, *transactionIndex};
     }
 
+    constexpr uint32_t limit = 500;
     boost::json::array txns;
     auto [blobs, retCursor] =
-        backend.fetchAccountTransactions(*account, cursor);
+        backend.fetchAccountTransactions(*account, limit, cursor);
     for (auto const& txnPlusMeta : blobs)
     {
         boost::json::object obj;

handlers/BookOffers.cpp

@@ -292,31 +292,25 @@ doBookOffers(
         return response;
     }
 
-    std::uint32_t limit = 2048;
+    std::uint32_t limit = 500;
     if (request.contains("limit") and
         request.at("limit").kind() == boost::json::kind::int64)
         limit = request.at("limit").as_int64();
 
-    ripple::uint256 cursor;
+    std::optional<ripple::uint256> cursor;
     if (request.contains("cursor"))
     {
-        cursor.parseHex(request.at("cursor").as_string().c_str());
+        cursor = {};
+        cursor->parseHex(request.at("cursor").as_string().c_str());
     }
 
     ripple::Book book = {
         {pay_currency, pay_issuer}, {get_currency, get_issuer}};
 
-    auto start = std::chrono::system_clock::now();
     ripple::uint256 bookBase = getBookBase(book);
-    std::vector<Backend::LedgerObject> offers;
+    auto start = std::chrono::system_clock::now();
-    if (!cursor.isZero())
+    auto [offers, retCursor] =
-    {
+        backend.fetchBookOffers(bookBase, *sequence, limit, cursor);
-        offers = backend.fetchBookOffers(bookBase, *sequence, cursor);
-    }
-    else
-    {
-        offers = backend.fetchBookOffers(bookBase, *sequence);
-    }
     auto end = std::chrono::system_clock::now();
 
     BOOST_LOG_TRIVIAL(warning) << "Time loading books from Postgres: "

reporting/BackendInterface.h

@@ -51,11 +51,11 @@ public:
         std::uint32_t ledgerSequence,
         std::uint32_t limit) const = 0;
 
-    // TODO needs to take in a limit, and return a cursor
+    virtual std::pair<std::vector<LedgerObject>, std::optional<ripple::uint256>>
-    virtual std::vector<LedgerObject>
     fetchBookOffers(
         ripple::uint256 const& book,
         uint32_t ledgerSequence,
+        std::uint32_t limit,
         std::optional<ripple::uint256> const& cursor = {}) const = 0;
 
     virtual std::vector<TransactionAndMetadata>
@@ -66,12 +66,12 @@ public:
         std::vector<ripple::uint256> const& keys,
         uint32_t sequence) const = 0;
 
-    // TODO needs to take in a limit
     virtual std::pair<
         std::vector<TransactionAndMetadata>,
         std::optional<AccountTransactionsCursor>>
     fetchAccountTransactions(
         ripple::AccountID const& account,
+        std::uint32_t limit,
         std::optional<AccountTransactionsCursor> const& cursor = {}) const = 0;
 
     // write methods

reporting/PostgresBackend.cpp

@@ -279,10 +279,11 @@ PostgresBackend::fetchLedgerPage(
     return {};
 }
 
-std::vector<LedgerObject>
+std::pair<std::vector<LedgerObject>, std::optional<ripple::uint256>>
 PostgresBackend::fetchBookOffers(
     ripple::uint256 const& book,
     uint32_t ledgerSequence,
+    std::uint32_t limit,
     std::optional<ripple::uint256> const& cursor) const
 {
     PgQuery pgQuery(pgPool_);
@@ -294,7 +295,8 @@ PostgresBackend::fetchBookOffers(
     if (cursor)
         sql << " AND key > \'" << ripple::strHex(*cursor) << "\'";
     sql << " ORDER BY key DESC, sequence DESC)"
-        << " sub WHERE NOT deleted";
+        << " sub WHERE NOT deleted"
+        << " LIMIT " << std::to_string(limit);
     auto res = pgQuery(sql.str().data());
     if (size_t numRows = checkResult(res, 1))
     {
@@ -316,9 +318,9 @@ PostgresBackend::fetchBookOffers(
             [](auto& blob, auto& key) {
                 return LedgerObject{std::move(key), std::move(blob)};
             });
-        return results;
+        return {results, results[results.size() - 1].key};
     }
-    return {};
+    return {{}, {}};
 }
 
 std::vector<TransactionAndMetadata>
@@ -405,6 +407,7 @@ std::pair<
     std::optional<AccountTransactionsCursor>>
 PostgresBackend::fetchAccountTransactions(
     ripple::AccountID const& account,
+    std::uint32_t limit,
     std::optional<AccountTransactionsCursor> const& cursor) const
 {
     PgQuery pgQuery(pgPool_);
@@ -415,7 +418,6 @@ PostgresBackend::fetchAccountTransactions(
     if (cursor)
         sql << " AND ledger_sequence < " << cursor->ledgerSequence
             << " AND transaction_index < " << cursor->transactionIndex;
-    uint32_t limit = 300;
     sql << " LIMIT " << std::to_string(limit);
     auto res = pgQuery(sql.str().data());
     if (size_t numRows = checkResult(res, 3))

reporting/PostgresBackend.h

@@ -37,10 +37,11 @@ public:
         std::uint32_t ledgerSequence,
         std::uint32_t limit) const override;
 
-    std::vector<LedgerObject>
+    std::pair<std::vector<LedgerObject>, std::optional<ripple::uint256>>
     fetchBookOffers(
         ripple::uint256 const& book,
         uint32_t ledgerSequence,
+        std::uint32_t limit,
         std::optional<ripple::uint256> const& cursor) const override;
 
     std::vector<TransactionAndMetadata>
@@ -57,6 +58,7 @@ public:
         std::optional<AccountTransactionsCursor>>
     fetchAccountTransactions(
         ripple::AccountID const& account,
+        std::uint32_t limit,
         std::optional<AccountTransactionsCursor> const& cursor) const override;
 
     void

reporting/ReportingBackend.cpp

@@ -1331,7 +1331,7 @@ CassandraFlatMapBackend::open()
         query << "SELECT key FROM " << tableName << "books "
               << " WHERE book = ? AND sequence <= ? AND deleted_at > ? AND"
                  " key > ? "
-                 " ORDER BY key ASC LIMIT 300 ALLOW FILTERING";
+                 " ORDER BY key ASC LIMIT ? ALLOW FILTERING";
 
         prepare_future =
             cass_session_prepare(session_.get(), query.str().c_str());
@@ -1380,7 +1380,7 @@ CassandraFlatMapBackend::open()
         query = {};
         query << " SELECT hash,seq_idx FROM " << tableName << "account_tx"
               << " WHERE account = ? "
-              << " AND seq_idx < ? LIMIT 300";
+              << " AND seq_idx < ? LIMIT ?";
 
         prepare_future =
             cass_session_prepare(session_.get(), query.str().c_str());

reporting/ReportingBackend.h

@@ -268,6 +268,7 @@ public:
         std::optional<AccountTransactionsCursor>>
     fetchAccountTransactions(
         ripple::AccountID const& account,
+        std::uint32_t limit,
         std::optional<AccountTransactionsCursor> const& cursor) const override
     {
         BOOST_LOG_TRIVIAL(debug) << "Starting doAccountTx";
@@ -304,6 +305,15 @@ public:
                 << cass_error_desc(rc);
             return {};
         }
+        rc = cass_statement_bind_int64(statement, 2, limit);
+        if (rc != CASS_OK)
+        {
+            cass_statement_free(statement);
+            BOOST_LOG_TRIVIAL(error)
+                << "Binding limit to account_tx query: " << rc << ", "
+                << cass_error_desc(rc);
+            return {};
+        }
 
         CassFuture* fut;
         do
@@ -1038,10 +1048,11 @@ public:
         return {{}, {}};
     }
 
-    std::vector<LedgerObject>
+    std::pair<std::vector<LedgerObject>, std::optional<ripple::uint256>>
     fetchBookOffers(
         ripple::uint256 const& book,
         uint32_t sequence,
+        std::uint32_t limit,
         std::optional<ripple::uint256> const& cursor) const override
     {
         BOOST_LOG_TRIVIAL(debug) << "Starting doBookOffers";
@@ -1098,6 +1109,15 @@ public:
                 << ", " << cass_error_desc(rc);
             return {};
         }
+        rc = cass_statement_bind_int64(statement, 4, limit);
+        if (rc != CASS_OK)
+        {
+            cass_statement_free(statement);
+            BOOST_LOG_TRIVIAL(error)
+                << "Binding Cassandra limit to doBookOffers query: " << rc
+                << ", " << cass_error_desc(rc);
+            return {};
+        }
         CassFuture* fut;
         do
         {
@@ -1152,10 +1172,10 @@ public:
             {
                 results.push_back({keys[i], objs[i]});
             }
-            return results;
+            return {results, results[results.size() - 1].key};
         }
 
-        return {};
+        return {{}, {}};
     }
     bool
     canFetchBatch()