diff --git a/handlers/AccountTx.cpp b/handlers/AccountTx.cpp index f9eaa23c..67f10517 100644 --- a/handlers/AccountTx.cpp +++ b/handlers/AccountTx.cpp @@ -172,9 +172,10 @@ doAccountTx( cursor = {*ledgerSequence, *transactionIndex}; } + constexpr uint32_t limit = 500; boost::json::array txns; auto [blobs, retCursor] = - backend.fetchAccountTransactions(*account, cursor); + backend.fetchAccountTransactions(*account, limit, cursor); for (auto const& txnPlusMeta : blobs) { boost::json::object obj; diff --git a/handlers/BookOffers.cpp b/handlers/BookOffers.cpp index ea56283c..e76120c0 100644 --- a/handlers/BookOffers.cpp +++ b/handlers/BookOffers.cpp @@ -292,31 +292,25 @@ doBookOffers( return response; } - std::uint32_t limit = 2048; + std::uint32_t limit = 500; if (request.contains("limit") and request.at("limit").kind() == boost::json::kind::int64) limit = request.at("limit").as_int64(); - ripple::uint256 cursor; + std::optional cursor; if (request.contains("cursor")) { - cursor.parseHex(request.at("cursor").as_string().c_str()); + cursor = {}; + cursor->parseHex(request.at("cursor").as_string().c_str()); } ripple::Book book = { {pay_currency, pay_issuer}, {get_currency, get_issuer}}; - auto start = std::chrono::system_clock::now(); ripple::uint256 bookBase = getBookBase(book); - std::vector offers; - if (!cursor.isZero()) - { - offers = backend.fetchBookOffers(bookBase, *sequence, cursor); - } - else - { - offers = backend.fetchBookOffers(bookBase, *sequence); - } + auto start = std::chrono::system_clock::now(); + auto [offers, retCursor] = + backend.fetchBookOffers(bookBase, *sequence, limit, cursor); auto end = std::chrono::system_clock::now(); BOOST_LOG_TRIVIAL(warning) << "Time loading books from Postgres: " diff --git a/reporting/BackendInterface.h b/reporting/BackendInterface.h index b2e2f9cb..31d7a75d 100644 --- a/reporting/BackendInterface.h +++ b/reporting/BackendInterface.h @@ -51,11 +51,11 @@ public: std::uint32_t ledgerSequence, std::uint32_t limit) const = 0; - // TODO needs to take in a limit, and return a cursor - virtual std::vector + virtual std::pair, std::optional> fetchBookOffers( ripple::uint256 const& book, uint32_t ledgerSequence, + std::uint32_t limit, std::optional const& cursor = {}) const = 0; virtual std::vector @@ -66,12 +66,12 @@ public: std::vector const& keys, uint32_t sequence) const = 0; - // TODO needs to take in a limit virtual std::pair< std::vector, std::optional> fetchAccountTransactions( ripple::AccountID const& account, + std::uint32_t limit, std::optional const& cursor = {}) const = 0; // write methods diff --git a/reporting/PostgresBackend.cpp b/reporting/PostgresBackend.cpp index 06d22dab..e8b8509b 100644 --- a/reporting/PostgresBackend.cpp +++ b/reporting/PostgresBackend.cpp @@ -279,10 +279,11 @@ PostgresBackend::fetchLedgerPage( return {}; } -std::vector +std::pair, std::optional> PostgresBackend::fetchBookOffers( ripple::uint256 const& book, uint32_t ledgerSequence, + std::uint32_t limit, std::optional const& cursor) const { PgQuery pgQuery(pgPool_); @@ -294,7 +295,8 @@ PostgresBackend::fetchBookOffers( if (cursor) sql << " AND key > \'" << ripple::strHex(*cursor) << "\'"; sql << " ORDER BY key DESC, sequence DESC)" - << " sub WHERE NOT deleted"; + << " sub WHERE NOT deleted" + << " LIMIT " << std::to_string(limit); auto res = pgQuery(sql.str().data()); if (size_t numRows = checkResult(res, 1)) { @@ -316,9 +318,9 @@ PostgresBackend::fetchBookOffers( [](auto& blob, auto& key) { return LedgerObject{std::move(key), std::move(blob)}; }); - return results; + return {results, results[results.size() - 1].key}; } - return {}; + return {{}, {}}; } std::vector @@ -405,6 +407,7 @@ std::pair< std::optional> PostgresBackend::fetchAccountTransactions( ripple::AccountID const& account, + std::uint32_t limit, std::optional const& cursor) const { PgQuery pgQuery(pgPool_); @@ -415,7 +418,6 @@ PostgresBackend::fetchAccountTransactions( if (cursor) sql << " AND ledger_sequence < " << cursor->ledgerSequence << " AND transaction_index < " << cursor->transactionIndex; - uint32_t limit = 300; sql << " LIMIT " << std::to_string(limit); auto res = pgQuery(sql.str().data()); if (size_t numRows = checkResult(res, 3)) diff --git a/reporting/PostgresBackend.h b/reporting/PostgresBackend.h index dd7948b1..240872eb 100644 --- a/reporting/PostgresBackend.h +++ b/reporting/PostgresBackend.h @@ -37,10 +37,11 @@ public: std::uint32_t ledgerSequence, std::uint32_t limit) const override; - std::vector + std::pair, std::optional> fetchBookOffers( ripple::uint256 const& book, uint32_t ledgerSequence, + std::uint32_t limit, std::optional const& cursor) const override; std::vector @@ -57,6 +58,7 @@ public: std::optional> fetchAccountTransactions( ripple::AccountID const& account, + std::uint32_t limit, std::optional const& cursor) const override; void diff --git a/reporting/ReportingBackend.cpp b/reporting/ReportingBackend.cpp index b3349828..c53a0e98 100644 --- a/reporting/ReportingBackend.cpp +++ b/reporting/ReportingBackend.cpp @@ -1331,7 +1331,7 @@ CassandraFlatMapBackend::open() query << "SELECT key FROM " << tableName << "books " << " WHERE book = ? AND sequence <= ? AND deleted_at > ? AND" " key > ? " - " ORDER BY key ASC LIMIT 300 ALLOW FILTERING"; + " ORDER BY key ASC LIMIT ? ALLOW FILTERING"; prepare_future = cass_session_prepare(session_.get(), query.str().c_str()); @@ -1380,7 +1380,7 @@ CassandraFlatMapBackend::open() query = {}; query << " SELECT hash,seq_idx FROM " << tableName << "account_tx" << " WHERE account = ? " - << " AND seq_idx < ? LIMIT 300"; + << " AND seq_idx < ? LIMIT ?"; prepare_future = cass_session_prepare(session_.get(), query.str().c_str()); diff --git a/reporting/ReportingBackend.h b/reporting/ReportingBackend.h index 0d926ec4..82576ba8 100644 --- a/reporting/ReportingBackend.h +++ b/reporting/ReportingBackend.h @@ -268,6 +268,7 @@ public: std::optional> fetchAccountTransactions( ripple::AccountID const& account, + std::uint32_t limit, std::optional const& cursor) const override { BOOST_LOG_TRIVIAL(debug) << "Starting doAccountTx"; @@ -304,6 +305,15 @@ public: << cass_error_desc(rc); return {}; } + rc = cass_statement_bind_int64(statement, 2, limit); + if (rc != CASS_OK) + { + cass_statement_free(statement); + BOOST_LOG_TRIVIAL(error) + << "Binding limit to account_tx query: " << rc << ", " + << cass_error_desc(rc); + return {}; + } CassFuture* fut; do @@ -1038,10 +1048,11 @@ public: return {{}, {}}; } - std::vector + std::pair, std::optional> fetchBookOffers( ripple::uint256 const& book, uint32_t sequence, + std::uint32_t limit, std::optional const& cursor) const override { BOOST_LOG_TRIVIAL(debug) << "Starting doBookOffers"; @@ -1098,6 +1109,15 @@ public: << ", " << cass_error_desc(rc); return {}; } + rc = cass_statement_bind_int64(statement, 4, limit); + if (rc != CASS_OK) + { + cass_statement_free(statement); + BOOST_LOG_TRIVIAL(error) + << "Binding Cassandra limit to doBookOffers query: " << rc + << ", " << cass_error_desc(rc); + return {}; + } CassFuture* fut; do { @@ -1152,10 +1172,10 @@ public: { results.push_back({keys[i], objs[i]}); } - return results; + return {results, results[results.size() - 1].key}; } - return {}; + return {{}, {}}; } bool canFetchBatch()