address pr feedback

handlers/BookOffers.cpp

@@ -48,7 +48,6 @@ doBookOffers(
     boost::json::object const& request,
     BackendInterface const& backend)
 {
-    std::cout << "enter" << std::endl;
     boost::json::object response;
 
     auto ledgerSequence = ledgerSequenceFromRequest(request, backend);

reporting/CassandraBackend.cpp

@@ -497,7 +497,7 @@ CassandraBackend::fetchLedgerObjects(
 BookOffersPage
 CassandraBackend::fetchBookOffers(
     ripple::uint256 const& book,
-    uint32_t sequence,
+    uint32_t ledgerSequence,
     std::uint32_t limit,
     std::optional<ripple::uint256> const& cursor) const
 {
@@ -506,53 +506,41 @@ CassandraBackend::fetchBookOffers(
     if(!rng)
         return {{},{}};
 
-    uint32_t upper = sequence;
+    auto readBooks = [this, &book](std::uint32_t sequence) 
-    auto lastPage = rng->maxSequence - (rng->maxSequence % 256);
+            -> std::pair<bool, std::vector<std::pair<std::uint64_t, ripple::uint256>>>
-
-    auto readBooks = [this](
-            ripple::uint256 const& book, 
-            std::uint32_t seq,
-            std::optional<ripple::uint256> const& cursor) 
-            -> std::vector<std::pair<std::uint64_t, ripple::uint256>>
     {
-        CassandraStatement statement{this->selectBook_};
+        CassandraStatement completeQuery{completeBook_};
+        completeQuery.bindInt(sequence);
+        CassandraResult completeResult = executeSyncRead(completeQuery);
+        bool complete = completeResult.hasResult();
+
+        CassandraStatement statement{selectBook_};
         std::vector<std::pair<std::uint64_t, ripple::uint256>> keys = {};
     
         statement.bindBytes(book.data(), 24);
-        statement.bindInt(seq);
+        statement.bindInt(sequence);
 
-        BOOST_LOG_TRIVIAL(info) << __func__ << " upper = " << std::to_string(seq)
+        BOOST_LOG_TRIVIAL(info) << __func__ << " upper = " << std::to_string(sequence)
                                 << " book = " << ripple::strHex(std::string((char*)book.data(), 24));
 
-        if (cursor)
+        ripple::uint256 zero = beast::zero;
-        {
+        statement.bindBytes(zero.data(), 8);
-            auto object = this->fetchLedgerObject(*cursor, seq);
+        statement.bindBytes(zero);
 
-            if(!object)
+        auto start = std::chrono::system_clock::now();
-                return {{}, {}};
 
-            ripple::SerialIter it{object->data(), object->size()};
+        CassandraResult result = executeSyncRead(statement);
-            ripple::SLE offer{it, *cursor};
-            ripple::uint256 bookDir = offer.getFieldH256(ripple::sfBookDirectory);
 
-            statement.bindBytes(bookDir.data() + 24, 8);
+        auto end = std::chrono::system_clock::now();
-            statement.bindBytes(*cursor);
+        auto duration = ((end - start).count()) / 1000000000.0;
-        }
-        else
-        {
-            ripple::uint256 zero = beast::zero;
-            statement.bindBytes(zero.data(), 8);
-            statement.bindBytes(zero);
-        }
 
-        // statement.bindUInt(limit);
+        BOOST_LOG_TRIVIAL(info) << "Book directory fetch took "
-        CassandraResult result = this->executeSyncRead(statement);
+                                << std::to_string(duration) << " seconds.";
 
         BOOST_LOG_TRIVIAL(debug) << __func__ << " - got keys";
         if (!result)
         {
-            std::cout << "could not sync read" << std::endl;
+            return {false, {{}, {}}};
-            return {{}, {}};
         }
 
         do
@@ -564,43 +552,36 @@ CassandraBackend::fetchBookOffers(
 
         } while (result.nextRow());
 
-        return keys;
+        return {complete, keys};
     };
 
-    std::vector<std::pair<std::uint64_t, ripple::uint256>> quality_keys;
+    auto upper = indexer_.getBookIndexOfSeq(ledgerSequence);
-    if (sequence != rng->minSequence)
+    auto [complete, quality_keys] = readBooks(upper);
-    {
-        upper = (sequence >> 8) << 8;
-        if (upper != sequence)
-            upper += (1 << 8);
-
-        quality_keys = readBooks(book, upper, cursor);
-    }
 
     BOOST_LOG_TRIVIAL(debug)
         << __func__ << " - populated keys. num keys = " << quality_keys.size();
 
-    std::cout << "KEYS SIZE: " << quality_keys.size() << std::endl;
-    if (!quality_keys.size())
-        return {{}, {}};
-
     std::optional<std::string> warning = {};
-    if (quality_keys[0].second.isZero())
+    if (!complete)
     {
         warning = "Data may be incomplete";
+        BOOST_LOG_TRIVIAL(info) << "May be incomplete. Fetching other page";
 
-        std::uint32_t lower = (sequence >> 8) << 8;
+        auto bookShift = indexer_.getBookShift();
+        std::uint32_t lower = upper - (1 << bookShift);
         auto originalKeys = std::move(quality_keys);
-        auto otherKeys = readBooks(book, lower, cursor);
+        auto [lowerComplete, otherKeys] = readBooks(lower);
-        quality_keys.reserve(originalKeys.size() + otherKeys.size());
 
+        assert(lowerComplete);
+
+        quality_keys.reserve(originalKeys.size() + otherKeys.size());
         std::merge(originalKeys.begin(), originalKeys.end(),
-                          otherKeys.begin(), otherKeys.end(),
+                   otherKeys.begin(), otherKeys.end(),
-                          std::back_inserter(quality_keys),
+                   std::back_inserter(quality_keys),
-                          [](auto pair1, auto pair2) 
+                   [](auto pair1, auto pair2) 
-                          {
+                   {
-                              return pair1.first < pair2.first;
+                        return pair1.first < pair2.first;
-                          });
+                   });
     }
 
     std::vector<ripple::uint256> keys(quality_keys.size());
@@ -609,7 +590,16 @@ CassandraBackend::fetchBookOffers(
                    [](auto pair) { return pair.second; });
 
     std::vector<LedgerObject> results;
-    std::vector<Blob> objs = fetchLedgerObjects(keys, sequence);
+
+    auto start = std::chrono::system_clock::now();
+
+    std::vector<Blob> objs = fetchLedgerObjects(keys, ledgerSequence);
+
+    auto end = std::chrono::system_clock::now();
+    auto duration = ((end - start).count()) / 1000000000.0;
+
+    BOOST_LOG_TRIVIAL(info) << "Book directory fetch took "
+                            << std::to_string(duration) << " seconds.";
 
     for (size_t i = 0; i < objs.size(); ++i)
     {
@@ -1116,7 +1106,6 @@ CassandraBackend::doOnlineDelete(uint32_t minLedgerToKeep) const
 void
 CassandraBackend::open(bool readOnly)
 {
-    std::cout << config_ << std::endl;
     auto getString = [this](std::string const& field) -> std::string {
         if (config_.contains(field))
         {
@@ -1567,6 +1556,15 @@ CassandraBackend::open(bool readOnly)
         if (!selectBook_.prepareStatement(query, session_.get()))
             continue;
 
+        query.str("");
+        query << "SELECT * FROM " << tablePrefix << "books "
+              << "WHERE book = "
+              << "0x" << ripple::strHex(ripple::uint256(beast::zero))
+              << " AND sequence = ?"; 
+        if (!completeBook_.prepareStatement(query, session_.get()))
+            continue;
+
+
         query.str("");
         query << " INSERT INTO " << tablePrefix << "account_tx"
               << " (account, seq_idx, hash) "

reporting/CassandraBackend.h

@@ -640,6 +640,7 @@ private:
     CassandraPreparedStatement selectKeys_;
     CassandraPreparedStatement getBook_;
     CassandraPreparedStatement selectBook_;
+    CassandraPreparedStatement completeBook_;
     CassandraPreparedStatement insertBook_;
     CassandraPreparedStatement insertBook2_;
     CassandraPreparedStatement deleteBook_;

reporting/PostgresBackend.cpp

@@ -409,7 +409,7 @@ PostgresBackend::fetchBookOffers(
         if (size_t numRows = checkResult(res, 1))
             complete = res.asInt(0, 0) != 0;
         else 
-            return {true, {}};
+            return {false, {}};
 
         sql.str("");
         sql << "SELECT book, offer_key FROM books "
@@ -445,7 +445,7 @@ PostgresBackend::fetchBookOffers(
             return {complete, results};
         }
 
-        return {true, {}};
+        return {complete, {}};
     };
 
     auto fetchObjects =