cleanup and postgres stubs

CMakeLists.txt

@@ -57,6 +57,7 @@ include(Postgres)
 target_sources(reporting PRIVATE
     reporting/ETLSource.cpp
     reporting/ReportingBackend.cpp
+    reporting/PostgresBackend.cpp
     reporting/Pg.cpp
     reporting/DBHelpers.cpp
     reporting/ReportingETL.cpp

handlers/AccountTx.cpp

@@ -148,7 +148,7 @@ doAccountTx(
         return response;
     }
 
-    std::optional<BackendInterface::AccountTransactionsCursor> cursor;
+    std::optional<Backend::AccountTransactionsCursor> cursor;
     if (request.contains("cursor"))
     {
         auto const& obj = request.at("cursor").as_object();

handlers/BookOffers.cpp

@@ -308,7 +308,7 @@ doBookOffers(
 
     auto start = std::chrono::system_clock::now();
     ripple::uint256 bookBase = getBookBase(book);
-    std::vector<BackendInterface::LedgerObject> offers;
+    std::vector<Backend::LedgerObject> offers;
     if (!cursor.isZero())
     {
         offers = backend.fetchBookOffers(bookBase, *sequence, cursor);

handlers/LedgerData.cpp

@@ -52,7 +52,7 @@ doLedgerData(
         request.contains("binary") ? request.at("binary").as_bool() : false;
     size_t limit = request.contains("limit") ? request.at("limit").as_int64()
                                              : (binary ? 2048 : 256);
-    BackendInterface::LedgerPage page;
+    Backend::LedgerPage page;
     auto start = std::chrono::system_clock::now();
     page = backend.fetchLedgerPage(cursor, ledger, limit);
 
@@ -62,7 +62,7 @@ doLedgerData(
         std::chrono::duration_cast<std::chrono::microseconds>(end - start)
             .count();
     boost::json::array objects;
-    std::vector<BackendInterface::LedgerObject>& results = page.objects;
+    std::vector<Backend::LedgerObject>& results = page.objects;
     std::optional<ripple::uint256> const& returnedCursor = page.cursor;
     BOOST_LOG_TRIVIAL(debug)
         << "doUpperBound returned " << results.size() << " results";

handlers/RPCHelpers.cpp

@@ -22,7 +22,7 @@ accountFromStringStrict(std::string const& account)
 std::pair<
     std::shared_ptr<ripple::STTx const>,
     std::shared_ptr<ripple::STObject const>>
-deserializeTxPlusMeta(BackendInterface::TransactionAndMetadata const& blobs)
+deserializeTxPlusMeta(Backend::TransactionAndMetadata const& blobs)
 {
     std::pair<
         std::shared_ptr<ripple::STTx const>,

handlers/RPCHelpers.h

@@ -12,7 +12,7 @@ accountFromStringStrict(std::string const& account);
 std::pair<
     std::shared_ptr<ripple::STTx const>,
     std::shared_ptr<ripple::STObject const>>
-deserializeTxPlusMeta(BackendInterface::TransactionAndMetadata const& blobs);
+deserializeTxPlusMeta(Backend::TransactionAndMetadata const& blobs);
 
 boost::json::object
 getJson(ripple::STBase const& obj);

reporting/ReportingBackend.cpp

@@ -1,4 +1,5 @@
 #include <reporting/ReportingBackend.h>
+namespace Backend {
 // Process the result of an asynchronous write. Retry on error
 // @param fut cassandra future associated with the write
 // @param cbData struct that holds the request parameters
@@ -1529,3 +1530,4 @@ CassandraFlatMapBackend::open()
     }
     BOOST_LOG_TRIVIAL(info) << "Opened database successfully";
 }
+}  // namespace Backend

reporting/ReportingBackend.h

@@ -34,6 +34,8 @@
 #include <reporting/BackendInterface.h>
 #include <reporting/DBHelpers.h>
 
+namespace Backend {
+
 void
 flatMapWriteCallback(CassFuture* fut, void* cbData);
 void
@@ -259,12 +261,11 @@ public:
     }
 
     std::pair<
-        std::vector<BackendInterface::TransactionAndMetadata>,
+        std::vector<TransactionAndMetadata>,
-        std::optional<BackendInterface::AccountTransactionsCursor>>
+        std::optional<AccountTransactionsCursor>>
     fetchAccountTransactions(
         ripple::AccountID const& account,
-        std::optional<BackendInterface::AccountTransactionsCursor> const&
+        std::optional<AccountTransactionsCursor> const& cursor) const override
-            cursor) const override
     {
         BOOST_LOG_TRIVIAL(debug) << "Starting doAccountTx";
         CassStatement* statement = cass_prepared_bind(selectAccountTx_);
@@ -707,7 +708,7 @@ public:
     // @param key the key of the object
     // @param pno object in which to store the result
     // @return result status of query
-    std::optional<BackendInterface::Blob>
+    std::optional<Blob>
     fetchLedgerObject(ripple::uint256 const& key, uint32_t sequence)
         const override
     {
@@ -837,7 +838,7 @@ public:
         return token + 1;
     }
 
-    std::optional<BackendInterface::TransactionAndMetadata>
+    std::optional<TransactionAndMetadata>
     fetchTransaction(ripple::uint256 const& hash) const override
     {
         BOOST_LOG_TRIVIAL(trace) << "Fetching from cassandra";
@@ -913,7 +914,7 @@ public:
             << " microseconds";
         return {{txResult, metaResult}};
     }
-    BackendInterface::LedgerPage
+    LedgerPage
     fetchLedgerPage(
         std::optional<ripple::uint256> const& cursor,
         std::uint32_t ledgerSequence,
@@ -1018,8 +1019,7 @@ public:
         if (keys.size())
         {
             std::vector<LedgerObject> results;
-            std::vector<BackendInterface::Blob> objs =
+            std::vector<Blob> objs = fetchLedgerObjects(keys, ledgerSequence);
-                fetchLedgerObjects(keys, ledgerSequence);
             for (size_t i = 0; i < objs.size(); ++i)
             {
                 results.push_back({keys[i], objs[i]});
@@ -1030,7 +1030,7 @@ public:
         return {{}, {}};
     }
 
-    std::vector<BackendInterface::LedgerObject>
+    std::vector<LedgerObject>
     fetchBookOffers(
         ripple::uint256 const& book,
         uint32_t sequence,
@@ -1139,8 +1139,7 @@ public:
         if (keys.size())
         {
             std::vector<LedgerObject> results;
-            std::vector<BackendInterface::Blob> objs =
+            std::vector<Blob> objs = fetchLedgerObjects(keys, sequence);
-                fetchLedgerObjects(keys, sequence);
             for (size_t i = 0; i < objs.size(); ++i)
             {
                 results.push_back({keys[i], objs[i]});
@@ -1160,7 +1159,7 @@ public:
     {
         CassandraFlatMapBackend const& backend;
         ripple::uint256 const& hash;
-        BackendInterface::TransactionAndMetadata& result;
+        TransactionAndMetadata& result;
         std::condition_variable& cv;
 
         std::atomic_uint32_t& numFinished;
@@ -1169,7 +1168,7 @@ public:
         ReadCallbackData(
             CassandraFlatMapBackend const& backend,
             ripple::uint256 const& hash,
-            BackendInterface::TransactionAndMetadata& result,
+            TransactionAndMetadata& result,
             std::condition_variable& cv,
             std::atomic_uint32_t& numFinished,
             size_t batchSize)
@@ -1185,7 +1184,7 @@ public:
         ReadCallbackData(ReadCallbackData const& other) = default;
     };
 
-    std::vector<BackendInterface::TransactionAndMetadata>
+    std::vector<TransactionAndMetadata>
     fetchTransactions(std::vector<ripple::uint256> const& hashes) const override
     {
         std::size_t const numHashes = hashes.size();
@@ -1194,8 +1193,7 @@ public:
         std::atomic_uint32_t numFinished = 0;
         std::condition_variable cv;
         std::mutex mtx;
-        std::vector<BackendInterface::TransactionAndMetadata> results{
+        std::vector<TransactionAndMetadata> results{numHashes};
-            numHashes};
         std::vector<std::shared_ptr<ReadCallbackData>> cbs;
         cbs.reserve(numHashes);
         for (std::size_t i = 0; i < hashes.size(); ++i)
@@ -1251,7 +1249,7 @@ public:
         CassandraFlatMapBackend const& backend;
         ripple::uint256 const& key;
         uint32_t sequence;
-        BackendInterface::Blob& result;
+        Blob& result;
         std::condition_variable& cv;
 
         std::atomic_uint32_t& numFinished;
@@ -1261,7 +1259,7 @@ public:
             CassandraFlatMapBackend const& backend,
             ripple::uint256 const& key,
             uint32_t sequence,
-            BackendInterface::Blob& result,
+            Blob& result,
             std::condition_variable& cv,
             std::atomic_uint32_t& numFinished,
             size_t batchSize)
@@ -1277,7 +1275,7 @@ public:
 
         ReadObjectCallbackData(ReadObjectCallbackData const& other) = default;
     };
-    std::vector<BackendInterface::Blob>
+    std::vector<Blob>
     fetchLedgerObjects(
         std::vector<ripple::uint256> const& keys,
         uint32_t sequence) const override
@@ -1288,7 +1286,7 @@ public:
         std::atomic_uint32_t numFinished = 0;
         std::condition_variable cv;
         std::mutex mtx;
-        std::vector<BackendInterface::Blob> results{numKeys};
+        std::vector<Blob> results{numKeys};
         std::vector<std::shared_ptr<ReadObjectCallbackData>> cbs;
         cbs.reserve(numKeys);
         for (std::size_t i = 0; i < keys.size(); ++i)
@@ -1743,12 +1741,16 @@ public:
     }
 
     void
-    writeAccountTransactions(AccountTransactionsData&& data) const override
+    writeAccountTransactions(
+        std::vector<AccountTransactionsData>&& data) const override
     {
-        numRequestsOutstanding_ += data.accounts.size();
+        for (auto& record : data)
-        WriteAccountTxCallbackData* cbData =
+        {
-            new WriteAccountTxCallbackData(this, std::move(data));
+            numRequestsOutstanding_ += record.accounts.size();
-        writeAccountTx(*cbData, false);
+            WriteAccountTxCallbackData* cbData =
+                new WriteAccountTxCallbackData(this, std::move(record));
+            writeAccountTx(*cbData, false);
+        }
     }
 
     void
@@ -1778,6 +1780,7 @@ public:
                 static_cast<cass_byte_t const*>(accountData),
                 account.size());
             if (rc != CASS_OK)
+
             {
                 cass_statement_free(statement);
                 std::stringstream ss;
@@ -1985,4 +1988,5 @@ public:
     flatMapGetCreatedCallback(CassFuture* fut, void* cbData);
 };
 
+}  // namespace Backend
 #endif

reporting/ReportingETL.cpp

@@ -127,10 +127,7 @@ ReportingETL::loadInitialLedger(uint32_t startingSequence)
 
     if (!stopping_)
     {
-        for (auto& data : accountTxData)
+        flatMapBackend_->writeAccountTransactions(std::move(accountTxData));
-        {
-            flatMapBackend_->writeAccountTransactions(std::move(data));
-        }
         bool success = flatMapBackend_->writeLedger(
             lgrInfo, std::move(*ledgerData->mutable_ledger_header()));
     }
@@ -301,10 +298,7 @@ ReportingETL::buildNextLedger(org::xrpl::rpc::v1::GetLedgerResponse& rawData)
             isDeleted,
             std::move(bookDir));
     }
-    for (auto& data : accountTxData)
+    flatMapBackend_->writeAccountTransactions(std::move(accountTxData));
-    {
-        flatMapBackend_->writeAccountTransactions(std::move(data));
-    }
     bool success = flatMapBackend_->writeLedger(
         lgrInfo, std::move(*rawData.mutable_ledger_header()));
     BOOST_LOG_TRIVIAL(debug)
@@ -638,7 +632,7 @@ ReportingETL::ReportingETL(
     boost::asio::io_context& ioc)
     : publishStrand_(ioc)
     , ioContext_(ioc)
-    , flatMapBackend_(makeBackend(config))
+    , flatMapBackend_(Backend::makeBackend(config))
     , pgPool_(make_PgPool(
           config.at("database").as_object().at("postgres").as_object()))
     , loadBalancer_(