Refactor ETL into smaller components (#649)

Fixes #666 and #664

src/backend/BackendFactory.h

@@ -43,9 +43,7 @@ make_Backend(boost::asio::io_context& ioc, clio::Config const& config)
     if (boost::iequals(type, "cassandra") or boost::iequals(type, "cassandra-new"))
     {
         auto cfg = config.section("database." + type);
-        auto ttl = config.valueOr<uint16_t>("online_delete", 0) * 4;
-        backend =
-            std::make_shared<Backend::Cassandra::CassandraBackend>(Backend::Cassandra::SettingsProvider{cfg, ttl});
+        backend = std::make_shared<Backend::Cassandra::CassandraBackend>(Backend::Cassandra::SettingsProvider{cfg});
     }
 
     if (!backend)

src/backend/BackendInterface.h

@@ -21,7 +21,7 @@
 
 #include <ripple/ledger/ReadView.h>
 #include <backend/DBHelpers.h>
-#include <backend/SimpleCache.h>
+#include <backend/LedgerCache.h>
 #include <backend/Types.h>
 #include <config/Config.h>
 #include <log/Logger.h>
@@ -168,7 +168,7 @@ class BackendInterface
 protected:
     mutable std::shared_mutex rngMtx_;
     std::optional<LedgerRange> range;
-    SimpleCache cache_;
+    LedgerCache cache_;
 
     /**
      * @brief Public read methods
@@ -182,23 +182,21 @@ public:
     BackendInterface() = default;
     virtual ~BackendInterface() = default;
 
-    /*! @brief LEDGER METHODS */
-public:
     /**
      * @brief Cache that holds states of the ledger
-     *
-     * const version holds the original cache state; the other tracks
-     * historical changes.
-     *
-     * @return SimpleCache const&
+     * @return Immutable cache
      */
-    SimpleCache const&
+    LedgerCache const&
     cache() const
     {
         return cache_;
     }
 
-    SimpleCache&
+    /**
+     * @brief Cache that holds states of the ledger
+     * @return Mutable cache
+     */
+    LedgerCache&
     cache()
     {
         return cache_;
@@ -563,18 +561,6 @@ public:
     bool
     finishWrites(std::uint32_t const ledgerSequence);
 
-    /**
-     * @brief Selectively delets parts of the database.
-     *
-     * @param numLedgersToKeep Unsigned 32-bit integer on number of ledgers to
-     * keep.
-     * @param yield Currently executing coroutine.
-     * @return true
-     * @return false
-     */
-    virtual bool
-    doOnlineDelete(std::uint32_t numLedgersToKeep, boost::asio::yield_context& yield) const = 0;
-
     virtual bool
     isTooBusy() const = 0;
 

src/backend/CassandraBackend.h

@@ -26,6 +26,7 @@
 #include <backend/cassandra/SettingsProvider.h>
 #include <backend/cassandra/impl/ExecutionStrategy.h>
 #include <log/Logger.h>
+#include <util/LedgerUtils.h>
 #include <util/Profiler.h>
 
 #include <ripple/app/tx/impl/details/NFTokenUtils.h>
@@ -231,7 +232,7 @@ public:
             {
                 if (auto const maybeValue = result.template get<std::vector<unsigned char>>(); maybeValue)
                 {
-                    return deserializeHeader(ripple::makeSlice(*maybeValue));
+                    return util::deserializeHeader(ripple::makeSlice(*maybeValue));
                 }
 
                 log_.error() << "Could not fetch ledger by sequence - no rows";
@@ -792,14 +793,6 @@ public:
         // probably was used in PG to start a transaction or smth.
     }
 
-    /*! Unused in this implementation */
-    bool
-    doOnlineDelete(std::uint32_t const numLedgersToKeep, boost::asio::yield_context& yield) const override
-    {
-        log_.trace() << __func__ << " call";
-        return true;
-    }
-
     bool
     isTooBusy() const override
     {

src/backend/DBHelpers.h

@@ -30,8 +30,9 @@
 
 #include <backend/Types.h>
 
-/// Struct used to keep track of what to write to
-/// account_transactions/account_tx tables
+/**
+ * @brief Struct used to keep track of what to write to account_transactions/account_tx tables
+ */
 struct AccountTransactionsData
 {
     boost::container::flat_set<ripple::AccountID> accounts;
@@ -50,8 +51,11 @@ struct AccountTransactionsData
     AccountTransactionsData() = default;
 };
 
-/// Represents a link from a tx to an NFT that was targeted/modified/created
-/// by it. Gets written to nf_token_transactions table and the like.
+/**
+ * @brief Represents a link from a tx to an NFT that was targeted/modified/created by it
+ *
+ * Gets written to nf_token_transactions table and the like.
+ */
 struct NFTTransactionsData
 {
     ripple::uint256 tokenID;
@@ -65,8 +69,11 @@ struct NFTTransactionsData
     }
 };
 
-/// Represents an NFT state at a particular ledger. Gets written to nf_tokens
-/// table and the like.
+/**
+ * @brief Represents an NFT state at a particular ledger.
+ *
+ * Gets written to nf_tokens table and the like.
+ */
 struct NFTsData
 {
     ripple::uint256 tokenID;
@@ -195,28 +202,6 @@ getBookBase(T const& key)
     return ret;
 }
 
-inline ripple::LedgerInfo
-deserializeHeader(ripple::Slice data)
-{
-    ripple::SerialIter sit(data.data(), data.size());
-
-    ripple::LedgerInfo info;
-
-    info.seq = sit.get32();
-    info.drops = sit.get64();
-    info.parentHash = sit.get256();
-    info.txHash = sit.get256();
-    info.accountHash = sit.get256();
-    info.parentCloseTime = ripple::NetClock::time_point{ripple::NetClock::duration{sit.get32()}};
-    info.closeTime = ripple::NetClock::time_point{ripple::NetClock::duration{sit.get32()}};
-    info.closeTimeResolution = ripple::NetClock::duration{sit.get8()};
-    info.closeFlags = sit.get8();
-
-    info.hash = sit.get256();
-
-    return info;
-}
-
 inline std::string
 uint256ToString(ripple::uint256 const& uint)
 {

src/backend/LedgerCache.cpp

@@ -17,18 +17,19 @@
 */
 //==============================================================================
 
-#include <backend/SimpleCache.h>
+#include <backend/LedgerCache.h>
+
 namespace Backend {
 
 uint32_t
-SimpleCache::latestLedgerSequence() const
+LedgerCache::latestLedgerSequence() const
 {
     std::shared_lock lck{mtx_};
     return latestSeq_;
 }
 
 void
-SimpleCache::update(std::vector<LedgerObject> const& objs, uint32_t seq, bool isBackground)
+LedgerCache::update(std::vector<LedgerObject> const& objs, uint32_t seq, bool isBackground)
 {
     if (disabled_)
         return;
@@ -64,7 +65,7 @@ SimpleCache::update(std::vector<LedgerObject> const& objs, uint32_t seq, bool is
 }
 
 std::optional<LedgerObject>
-SimpleCache::getSuccessor(ripple::uint256 const& key, uint32_t seq) const
+LedgerCache::getSuccessor(ripple::uint256 const& key, uint32_t seq) const
 {
     if (!full_)
         return {};
@@ -80,7 +81,7 @@ SimpleCache::getSuccessor(ripple::uint256 const& key, uint32_t seq) const
 }
 
 std::optional<LedgerObject>
-SimpleCache::getPredecessor(ripple::uint256 const& key, uint32_t seq) const
+LedgerCache::getPredecessor(ripple::uint256 const& key, uint32_t seq) const
 {
     if (!full_)
         return {};
@@ -93,8 +94,9 @@ SimpleCache::getPredecessor(ripple::uint256 const& key, uint32_t seq) const
     --e;
     return {{e->first, e->second.blob}};
 }
+
 std::optional<Blob>
-SimpleCache::get(ripple::uint256 const& key, uint32_t seq) const
+LedgerCache::get(ripple::uint256 const& key, uint32_t seq) const
 {
     std::shared_lock lck{mtx_};
     if (seq > latestSeq_)
@@ -110,13 +112,13 @@ SimpleCache::get(ripple::uint256 const& key, uint32_t seq) const
 }
 
 void
-SimpleCache::setDisabled()
+LedgerCache::setDisabled()
 {
     disabled_ = true;
 }
 
 void
-SimpleCache::setFull()
+LedgerCache::setFull()
 {
     if (disabled_)
         return;
@@ -127,28 +129,32 @@ SimpleCache::setFull()
 }
 
 bool
-SimpleCache::isFull() const
+LedgerCache::isFull() const
 {
     return full_;
 }
+
 size_t
-SimpleCache::size() const
+LedgerCache::size() const
 {
     std::shared_lock lck{mtx_};
     return map_.size();
 }
+
 float
-SimpleCache::getObjectHitRate() const
+LedgerCache::getObjectHitRate() const
 {
     if (!objectReqCounter_)
         return 1;
     return ((float)objectHitCounter_) / objectReqCounter_;
 }
+
 float
-SimpleCache::getSuccessorHitRate() const
+LedgerCache::getSuccessorHitRate() const
 {
     if (!successorReqCounter_)
         return 1;
     return ((float)successorHitCounter_) / successorReqCounter_;
 }
+
 }  // namespace Backend

src/backend/LedgerCache.h

@@ -27,8 +27,10 @@
 #include <shared_mutex>
 #include <utility>
 #include <vector>
+
 namespace Backend {
-class SimpleCache
+
+class LedgerCache
 {
     struct CacheEntry
     {
@@ -39,22 +41,23 @@ class SimpleCache
     // counters for fetchLedgerObject(s) hit rate
     mutable std::atomic_uint32_t objectReqCounter_ = 0;
     mutable std::atomic_uint32_t objectHitCounter_ = 0;
+
     // counters for fetchSuccessorKey hit rate
     mutable std::atomic_uint32_t successorReqCounter_ = 0;
     mutable std::atomic_uint32_t successorHitCounter_ = 0;
 
     std::map<ripple::uint256, CacheEntry> map_;
+
     mutable std::shared_mutex mtx_;
     uint32_t latestSeq_ = 0;
     std::atomic_bool full_ = false;
     std::atomic_bool disabled_ = false;
-    // temporary set to prevent background thread from writing already deleted
-    // data. not used when cache is full
+
+    // temporary set to prevent background thread from writing already deleted data. not used when cache is full
     std::unordered_set<ripple::uint256, ripple::hardened_hash<>> deletes_;
 
 public:
-    // Update the cache with new ledger objects
-    // set isBackground to true when writing old data from a background thread
+    // Update the cache with new ledger objects set isBackground to true when writing old data from a background thread
     void
     update(std::vector<LedgerObject> const& blobs, uint32_t seq, bool isBackground = false);