Add database counters:

Fix bug where DatabaseRotateImp::getBackend and ::sync utilized the writable
backend without a lock. ::getBackend was replaced with ::getCounters.

src/ripple/nodestore/Backend.h

@@ -39,13 +39,27 @@ namespace NodeStore {
 class Backend
 {
 public:
+    template <typename T>
     struct Counters
     {
-        std::atomic<std::uint64_t> writeDurationUs{0};
+        Counters() = default;
-        std::atomic<std::uint64_t> writeRetries{0};
+        Counters(Counters const&) = default;
-        std::atomic<std::uint64_t> writesDelayed{0};
+
-        std::atomic<std::uint64_t> readRetries{0};
+        template <typename U>
-        std::atomic<std::uint64_t> readErrors{0};
+        Counters(Counters<U> const& other)
+            : writeDurationUs(other.writeDurationUs)
+            , writeRetries(other.writeRetries)
+            , writesDelayed(other.writesDelayed)
+            , readRetries(other.readRetries)
+            , readErrors(other.readErrors)
+        {
+        }
+
+        T writeDurationUs = {};
+        T writeRetries = {};
+        T writesDelayed = {};
+        T readRetries = {};
+        T readErrors = {};
     };
 
     /** Destroy the backend.
@@ -143,8 +157,16 @@ public:
     virtual int
     fdRequired() const = 0;
 
-    virtual Counters const&
+    /** Returns read and write stats.
-    counters() const = 0;
+
+        @note The Counters struct is specific to and only used
+              by CassandraBackend.
+    */
+    virtual std::optional<Counters<std::uint64_t>>
+    counters() const
+    {
+        return std::nullopt;
+    }
 
     /** Returns true if the backend uses permanent storage. */
     bool

src/ripple/nodestore/Database.h

@@ -20,8 +20,6 @@
 #ifndef RIPPLE_NODESTORE_DATABASE_H_INCLUDED
 #define RIPPLE_NODESTORE_DATABASE_H_INCLUDED
 
-#include <ripple/basics/KeyCache.h>
-#include <ripple/basics/TaggedCache.h>
 #include <ripple/core/Stoppable.h>
 #include <ripple/nodestore/Backend.h>
 #include <ripple/nodestore/NodeObject.h>
@@ -178,9 +176,6 @@ public:
     virtual void
     sweep() = 0;
 
-    virtual Backend&
-    getBackend() = 0;
-
     /** Gather statistics pertaining to read and write activities.
      *
      * @param obj Json object reference into which to place counters.
@@ -258,10 +253,6 @@ protected:
         storeSz_ += sz;
     }
 
-    // Called by the public asyncFetch function
-    void
-    asyncFetch(uint256 const& hash, std::uint32_t ledgerSeq);
-
     // Called by the public import function
     void
     importInternal(Backend& dstBackend, Database& srcDB);
@@ -322,6 +313,17 @@ private:
     virtual void
     for_each(std::function<void(std::shared_ptr<NodeObject>)> f) = 0;
 
+    /** Retrieve backend read and write stats.
+
+        @note The Counters struct is specific to and only used
+              by CassandraBackend.
+    */
+    virtual std::optional<Backend::Counters<std::uint64_t>>
+    getCounters() const
+    {
+        return std::nullopt;
+    }
+
     void
     threadEntry();
 };

src/ripple/nodestore/backend/CassandraFactory.cpp

@@ -826,7 +826,7 @@ public:
         return 0;
     }
 
-    Counters const&
+    std::optional<Counters<std::uint64_t>>
     counters() const override
     {
         return counters_;

src/ripple/nodestore/backend/MemoryFactory.cpp

@@ -220,13 +220,6 @@ public:
     {
         return 0;
     }
-
-    Counters const&
-    counters() const override
-    {
-        static Counters counters;
-        return counters;
-    }
 };
 
 //------------------------------------------------------------------------------

src/ripple/nodestore/backend/NuDBFactory.cpp

@@ -328,13 +328,6 @@ public:
     {
         return 3;
     }
-
-    Counters const&
-    counters() const override
-    {
-        static Counters counters;
-        return counters;
-    }
 };
 
 //------------------------------------------------------------------------------

src/ripple/nodestore/backend/NullFactory.cpp

@@ -115,13 +115,6 @@ public:
         return 0;
     }
 
-    Counters const&
-    counters() const override
-    {
-        static Counters counters;
-        return counters;
-    }
-
 private:
 };
 

src/ripple/nodestore/backend/RocksDBFactory.cpp

@@ -436,13 +436,6 @@ public:
     {
         return fdRequired_;
     }
-
-    Counters const&
-    counters() const override
-    {
-        static Counters counters;
-        return counters;
-    }
 };
 
 //------------------------------------------------------------------------------

src/ripple/nodestore/impl/Database.cpp

@@ -325,12 +325,15 @@ Database::getCountsJson(Json::Value& obj)
     obj[jss::node_written_bytes] = std::to_string(storeSz_);
     obj[jss::node_read_bytes] = std::to_string(fetchSz_);
     obj[jss::node_reads_duration_us] = std::to_string(fetchDurationUs_);
-    auto const& c = getBackend().counters();
+
-    obj[jss::node_read_errors] = std::to_string(c.readErrors);
+    if (auto c = getCounters())
-    obj[jss::node_read_retries] = std::to_string(c.readRetries);
+    {
-    obj[jss::node_write_retries] = std::to_string(c.writeRetries);
+        obj[jss::node_read_errors] = std::to_string(c->readErrors);
-    obj[jss::node_writes_delayed] = std::to_string(c.writesDelayed);
+        obj[jss::node_read_retries] = std::to_string(c->readRetries);
-    obj[jss::node_writes_duration_us] = std::to_string(c.writeDurationUs);
+        obj[jss::node_write_retries] = std::to_string(c->writeRetries);
+        obj[jss::node_writes_delayed] = std::to_string(c->writesDelayed);
+        obj[jss::node_writes_duration_us] = std::to_string(c->writeDurationUs);
+    }
 }
 
 }  // namespace NodeStore

src/ripple/nodestore/impl/DatabaseNodeImp.h

@@ -20,6 +20,7 @@
 #ifndef RIPPLE_NODESTORE_DATABASENODEIMP_H_INCLUDED
 #define RIPPLE_NODESTORE_DATABASENODEIMP_H_INCLUDED
 
+#include <ripple/basics/TaggedCache.h>
 #include <ripple/basics/chrono.h>
 #include <ripple/nodestore/Database.h>
 
@@ -133,12 +134,6 @@ public:
     void
     sweep() override;
 
-    Backend&
-    getBackend() override
-    {
-        return *backend_;
-    };
-
 private:
     // Cache for database objects. This cache is not always initialized. Check
     // for null before using.
@@ -157,6 +152,12 @@ private:
     {
         backend_->for_each(f);
     }
+
+    std::optional<Backend::Counters<std::uint64_t>>
+    getCounters() const override
+    {
+        return backend_->counters();
+    }
 };
 
 }  // namespace NodeStore

src/ripple/nodestore/impl/DatabaseRotatingImp.cpp

@@ -93,6 +93,13 @@ DatabaseRotatingImp::storeLedger(std::shared_ptr<Ledger const> const& srcLedger)
     return Database::storeLedger(*srcLedger, backend);
 }
 
+void
+DatabaseRotatingImp::sync()
+{
+    std::lock_guard lock(mutex_);
+    writableBackend_->sync();
+}
+
 void
 DatabaseRotatingImp::store(
     NodeObjectType type,

src/ripple/nodestore/impl/DatabaseRotatingImp.h

@@ -74,10 +74,7 @@ public:
         override;
 
     void
-    sync() override
+    sync() override;
-    {
-        writableBackend_->sync();
-    }
 
     bool
     storeLedger(std::shared_ptr<Ledger const> const& srcLedger) override;
@@ -85,12 +82,6 @@ public:
     void
     sweep() override;
 
-    Backend&
-    getBackend() override
-    {
-        return *writableBackend_;
-    }
-
 private:
     std::shared_ptr<Backend> writableBackend_;
     std::shared_ptr<Backend> archiveBackend_;
@@ -102,13 +93,6 @@ private:
         std::shared_ptr<Backend> const& archiveBackend;
     };
 
-    Backends
-    getBackends() const
-    {
-        std::lock_guard lock(mutex_);
-        return Backends{writableBackend_, archiveBackend_};
-    }
-
     std::shared_ptr<NodeObject>
     fetchNodeObject(
         uint256 const& hash,

src/ripple/nodestore/impl/DatabaseShardImp.cpp

@@ -540,12 +540,6 @@ DatabaseShardImp::importShard(
     return true;
 }
 
-Backend&
-DatabaseShardImp::getBackend()
-{
-    return app_.getNodeStore().getBackend();
-}
-
 std::shared_ptr<Ledger>
 DatabaseShardImp::fetchLedger(uint256 const& hash, std::uint32_t ledgerSeq)
 {

src/ripple/nodestore/impl/DatabaseShardImp.h

@@ -161,9 +161,6 @@ public:
     void
     sweep() override;
 
-    Backend&
-    getBackend() override;
-
 private:
     enum class PathDesignation : uint8_t {
         none,       // No path specified