diff --git a/cfg/xrpld-example.cfg b/cfg/xrpld-example.cfg index effc62c274..9e334e6f4f 100644 --- a/cfg/xrpld-example.cfg +++ b/cfg/xrpld-example.cfg @@ -953,6 +953,21 @@ # # Optional keys for NuDB and RocksDB: # +# cache_size Size of cache for database records. Default is 16384. +# Setting this value to 0 will use the default value. +# +# cache_age Length of time in minutes to keep database records +# cached. Default is 5 minutes. Setting this value to +# 0 will use the default value. +# +# Note: if cache_size or cache_age is not specified, +# default values will be used for the unspecified +# parameter. +# +# Note: the cache will not be created if online_delete +# is specified, because the rotating NodeStore does +# not use this cache). +# # fast_load Boolean. If set, load the last persisted ledger # from disk upon process start before syncing to # the network. This is likely to improve performance diff --git a/include/xrpl/nodestore/Database.h b/include/xrpl/nodestore/Database.h index ca2dde560c..438a3cc7fc 100644 --- a/include/xrpl/nodestore/Database.h +++ b/include/xrpl/nodestore/Database.h @@ -131,6 +131,10 @@ public: std::uint32_t ledgerSeq, std::function const&)>&& callback); + /** Remove expired entries from the positive and negative caches. */ + virtual void + sweep() = 0; + /** Gather statistics pertaining to read and write activities. * * @param obj Json object reference into which to place counters. diff --git a/include/xrpl/nodestore/detail/DatabaseNodeImp.h b/include/xrpl/nodestore/detail/DatabaseNodeImp.h index dd94b27075..951c60c8c7 100644 --- a/include/xrpl/nodestore/detail/DatabaseNodeImp.h +++ b/include/xrpl/nodestore/detail/DatabaseNodeImp.h @@ -22,6 +22,32 @@ public: beast::Journal j) : Database(scheduler, readThreads, config, j), backend_(std::move(backend)) { + std::optional cacheSize, cacheAge; + + if (config.exists("cache_size")) + { + cacheSize = get(config, "cache_size"); + if (cacheSize.value() < 0) + Throw("Specified negative value for cache_size"); + } + + if (config.exists("cache_age")) + { + cacheAge = get(config, "cache_age"); + if (cacheAge.value() < 0) + Throw("Specified negative value for cache_age"); + } + + if (cacheSize.has_value() || cacheAge.has_value()) + { + cache_ = std::make_shared>( + "DatabaseNodeImp", + cacheSize.value_or(0), + std::chrono::minutes(cacheAge.value_or(0)), + stopwatch(), + j); + } + XRPL_ASSERT( backend_, "xrpl::NodeStore::DatabaseNodeImp::DatabaseNodeImp : non-null " @@ -73,7 +99,13 @@ public: std::uint32_t ledgerSeq, std::function const&)>&& callback) override; + void + sweep() override; + private: + // Cache for database objects. This cache is not always initialized. Check + // for null before using. + std::shared_ptr> cache_; // Persistent key/value storage std::shared_ptr backend_; diff --git a/include/xrpl/nodestore/detail/DatabaseRotatingImp.h b/include/xrpl/nodestore/detail/DatabaseRotatingImp.h index 39441ef4d8..1ba9435a5f 100644 --- a/include/xrpl/nodestore/detail/DatabaseRotatingImp.h +++ b/include/xrpl/nodestore/detail/DatabaseRotatingImp.h @@ -55,6 +55,9 @@ public: void sync() override; + void + sweep() override; + private: std::shared_ptr writableBackend_; std::shared_ptr archiveBackend_; diff --git a/src/libxrpl/nodestore/DatabaseNodeImp.cpp b/src/libxrpl/nodestore/DatabaseNodeImp.cpp index a51c34079b..9323d69131 100644 --- a/src/libxrpl/nodestore/DatabaseNodeImp.cpp +++ b/src/libxrpl/nodestore/DatabaseNodeImp.cpp @@ -24,6 +24,13 @@ DatabaseNodeImp::store(NodeObjectType type, Blob&& data, uint256 const& hash, st auto obj = NodeObject::createObject(type, std::move(data), hash); backend_->store(obj); + if (cache_) + { + // After the store, replace a negative cache entry if there is one + cache_->canonicalize(hash, obj, [](std::shared_ptr const& n) { + return n->getType() == NodeObjectType::Dummy; + }); + } } void @@ -32,9 +39,25 @@ DatabaseNodeImp::asyncFetch( std::uint32_t ledgerSeq, std::function const&)>&& callback) { + if (cache_) + { + std::shared_ptr const obj = cache_->fetch(hash); + if (obj) + { + callback(obj->getType() == NodeObjectType::Dummy ? nullptr : obj); + return; + } + } Database::asyncFetch(hash, ledgerSeq, std::move(callback)); } +void +DatabaseNodeImp::sweep() +{ + if (cache_) + cache_->sweep(); +} + std::shared_ptr DatabaseNodeImp::fetchNodeObject( uint256 const& hash, @@ -42,32 +65,58 @@ DatabaseNodeImp::fetchNodeObject( FetchReport& fetchReport, bool duplicate) { - std::shared_ptr nodeObject = nullptr; - Status status = Status::Ok; + std::shared_ptr nodeObject = cache_ ? cache_->fetch(hash) : nullptr; + if (!nodeObject) + { + JLOG(j_.trace()) << "fetchNodeObject " << hash << ": record not " + << (cache_ ? "cached" : "found"); - try - { - status = backend_->fetch(hash, &nodeObject); - } - catch (std::exception const& e) - { - JLOG(j_.fatal()) << "fetchNodeObject " << hash - << ": Exception fetching from backend: " << e.what(); - rethrow(); - } + Status status = Status::Ok; + try + { + status = backend_->fetch(hash, &nodeObject); + } + catch (std::exception const& e) + { + JLOG(j_.fatal()) << "fetchNodeObject " << hash + << ": Exception fetching from backend: " << e.what(); + rethrow(); + } - switch (status) + switch (status) + { + case Status::Ok: + if (cache_) + { + if (nodeObject) + { + cache_->canonicalizeReplaceClient(hash, nodeObject); + } + else + { + auto notFound = NodeObject::createObject(NodeObjectType::Dummy, {}, hash); + cache_->canonicalizeReplaceClient(hash, notFound); + if (notFound->getType() != NodeObjectType::Dummy) + nodeObject = notFound; + } + } + break; + case Status::NotFound: + break; + case Status::DataCorrupt: + JLOG(j_.fatal()) << "fetchNodeObject " << hash << ": nodestore data is corrupted"; + break; + default: + JLOG(j_.warn()) << "fetchNodeObject " << hash << ": backend returns unknown result " + << static_cast(status); + break; + } + } + else { - case Status::Ok: - case Status::NotFound: - break; - case Status::DataCorrupt: - JLOG(j_.fatal()) << "fetchNodeObject " << hash << ": nodestore data is corrupted"; - break; - default: - JLOG(j_.warn()) << "fetchNodeObject " << hash << ": backend returns unknown result " - << static_cast(status); - break; + JLOG(j_.trace()) << "fetchNodeObject " << hash << ": record found in cache"; + if (nodeObject->getType() == NodeObjectType::Dummy) + nodeObject.reset(); } if (nodeObject) diff --git a/src/libxrpl/nodestore/DatabaseRotatingImp.cpp b/src/libxrpl/nodestore/DatabaseRotatingImp.cpp index 24b0e2de2e..1de6a83b4c 100644 --- a/src/libxrpl/nodestore/DatabaseRotatingImp.cpp +++ b/src/libxrpl/nodestore/DatabaseRotatingImp.cpp @@ -113,6 +113,12 @@ DatabaseRotatingImp::store(NodeObjectType type, Blob&& data, uint256 const& hash storeStats(1, nObj->getData().size()); } +void +DatabaseRotatingImp::sweep() +{ + // Nothing to do. +} + std::shared_ptr DatabaseRotatingImp::fetchNodeObject( uint256 const& hash, diff --git a/src/test/app/SHAMapStore_test.cpp b/src/test/app/SHAMapStore_test.cpp index fc5ef02465..6e279eadb2 100644 --- a/src/test/app/SHAMapStore_test.cpp +++ b/src/test/app/SHAMapStore_test.cpp @@ -520,6 +520,25 @@ public: ///////////////////////////////////////////////////////////// // Create NodeStore with two backends to allow online deletion of data. // Normally, SHAMapStoreImp handles all these details. + auto nscfg = env.app().config().section(ConfigSection::nodeDatabase()); + + // Provide default values. + if (!nscfg.exists("cache_size")) + { + nscfg.set( + "cache_size", + std::to_string( + env.app().config().getValueFor(SizedItem::TreeCacheSize, std::nullopt))); + } + + if (!nscfg.exists("cache_age")) + { + nscfg.set( + "cache_age", + std::to_string( + env.app().config().getValueFor(SizedItem::TreeCacheAge, std::nullopt))); + } + NodeStoreScheduler scheduler(env.app().getJobQueue()); std::string const writableDb = "write"; @@ -528,7 +547,6 @@ public: auto archiveBackend = makeBackendRotating(env, scheduler, archiveDb); static constexpr int kReadThreads = 4; - auto nscfg = env.app().config().section(ConfigSection::nodeDatabase()); auto dbr = std::make_unique( scheduler, kReadThreads, diff --git a/src/xrpld/app/main/Application.cpp b/src/xrpld/app/main/Application.cpp index a18462f0f7..508dfc8590 100644 --- a/src/xrpld/app/main/Application.cpp +++ b/src/xrpld/app/main/Application.cpp @@ -998,6 +998,10 @@ public: JLOG(journal_.debug()) << "MasterTransaction sweep. Size before: " << oldMasterTxSize << "; size after: " << masterTxCache.size(); } + { + // Sweep NodeStore database cache(s), if enabled. + getNodeStore().sweep(); + } { std::size_t const oldLedgerMasterCacheSize = getLedgerMaster().getFetchPackCacheSize(); diff --git a/src/xrpld/app/misc/SHAMapStoreImp.cpp b/src/xrpld/app/misc/SHAMapStoreImp.cpp index fc3f71c0cc..3f4bd6280f 100644 --- a/src/xrpld/app/misc/SHAMapStoreImp.cpp +++ b/src/xrpld/app/misc/SHAMapStoreImp.cpp @@ -165,6 +165,22 @@ std::unique_ptr SHAMapStoreImp::makeNodeStore(int readThreads) { auto nscfg = app_.config().section(ConfigSection::nodeDatabase()); + + // Provide default values. + if (!nscfg.exists("cache_size")) + { + nscfg.set( + "cache_size", + std::to_string(app_.config().getValueFor(SizedItem::TreeCacheSize, std::nullopt))); + } + + if (!nscfg.exists("cache_age")) + { + nscfg.set( + "cache_age", + std::to_string(app_.config().getValueFor(SizedItem::TreeCacheAge, std::nullopt))); + } + std::unique_ptr db; if (deleteInterval_ != 0u) @@ -254,6 +270,8 @@ SHAMapStoreImp::run() LedgerIndex lastRotated = stateDb_.getState().lastRotated; netOPs_ = &app_.getOPs(); ledgerMaster_ = &app_.getLedgerMaster(); + fullBelowCache_ = &(*app_.getNodeFamily().getFullBelowCache()); + treeNodeCache_ = &(*app_.getNodeFamily().getTreeNodeCache()); if (advisoryDelete_) canDelete_ = stateDb_.getCanDelete(); @@ -542,16 +560,16 @@ SHAMapStoreImp::clearCaches(LedgerIndex validatedSeq) // Also clear the FullBelowCache so its generation counter is bumped. // This prevents stale "full below" markers from persisting across // backend rotation/online deletion and interfering with SHAMap sync. - app_.getNodeFamily().getFullBelowCache()->clear(); + fullBelowCache_->clear(); } void SHAMapStoreImp::freshenCaches() { - if (freshenCache(*app_.getNodeFamily().getTreeNodeCache())) + if (freshenCache(*treeNodeCache_)) + return; + if (freshenCache(app_.getMasterTransaction().getCache())) return; - - freshenCache(app_.getMasterTransaction().getCache()); } void diff --git a/src/xrpld/app/misc/SHAMapStoreImp.h b/src/xrpld/app/misc/SHAMapStoreImp.h index 4f0ce04e0d..1803c15e71 100644 --- a/src/xrpld/app/misc/SHAMapStoreImp.h +++ b/src/xrpld/app/misc/SHAMapStoreImp.h @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include @@ -93,6 +95,8 @@ private: // as of run() or before NetworkOPs* netOPs_ = nullptr; LedgerMaster* ledgerMaster_ = nullptr; + FullBelowCache* fullBelowCache_ = nullptr; + TreeNodeCache* treeNodeCache_ = nullptr; static constexpr auto kNodeStoreName = "NodeStore";