From 8c00fd10d1c1f4ad598729eb6a91c0d6283124a4 Mon Sep 17 00:00:00 2001 From: Denis Angell Date: Mon, 11 Nov 2024 21:26:27 +0100 Subject: [PATCH] [fold] rename `MemoryFactory` -> `MemDBFactory` --- Builds/CMake/RippledCore.cmake | 2 +- .../app/rdb/impl/RelationalDatabase.cpp | 8 +- src/ripple/core/Config.h | 6 +- src/ripple/nodestore/backend/MemDBFactory.cpp | 240 +++++++++++++++++ .../nodestore/backend/MemoryFactory.cpp | 235 ++++++++--------- .../nodestore/backend/TestMemoryFactory.cpp | 243 ------------------ src/test/jtx/impl/envconfig.cpp | 2 +- src/test/nodestore/Backend_test.cpp | 2 +- src/test/nodestore/Database_test.cpp | 4 +- src/test/shamap/common.h | 2 +- 10 files changed, 371 insertions(+), 373 deletions(-) create mode 100644 src/ripple/nodestore/backend/MemDBFactory.cpp delete mode 100644 src/ripple/nodestore/backend/TestMemoryFactory.cpp diff --git a/Builds/CMake/RippledCore.cmake b/Builds/CMake/RippledCore.cmake index dea2600e3..18f85b0b9 100644 --- a/Builds/CMake/RippledCore.cmake +++ b/Builds/CMake/RippledCore.cmake @@ -538,8 +538,8 @@ target_sources (rippled PRIVATE subdir: nodestore #]===============================] src/ripple/nodestore/backend/CassandraFactory.cpp + src/ripple/nodestore/backend/MemDBFactory.cpp src/ripple/nodestore/backend/MemoryFactory.cpp - src/ripple/nodestore/backend/TestMemoryFactory.cpp src/ripple/nodestore/backend/FlatmapFactory.cpp src/ripple/nodestore/backend/NuDBFactory.cpp src/ripple/nodestore/backend/NullFactory.cpp diff --git a/src/ripple/app/rdb/impl/RelationalDatabase.cpp b/src/ripple/app/rdb/impl/RelationalDatabase.cpp index 8c51db2d9..2b0625e6c 100644 --- a/src/ripple/app/rdb/impl/RelationalDatabase.cpp +++ b/src/ripple/app/rdb/impl/RelationalDatabase.cpp @@ -40,7 +40,7 @@ RelationalDatabase::init( { bool use_sqlite = false; bool use_postgres = false; - bool use_memory = false; + bool use_memdb = false; bool use_flatmap = false; if (config.reporting()) @@ -56,9 +56,9 @@ RelationalDatabase::init( { use_sqlite = true; } - else if (boost::iequals(get(rdb_section, "backend"), "memory")) + else if (boost::iequals(get(rdb_section, "backend"), "memdb")) { - use_memory = true; + use_memdb = true; } else if (boost::iequals(get(rdb_section, "backend"), "flatmap")) { @@ -85,7 +85,7 @@ RelationalDatabase::init( { return getPostgresDatabase(app, config, jobQueue); } - else if (use_memory) + else if (use_memdb) { return getMemoryDatabase(app, config, jobQueue); } diff --git a/src/ripple/core/Config.h b/src/ripple/core/Config.h index a98d6d9aa..20ef91b93 100644 --- a/src/ripple/core/Config.h +++ b/src/ripple/core/Config.h @@ -357,13 +357,13 @@ public: static bool const isMem = (!section(SECTION_RELATIONAL_DB).empty() && boost::beast::iequals( - get(section(SECTION_RELATIONAL_DB), "backend"), "memory")) || + get(section(SECTION_RELATIONAL_DB), "backend"), "memdb")) || (!section("node_db").empty() && (boost::beast::iequals( - get(section("node_db"), "type"), "memory") || + get(section("node_db"), "type"), "memdb") || boost::beast::iequals( get(section("node_db"), "type"), "flatmap"))); - // RHNOTE: testmemory type is not selected for here because it breaks + // RHNOTE: memory type is not selected for here because it breaks // tests return isMem; } diff --git a/src/ripple/nodestore/backend/MemDBFactory.cpp b/src/ripple/nodestore/backend/MemDBFactory.cpp new file mode 100644 index 000000000..aa354cf76 --- /dev/null +++ b/src/ripple/nodestore/backend/MemDBFactory.cpp @@ -0,0 +1,240 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ripple { +namespace NodeStore { + +class MemDBBackend : public Backend +{ +private: + std::string name_; + beast::Journal journal_; + bool isOpen_{false}; + + struct base_uint_hasher + { + using result_type = std::size_t; + + result_type + operator()(base_uint<256> const& value) const + { + return hardened_hash<>{}(value); + } + }; + + using DataStore = + std::map>; // Store compressed blob + // data + mutable std::recursive_mutex + mutex_; // Only needed for std::map implementation + + DataStore table_; + +public: + MemDBBackend( + size_t keyBytes, + Section const& keyValues, + beast::Journal journal) + : name_(get(keyValues, "path")), journal_(journal) + { + boost::ignore_unused(journal_); + if (name_.empty()) + name_ = "node_db"; + } + + ~MemDBBackend() override + { + close(); + } + + std::string + getName() override + { + return name_; + } + + void + open(bool createIfMissing) override + { + std::lock_guard lock(mutex_); + if (isOpen_) + Throw("already open"); + isOpen_ = true; + } + + bool + isOpen() override + { + return isOpen_; + } + + void + close() override + { + std::lock_guard lock(mutex_); + table_.clear(); + isOpen_ = false; + } + + Status + fetch(void const* key, std::shared_ptr* pObject) override + { + if (!isOpen_) + return notFound; + + uint256 const hash(uint256::fromVoid(key)); + + std::lock_guard lock(mutex_); + auto it = table_.find(hash); + if (it == table_.end()) + return notFound; + + nudb::detail::buffer bf; + auto const result = + nodeobject_decompress(it->second.data(), it->second.size(), bf); + DecodedBlob decoded(hash.data(), result.first, result.second); + if (!decoded.wasOk()) + return dataCorrupt; + *pObject = decoded.createObject(); + return ok; + } + + std::pair>, Status> + fetchBatch(std::vector const& hashes) override + { + std::vector> results; + results.reserve(hashes.size()); + for (auto const& h : hashes) + { + std::shared_ptr nObj; + Status status = fetch(h->begin(), &nObj); + if (status != ok) + results.push_back({}); + else + results.push_back(nObj); + } + return {results, ok}; + } + + void + store(std::shared_ptr const& object) override + { + if (!isOpen_) + return; + + if (!object) + return; + + EncodedBlob encoded(object); + nudb::detail::buffer bf; + auto const result = + nodeobject_compress(encoded.getData(), encoded.getSize(), bf); + + std::vector compressed( + static_cast(result.first), + static_cast(result.first) + result.second); + + std::lock_guard lock(mutex_); + table_[object->getHash()] = std::move(compressed); + } + + void + storeBatch(Batch const& batch) override + { + for (auto const& e : batch) + store(e); + } + + void + sync() override + { + } + + void + for_each(std::function)> f) override + { + if (!isOpen_) + return; + + std::lock_guard lock(mutex_); + for (const auto& entry : table_) + { + nudb::detail::buffer bf; + auto const result = nodeobject_decompress( + entry.second.data(), entry.second.size(), bf); + DecodedBlob decoded( + entry.first.data(), result.first, result.second); + if (decoded.wasOk()) + f(decoded.createObject()); + } + } + + int + getWriteLoad() override + { + return 0; + } + + void + setDeletePath() override + { + close(); + } + + int + fdRequired() const override + { + return 0; + } + +private: + size_t + size() const + { + std::lock_guard lock(mutex_); + return table_.size(); + } +}; + +class MemDBFactory : public Factory +{ +public: + MemDBFactory() + { + Manager::instance().insert(*this); + } + + ~MemDBFactory() override + { + Manager::instance().erase(*this); + } + + std::string + getName() const override + { + return "MemDB"; + } + + std::unique_ptr + createInstance( + size_t keyBytes, + Section const& keyValues, + std::size_t burstSize, + Scheduler& scheduler, + beast::Journal journal) override + { + return std::make_unique(keyBytes, keyValues, journal); + } +}; + +} // namespace NodeStore +} // namespace ripple diff --git a/src/ripple/nodestore/backend/MemoryFactory.cpp b/src/ripple/nodestore/backend/MemoryFactory.cpp index a47329b32..a1c7a4cfb 100644 --- a/src/ripple/nodestore/backend/MemoryFactory.cpp +++ b/src/ripple/nodestore/backend/MemoryFactory.cpp @@ -1,43 +1,89 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + #include #include #include -#include -#include -#include #include #include -#include +#include #include #include namespace ripple { namespace NodeStore { +struct MemoryDB +{ + explicit MemoryDB() = default; + + std::mutex mutex; + bool open = false; + std::map> table; +}; + +class MemoryFactory : public Factory +{ +private: + std::mutex mutex_; + std::map map_; + +public: + MemoryFactory(); + ~MemoryFactory() override; + + std::string + getName() const override; + + std::unique_ptr + createInstance( + size_t keyBytes, + Section const& keyValues, + std::size_t burstSize, + Scheduler& scheduler, + beast::Journal journal) override; + + MemoryDB& + open(std::string const& path) + { + std::lock_guard _(mutex_); + auto const result = map_.emplace( + std::piecewise_construct, std::make_tuple(path), std::make_tuple()); + MemoryDB& db = result.first->second; + if (db.open) + Throw("already open"); + return db; + } +}; + +static MemoryFactory memoryFactory; + +//------------------------------------------------------------------------------ + class MemoryBackend : public Backend { private: + using Map = std::map>; + std::string name_; - beast::Journal journal_; - bool isOpen_{false}; - - struct base_uint_hasher - { - using result_type = std::size_t; - - result_type - operator()(base_uint<256> const& value) const - { - return hardened_hash<>{}(value); - } - }; - - using DataStore = - std::map>; // Store compressed blob - // data - mutable std::recursive_mutex - mutex_; // Only needed for std::map implementation - - DataStore table_; + beast::Journal const journal_; + MemoryDB* db_{nullptr}; public: MemoryBackend( @@ -46,9 +92,9 @@ public: beast::Journal journal) : name_(get(keyValues, "path")), journal_(journal) { - boost::ignore_unused(journal_); + boost::ignore_unused(journal_); // Keep unused journal_ just in case. if (name_.empty()) - name_ = "node_db"; + Throw("Missing path in TestMemory backend"); } ~MemoryBackend() override @@ -65,46 +111,38 @@ public: void open(bool createIfMissing) override { - std::lock_guard lock(mutex_); - if (isOpen_) - Throw("already open"); - isOpen_ = true; + db_ = &memoryFactory.open(name_); } bool isOpen() override { - return isOpen_; + return static_cast(db_); } void close() override { - std::lock_guard lock(mutex_); - table_.clear(); - isOpen_ = false; + db_ = nullptr; } + //-------------------------------------------------------------------------- + Status fetch(void const* key, std::shared_ptr* pObject) override { - if (!isOpen_) - return notFound; - + assert(db_); uint256 const hash(uint256::fromVoid(key)); - std::lock_guard lock(mutex_); - auto it = table_.find(hash); - if (it == table_.end()) - return notFound; + std::lock_guard _(db_->mutex); - nudb::detail::buffer bf; - auto const result = - nodeobject_decompress(it->second.data(), it->second.size(), bf); - DecodedBlob decoded(hash.data(), result.first, result.second); - if (!decoded.wasOk()) - return dataCorrupt; - *pObject = decoded.createObject(); + Map::iterator iter = db_->table.find(hash); + if (iter == db_->table.end()) + { + pObject->reset(); + return notFound; + } + *pObject = iter->second; return ok; } @@ -122,29 +160,16 @@ public: else results.push_back(nObj); } + return {results, ok}; } void store(std::shared_ptr const& object) override { - if (!isOpen_) - return; - - if (!object) - return; - - EncodedBlob encoded(object); - nudb::detail::buffer bf; - auto const result = - nodeobject_compress(encoded.getData(), encoded.getSize(), bf); - - std::vector compressed( - static_cast(result.first), - static_cast(result.first) + result.second); - - std::lock_guard lock(mutex_); - table_[object->getHash()] = std::move(compressed); + assert(db_); + std::lock_guard _(db_->mutex); + db_->table.emplace(object->getHash(), object); } void @@ -162,20 +187,9 @@ public: void for_each(std::function)> f) override { - if (!isOpen_) - return; - - std::lock_guard lock(mutex_); - for (const auto& entry : table_) - { - nudb::detail::buffer bf; - auto const result = nodeobject_decompress( - entry.second.data(), entry.second.size(), bf); - DecodedBlob decoded( - entry.first.data(), result.first, result.second); - if (decoded.wasOk()) - f(decoded.createObject()); - } + assert(db_); + for (auto const& e : db_->table) + f(e.second); } int @@ -187,7 +201,6 @@ public: void setDeletePath() override { - close(); } int @@ -195,48 +208,36 @@ public: { return 0; } - -private: - size_t - size() const - { - std::lock_guard lock(mutex_); - return table_.size(); - } }; -class MemoryFactory : public Factory +//------------------------------------------------------------------------------ + +MemoryFactory::MemoryFactory() { -public: - MemoryFactory() - { - Manager::instance().insert(*this); - } + Manager::instance().insert(*this); +} - ~MemoryFactory() override - { - Manager::instance().erase(*this); - } +MemoryFactory::~MemoryFactory() +{ + Manager::instance().erase(*this); +} - std::string - getName() const override - { - return "Memory"; - } +std::string +MemoryFactory::getName() const +{ + return "Memory"; +} - std::unique_ptr - createInstance( - size_t keyBytes, - Section const& keyValues, - std::size_t burstSize, - Scheduler& scheduler, - beast::Journal journal) override - { - return std::make_unique(keyBytes, keyValues, journal); - } -}; - -static MemoryFactory memoryFactory; +std::unique_ptr +MemoryFactory::createInstance( + size_t keyBytes, + Section const& keyValues, + std::size_t, + Scheduler& scheduler, + beast::Journal journal) +{ + return std::make_unique(keyBytes, keyValues, journal); +} } // namespace NodeStore } // namespace ripple diff --git a/src/ripple/nodestore/backend/TestMemoryFactory.cpp b/src/ripple/nodestore/backend/TestMemoryFactory.cpp deleted file mode 100644 index cef57a1c9..000000000 --- a/src/ripple/nodestore/backend/TestMemoryFactory.cpp +++ /dev/null @@ -1,243 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ripple { -namespace NodeStore { - -struct TestMemoryDB -{ - explicit TestMemoryDB() = default; - - std::mutex mutex; - bool open = false; - std::map> table; -}; - -class TestMemoryFactory : public Factory -{ -private: - std::mutex mutex_; - std::map map_; - -public: - TestMemoryFactory(); - ~TestMemoryFactory() override; - - std::string - getName() const override; - - std::unique_ptr - createInstance( - size_t keyBytes, - Section const& keyValues, - std::size_t burstSize, - Scheduler& scheduler, - beast::Journal journal) override; - - TestMemoryDB& - open(std::string const& path) - { - std::lock_guard _(mutex_); - auto const result = map_.emplace( - std::piecewise_construct, std::make_tuple(path), std::make_tuple()); - TestMemoryDB& db = result.first->second; - if (db.open) - Throw("already open"); - return db; - } -}; - -static TestMemoryFactory testMemoryFactory; - -//------------------------------------------------------------------------------ - -class TestMemoryBackend : public Backend -{ -private: - using Map = std::map>; - - std::string name_; - beast::Journal const journal_; - TestMemoryDB* db_{nullptr}; - -public: - TestMemoryBackend( - size_t keyBytes, - Section const& keyValues, - beast::Journal journal) - : name_(get(keyValues, "path")), journal_(journal) - { - boost::ignore_unused(journal_); // Keep unused journal_ just in case. - if (name_.empty()) - Throw("Missing path in TestMemory backend"); - } - - ~TestMemoryBackend() override - { - close(); - } - - std::string - getName() override - { - return name_; - } - - void - open(bool createIfMissing) override - { - db_ = &testMemoryFactory.open(name_); - } - - bool - isOpen() override - { - return static_cast(db_); - } - - void - close() override - { - db_ = nullptr; - } - - //-------------------------------------------------------------------------- - - Status - fetch(void const* key, std::shared_ptr* pObject) override - { - assert(db_); - uint256 const hash(uint256::fromVoid(key)); - - std::lock_guard _(db_->mutex); - - Map::iterator iter = db_->table.find(hash); - if (iter == db_->table.end()) - { - pObject->reset(); - return notFound; - } - *pObject = iter->second; - return ok; - } - - std::pair>, Status> - fetchBatch(std::vector const& hashes) override - { - std::vector> results; - results.reserve(hashes.size()); - for (auto const& h : hashes) - { - std::shared_ptr nObj; - Status status = fetch(h->begin(), &nObj); - if (status != ok) - results.push_back({}); - else - results.push_back(nObj); - } - - return {results, ok}; - } - - void - store(std::shared_ptr const& object) override - { - assert(db_); - std::lock_guard _(db_->mutex); - db_->table.emplace(object->getHash(), object); - } - - void - storeBatch(Batch const& batch) override - { - for (auto const& e : batch) - store(e); - } - - void - sync() override - { - } - - void - for_each(std::function)> f) override - { - assert(db_); - for (auto const& e : db_->table) - f(e.second); - } - - int - getWriteLoad() override - { - return 0; - } - - void - setDeletePath() override - { - } - - int - fdRequired() const override - { - return 0; - } -}; - -//------------------------------------------------------------------------------ - -TestMemoryFactory::TestMemoryFactory() -{ - Manager::instance().insert(*this); -} - -TestMemoryFactory::~TestMemoryFactory() -{ - Manager::instance().erase(*this); -} - -std::string -TestMemoryFactory::getName() const -{ - return "TestMemory"; -} - -std::unique_ptr -TestMemoryFactory::createInstance( - size_t keyBytes, - Section const& keyValues, - std::size_t, - Scheduler& scheduler, - beast::Journal journal) -{ - return std::make_unique(keyBytes, keyValues, journal); -} - -} // namespace NodeStore -} // namespace ripple diff --git a/src/test/jtx/impl/envconfig.cpp b/src/test/jtx/impl/envconfig.cpp index de5d91272..13999922a 100644 --- a/src/test/jtx/impl/envconfig.cpp +++ b/src/test/jtx/impl/envconfig.cpp @@ -49,7 +49,7 @@ setupConfigForUnitTests(Config& cfg) cfg.FEES.account_reserve = XRP(200).value().xrp().drops(); cfg.FEES.owner_reserve = XRP(50).value().xrp().drops(); - cfg.overwrite(ConfigSection::nodeDatabase(), "type", "testmemory"); + cfg.overwrite(ConfigSection::nodeDatabase(), "type", "memory"); cfg.overwrite(ConfigSection::nodeDatabase(), "path", "main"); cfg.deprecatedClearSection(ConfigSection::importNodeDatabase()); cfg.legacy("database_path", ""); diff --git a/src/test/nodestore/Backend_test.cpp b/src/test/nodestore/Backend_test.cpp index 5fdcc188f..ef6b99c99 100644 --- a/src/test/nodestore/Backend_test.cpp +++ b/src/test/nodestore/Backend_test.cpp @@ -107,8 +107,8 @@ public: { std::uint64_t const seedValue = 50; - testBackend("testmemory", seedValue); testBackend("memory", seedValue); + testBackend("memdb", seedValue); testBackend("nudb", seedValue); #if RIPPLE_ROCKSDB_AVAILABLE diff --git a/src/test/nodestore/Database_test.cpp b/src/test/nodestore/Database_test.cpp index 0ce61067f..87ae5b148 100644 --- a/src/test/nodestore/Database_test.cpp +++ b/src/test/nodestore/Database_test.cpp @@ -659,10 +659,10 @@ public: testConfig(); - testNodeStore("testmemory", false, seedValue); - testNodeStore("memory", false, seedValue); + testNodeStore("memdb", false, seedValue); + // Persistent backend tests { testNodeStore("nudb", true, seedValue); diff --git a/src/test/shamap/common.h b/src/test/shamap/common.h index 865584084..d89acb988 100644 --- a/src/test/shamap/common.h +++ b/src/test/shamap/common.h @@ -57,7 +57,7 @@ public: , j_(j) { Section testSection; - testSection.set("type", "testmemory"); + testSection.set("type", "memory"); testSection.set("path", "SHAMap_test"); db_ = NodeStore::Manager::instance().make_Database( megabytes(4), scheduler_, 1, testSection, j);