[fold] rename MemoryFactory -> MemDBFactory

This commit is contained in:
Denis Angell
2024-11-11 21:26:27 +01:00
parent febdb6a997
commit 8c00fd10d1
10 changed files with 371 additions and 373 deletions

View File

@@ -538,8 +538,8 @@ target_sources (rippled PRIVATE
subdir: nodestore subdir: nodestore
#]===============================] #]===============================]
src/ripple/nodestore/backend/CassandraFactory.cpp src/ripple/nodestore/backend/CassandraFactory.cpp
src/ripple/nodestore/backend/MemDBFactory.cpp
src/ripple/nodestore/backend/MemoryFactory.cpp src/ripple/nodestore/backend/MemoryFactory.cpp
src/ripple/nodestore/backend/TestMemoryFactory.cpp
src/ripple/nodestore/backend/FlatmapFactory.cpp src/ripple/nodestore/backend/FlatmapFactory.cpp
src/ripple/nodestore/backend/NuDBFactory.cpp src/ripple/nodestore/backend/NuDBFactory.cpp
src/ripple/nodestore/backend/NullFactory.cpp src/ripple/nodestore/backend/NullFactory.cpp

View File

@@ -40,7 +40,7 @@ RelationalDatabase::init(
{ {
bool use_sqlite = false; bool use_sqlite = false;
bool use_postgres = false; bool use_postgres = false;
bool use_memory = false; bool use_memdb = false;
bool use_flatmap = false; bool use_flatmap = false;
if (config.reporting()) if (config.reporting())
@@ -56,9 +56,9 @@ RelationalDatabase::init(
{ {
use_sqlite = true; use_sqlite = true;
} }
else if (boost::iequals(get(rdb_section, "backend"), "memory")) else if (boost::iequals(get(rdb_section, "backend"), "memdb"))
{ {
use_memory = true; use_memdb = true;
} }
else if (boost::iequals(get(rdb_section, "backend"), "flatmap")) else if (boost::iequals(get(rdb_section, "backend"), "flatmap"))
{ {
@@ -85,7 +85,7 @@ RelationalDatabase::init(
{ {
return getPostgresDatabase(app, config, jobQueue); return getPostgresDatabase(app, config, jobQueue);
} }
else if (use_memory) else if (use_memdb)
{ {
return getMemoryDatabase(app, config, jobQueue); return getMemoryDatabase(app, config, jobQueue);
} }

View File

@@ -357,13 +357,13 @@ public:
static bool const isMem = static bool const isMem =
(!section(SECTION_RELATIONAL_DB).empty() && (!section(SECTION_RELATIONAL_DB).empty() &&
boost::beast::iequals( boost::beast::iequals(
get(section(SECTION_RELATIONAL_DB), "backend"), "memory")) || get(section(SECTION_RELATIONAL_DB), "backend"), "memdb")) ||
(!section("node_db").empty() && (!section("node_db").empty() &&
(boost::beast::iequals( (boost::beast::iequals(
get(section("node_db"), "type"), "memory") || get(section("node_db"), "type"), "memdb") ||
boost::beast::iequals( boost::beast::iequals(
get(section("node_db"), "type"), "flatmap"))); get(section("node_db"), "type"), "flatmap")));
// RHNOTE: testmemory type is not selected for here because it breaks // RHNOTE: memory type is not selected for here because it breaks
// tests // tests
return isMem; return isMem;
} }

View File

@@ -0,0 +1,240 @@
#include <ripple/basics/contract.h>
#include <ripple/nodestore/Factory.h>
#include <ripple/nodestore/Manager.h>
#include <ripple/nodestore/impl/DecodedBlob.h>
#include <ripple/nodestore/impl/EncodedBlob.h>
#include <ripple/nodestore/impl/codec.h>
#include <boost/beast/core/string.hpp>
#include <boost/core/ignore_unused.hpp>
#include <boost/unordered/concurrent_flat_map.hpp>
#include <memory>
#include <mutex>
namespace ripple {
namespace NodeStore {
class MemDBBackend : public Backend
{
private:
std::string name_;
beast::Journal journal_;
bool isOpen_{false};
struct base_uint_hasher
{
using result_type = std::size_t;
result_type
operator()(base_uint<256> const& value) const
{
return hardened_hash<>{}(value);
}
};
using DataStore =
std::map<uint256, std::vector<std::uint8_t>>; // Store compressed blob
// data
mutable std::recursive_mutex
mutex_; // Only needed for std::map implementation
DataStore table_;
public:
MemDBBackend(
size_t keyBytes,
Section const& keyValues,
beast::Journal journal)
: name_(get(keyValues, "path")), journal_(journal)
{
boost::ignore_unused(journal_);
if (name_.empty())
name_ = "node_db";
}
~MemDBBackend() override
{
close();
}
std::string
getName() override
{
return name_;
}
void
open(bool createIfMissing) override
{
std::lock_guard lock(mutex_);
if (isOpen_)
Throw<std::runtime_error>("already open");
isOpen_ = true;
}
bool
isOpen() override
{
return isOpen_;
}
void
close() override
{
std::lock_guard lock(mutex_);
table_.clear();
isOpen_ = false;
}
Status
fetch(void const* key, std::shared_ptr<NodeObject>* pObject) override
{
if (!isOpen_)
return notFound;
uint256 const hash(uint256::fromVoid(key));
std::lock_guard lock(mutex_);
auto it = table_.find(hash);
if (it == table_.end())
return notFound;
nudb::detail::buffer bf;
auto const result =
nodeobject_decompress(it->second.data(), it->second.size(), bf);
DecodedBlob decoded(hash.data(), result.first, result.second);
if (!decoded.wasOk())
return dataCorrupt;
*pObject = decoded.createObject();
return ok;
}
std::pair<std::vector<std::shared_ptr<NodeObject>>, Status>
fetchBatch(std::vector<uint256 const*> const& hashes) override
{
std::vector<std::shared_ptr<NodeObject>> results;
results.reserve(hashes.size());
for (auto const& h : hashes)
{
std::shared_ptr<NodeObject> nObj;
Status status = fetch(h->begin(), &nObj);
if (status != ok)
results.push_back({});
else
results.push_back(nObj);
}
return {results, ok};
}
void
store(std::shared_ptr<NodeObject> const& object) override
{
if (!isOpen_)
return;
if (!object)
return;
EncodedBlob encoded(object);
nudb::detail::buffer bf;
auto const result =
nodeobject_compress(encoded.getData(), encoded.getSize(), bf);
std::vector<std::uint8_t> compressed(
static_cast<const std::uint8_t*>(result.first),
static_cast<const std::uint8_t*>(result.first) + result.second);
std::lock_guard lock(mutex_);
table_[object->getHash()] = std::move(compressed);
}
void
storeBatch(Batch const& batch) override
{
for (auto const& e : batch)
store(e);
}
void
sync() override
{
}
void
for_each(std::function<void(std::shared_ptr<NodeObject>)> f) override
{
if (!isOpen_)
return;
std::lock_guard lock(mutex_);
for (const auto& entry : table_)
{
nudb::detail::buffer bf;
auto const result = nodeobject_decompress(
entry.second.data(), entry.second.size(), bf);
DecodedBlob decoded(
entry.first.data(), result.first, result.second);
if (decoded.wasOk())
f(decoded.createObject());
}
}
int
getWriteLoad() override
{
return 0;
}
void
setDeletePath() override
{
close();
}
int
fdRequired() const override
{
return 0;
}
private:
size_t
size() const
{
std::lock_guard lock(mutex_);
return table_.size();
}
};
class MemDBFactory : public Factory
{
public:
MemDBFactory()
{
Manager::instance().insert(*this);
}
~MemDBFactory() override
{
Manager::instance().erase(*this);
}
std::string
getName() const override
{
return "MemDB";
}
std::unique_ptr<Backend>
createInstance(
size_t keyBytes,
Section const& keyValues,
std::size_t burstSize,
Scheduler& scheduler,
beast::Journal journal) override
{
return std::make_unique<MemDBBackend>(keyBytes, keyValues, journal);
}
};
} // namespace NodeStore
} // namespace ripple

View File

@@ -1,43 +1,89 @@
//------------------------------------------------------------------------------
/*
This file is part of rippled: https://github.com/ripple/rippled
Copyright (c) 2012, 2013 Ripple Labs Inc.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include <ripple/basics/contract.h> #include <ripple/basics/contract.h>
#include <ripple/nodestore/Factory.h> #include <ripple/nodestore/Factory.h>
#include <ripple/nodestore/Manager.h> #include <ripple/nodestore/Manager.h>
#include <ripple/nodestore/impl/DecodedBlob.h>
#include <ripple/nodestore/impl/EncodedBlob.h>
#include <ripple/nodestore/impl/codec.h>
#include <boost/beast/core/string.hpp> #include <boost/beast/core/string.hpp>
#include <boost/core/ignore_unused.hpp> #include <boost/core/ignore_unused.hpp>
#include <boost/unordered/concurrent_flat_map.hpp> #include <map>
#include <memory> #include <memory>
#include <mutex> #include <mutex>
namespace ripple { namespace ripple {
namespace NodeStore { namespace NodeStore {
struct MemoryDB
{
explicit MemoryDB() = default;
std::mutex mutex;
bool open = false;
std::map<uint256 const, std::shared_ptr<NodeObject>> table;
};
class MemoryFactory : public Factory
{
private:
std::mutex mutex_;
std::map<std::string, MemoryDB, boost::beast::iless> map_;
public:
MemoryFactory();
~MemoryFactory() override;
std::string
getName() const override;
std::unique_ptr<Backend>
createInstance(
size_t keyBytes,
Section const& keyValues,
std::size_t burstSize,
Scheduler& scheduler,
beast::Journal journal) override;
MemoryDB&
open(std::string const& path)
{
std::lock_guard _(mutex_);
auto const result = map_.emplace(
std::piecewise_construct, std::make_tuple(path), std::make_tuple());
MemoryDB& db = result.first->second;
if (db.open)
Throw<std::runtime_error>("already open");
return db;
}
};
static MemoryFactory memoryFactory;
//------------------------------------------------------------------------------
class MemoryBackend : public Backend class MemoryBackend : public Backend
{ {
private: private:
using Map = std::map<uint256 const, std::shared_ptr<NodeObject>>;
std::string name_; std::string name_;
beast::Journal journal_; beast::Journal const journal_;
bool isOpen_{false}; MemoryDB* db_{nullptr};
struct base_uint_hasher
{
using result_type = std::size_t;
result_type
operator()(base_uint<256> const& value) const
{
return hardened_hash<>{}(value);
}
};
using DataStore =
std::map<uint256, std::vector<std::uint8_t>>; // Store compressed blob
// data
mutable std::recursive_mutex
mutex_; // Only needed for std::map implementation
DataStore table_;
public: public:
MemoryBackend( MemoryBackend(
@@ -46,9 +92,9 @@ public:
beast::Journal journal) beast::Journal journal)
: name_(get(keyValues, "path")), journal_(journal) : name_(get(keyValues, "path")), journal_(journal)
{ {
boost::ignore_unused(journal_); boost::ignore_unused(journal_); // Keep unused journal_ just in case.
if (name_.empty()) if (name_.empty())
name_ = "node_db"; Throw<std::runtime_error>("Missing path in TestMemory backend");
} }
~MemoryBackend() override ~MemoryBackend() override
@@ -65,46 +111,38 @@ public:
void void
open(bool createIfMissing) override open(bool createIfMissing) override
{ {
std::lock_guard lock(mutex_); db_ = &memoryFactory.open(name_);
if (isOpen_)
Throw<std::runtime_error>("already open");
isOpen_ = true;
} }
bool bool
isOpen() override isOpen() override
{ {
return isOpen_; return static_cast<bool>(db_);
} }
void void
close() override close() override
{ {
std::lock_guard lock(mutex_); db_ = nullptr;
table_.clear();
isOpen_ = false;
} }
//--------------------------------------------------------------------------
Status Status
fetch(void const* key, std::shared_ptr<NodeObject>* pObject) override fetch(void const* key, std::shared_ptr<NodeObject>* pObject) override
{ {
if (!isOpen_) assert(db_);
return notFound;
uint256 const hash(uint256::fromVoid(key)); uint256 const hash(uint256::fromVoid(key));
std::lock_guard lock(mutex_); std::lock_guard _(db_->mutex);
auto it = table_.find(hash);
if (it == table_.end())
return notFound;
nudb::detail::buffer bf; Map::iterator iter = db_->table.find(hash);
auto const result = if (iter == db_->table.end())
nodeobject_decompress(it->second.data(), it->second.size(), bf); {
DecodedBlob decoded(hash.data(), result.first, result.second); pObject->reset();
if (!decoded.wasOk()) return notFound;
return dataCorrupt; }
*pObject = decoded.createObject(); *pObject = iter->second;
return ok; return ok;
} }
@@ -122,29 +160,16 @@ public:
else else
results.push_back(nObj); results.push_back(nObj);
} }
return {results, ok}; return {results, ok};
} }
void void
store(std::shared_ptr<NodeObject> const& object) override store(std::shared_ptr<NodeObject> const& object) override
{ {
if (!isOpen_) assert(db_);
return; std::lock_guard _(db_->mutex);
db_->table.emplace(object->getHash(), object);
if (!object)
return;
EncodedBlob encoded(object);
nudb::detail::buffer bf;
auto const result =
nodeobject_compress(encoded.getData(), encoded.getSize(), bf);
std::vector<std::uint8_t> compressed(
static_cast<const std::uint8_t*>(result.first),
static_cast<const std::uint8_t*>(result.first) + result.second);
std::lock_guard lock(mutex_);
table_[object->getHash()] = std::move(compressed);
} }
void void
@@ -162,20 +187,9 @@ public:
void void
for_each(std::function<void(std::shared_ptr<NodeObject>)> f) override for_each(std::function<void(std::shared_ptr<NodeObject>)> f) override
{ {
if (!isOpen_) assert(db_);
return; for (auto const& e : db_->table)
f(e.second);
std::lock_guard lock(mutex_);
for (const auto& entry : table_)
{
nudb::detail::buffer bf;
auto const result = nodeobject_decompress(
entry.second.data(), entry.second.size(), bf);
DecodedBlob decoded(
entry.first.data(), result.first, result.second);
if (decoded.wasOk())
f(decoded.createObject());
}
} }
int int
@@ -187,7 +201,6 @@ public:
void void
setDeletePath() override setDeletePath() override
{ {
close();
} }
int int
@@ -195,48 +208,36 @@ public:
{ {
return 0; return 0;
} }
private:
size_t
size() const
{
std::lock_guard lock(mutex_);
return table_.size();
}
}; };
class MemoryFactory : public Factory //------------------------------------------------------------------------------
MemoryFactory::MemoryFactory()
{ {
public: Manager::instance().insert(*this);
MemoryFactory() }
{
Manager::instance().insert(*this);
}
~MemoryFactory() override MemoryFactory::~MemoryFactory()
{ {
Manager::instance().erase(*this); Manager::instance().erase(*this);
} }
std::string std::string
getName() const override MemoryFactory::getName() const
{ {
return "Memory"; return "Memory";
} }
std::unique_ptr<Backend> std::unique_ptr<Backend>
createInstance( MemoryFactory::createInstance(
size_t keyBytes, size_t keyBytes,
Section const& keyValues, Section const& keyValues,
std::size_t burstSize, std::size_t,
Scheduler& scheduler, Scheduler& scheduler,
beast::Journal journal) override beast::Journal journal)
{ {
return std::make_unique<MemoryBackend>(keyBytes, keyValues, journal); return std::make_unique<MemoryBackend>(keyBytes, keyValues, journal);
} }
};
static MemoryFactory memoryFactory;
} // namespace NodeStore } // namespace NodeStore
} // namespace ripple } // namespace ripple

View File

@@ -1,243 +0,0 @@
//------------------------------------------------------------------------------
/*
This file is part of rippled: https://github.com/ripple/rippled
Copyright (c) 2012, 2013 Ripple Labs Inc.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include <ripple/basics/contract.h>
#include <ripple/nodestore/Factory.h>
#include <ripple/nodestore/Manager.h>
#include <boost/beast/core/string.hpp>
#include <boost/core/ignore_unused.hpp>
#include <map>
#include <memory>
#include <mutex>
namespace ripple {
namespace NodeStore {
struct TestMemoryDB
{
explicit TestMemoryDB() = default;
std::mutex mutex;
bool open = false;
std::map<uint256 const, std::shared_ptr<NodeObject>> table;
};
class TestMemoryFactory : public Factory
{
private:
std::mutex mutex_;
std::map<std::string, TestMemoryDB, boost::beast::iless> map_;
public:
TestMemoryFactory();
~TestMemoryFactory() override;
std::string
getName() const override;
std::unique_ptr<Backend>
createInstance(
size_t keyBytes,
Section const& keyValues,
std::size_t burstSize,
Scheduler& scheduler,
beast::Journal journal) override;
TestMemoryDB&
open(std::string const& path)
{
std::lock_guard _(mutex_);
auto const result = map_.emplace(
std::piecewise_construct, std::make_tuple(path), std::make_tuple());
TestMemoryDB& db = result.first->second;
if (db.open)
Throw<std::runtime_error>("already open");
return db;
}
};
static TestMemoryFactory testMemoryFactory;
//------------------------------------------------------------------------------
class TestMemoryBackend : public Backend
{
private:
using Map = std::map<uint256 const, std::shared_ptr<NodeObject>>;
std::string name_;
beast::Journal const journal_;
TestMemoryDB* db_{nullptr};
public:
TestMemoryBackend(
size_t keyBytes,
Section const& keyValues,
beast::Journal journal)
: name_(get(keyValues, "path")), journal_(journal)
{
boost::ignore_unused(journal_); // Keep unused journal_ just in case.
if (name_.empty())
Throw<std::runtime_error>("Missing path in TestMemory backend");
}
~TestMemoryBackend() override
{
close();
}
std::string
getName() override
{
return name_;
}
void
open(bool createIfMissing) override
{
db_ = &testMemoryFactory.open(name_);
}
bool
isOpen() override
{
return static_cast<bool>(db_);
}
void
close() override
{
db_ = nullptr;
}
//--------------------------------------------------------------------------
Status
fetch(void const* key, std::shared_ptr<NodeObject>* pObject) override
{
assert(db_);
uint256 const hash(uint256::fromVoid(key));
std::lock_guard _(db_->mutex);
Map::iterator iter = db_->table.find(hash);
if (iter == db_->table.end())
{
pObject->reset();
return notFound;
}
*pObject = iter->second;
return ok;
}
std::pair<std::vector<std::shared_ptr<NodeObject>>, Status>
fetchBatch(std::vector<uint256 const*> const& hashes) override
{
std::vector<std::shared_ptr<NodeObject>> results;
results.reserve(hashes.size());
for (auto const& h : hashes)
{
std::shared_ptr<NodeObject> nObj;
Status status = fetch(h->begin(), &nObj);
if (status != ok)
results.push_back({});
else
results.push_back(nObj);
}
return {results, ok};
}
void
store(std::shared_ptr<NodeObject> const& object) override
{
assert(db_);
std::lock_guard _(db_->mutex);
db_->table.emplace(object->getHash(), object);
}
void
storeBatch(Batch const& batch) override
{
for (auto const& e : batch)
store(e);
}
void
sync() override
{
}
void
for_each(std::function<void(std::shared_ptr<NodeObject>)> f) override
{
assert(db_);
for (auto const& e : db_->table)
f(e.second);
}
int
getWriteLoad() override
{
return 0;
}
void
setDeletePath() override
{
}
int
fdRequired() const override
{
return 0;
}
};
//------------------------------------------------------------------------------
TestMemoryFactory::TestMemoryFactory()
{
Manager::instance().insert(*this);
}
TestMemoryFactory::~TestMemoryFactory()
{
Manager::instance().erase(*this);
}
std::string
TestMemoryFactory::getName() const
{
return "TestMemory";
}
std::unique_ptr<Backend>
TestMemoryFactory::createInstance(
size_t keyBytes,
Section const& keyValues,
std::size_t,
Scheduler& scheduler,
beast::Journal journal)
{
return std::make_unique<TestMemoryBackend>(keyBytes, keyValues, journal);
}
} // namespace NodeStore
} // namespace ripple

View File

@@ -49,7 +49,7 @@ setupConfigForUnitTests(Config& cfg)
cfg.FEES.account_reserve = XRP(200).value().xrp().drops(); cfg.FEES.account_reserve = XRP(200).value().xrp().drops();
cfg.FEES.owner_reserve = XRP(50).value().xrp().drops(); cfg.FEES.owner_reserve = XRP(50).value().xrp().drops();
cfg.overwrite(ConfigSection::nodeDatabase(), "type", "testmemory"); cfg.overwrite(ConfigSection::nodeDatabase(), "type", "memory");
cfg.overwrite(ConfigSection::nodeDatabase(), "path", "main"); cfg.overwrite(ConfigSection::nodeDatabase(), "path", "main");
cfg.deprecatedClearSection(ConfigSection::importNodeDatabase()); cfg.deprecatedClearSection(ConfigSection::importNodeDatabase());
cfg.legacy("database_path", ""); cfg.legacy("database_path", "");

View File

@@ -107,8 +107,8 @@ public:
{ {
std::uint64_t const seedValue = 50; std::uint64_t const seedValue = 50;
testBackend("testmemory", seedValue);
testBackend("memory", seedValue); testBackend("memory", seedValue);
testBackend("memdb", seedValue);
testBackend("nudb", seedValue); testBackend("nudb", seedValue);
#if RIPPLE_ROCKSDB_AVAILABLE #if RIPPLE_ROCKSDB_AVAILABLE

View File

@@ -659,10 +659,10 @@ public:
testConfig(); testConfig();
testNodeStore("testmemory", false, seedValue);
testNodeStore("memory", false, seedValue); testNodeStore("memory", false, seedValue);
testNodeStore("memdb", false, seedValue);
// Persistent backend tests // Persistent backend tests
{ {
testNodeStore("nudb", true, seedValue); testNodeStore("nudb", true, seedValue);

View File

@@ -57,7 +57,7 @@ public:
, j_(j) , j_(j)
{ {
Section testSection; Section testSection;
testSection.set("type", "testmemory"); testSection.set("type", "memory");
testSection.set("path", "SHAMap_test"); testSection.set("path", "SHAMap_test");
db_ = NodeStore::Manager::instance().make_Database( db_ = NodeStore::Manager::instance().make_Database(
megabytes(4), scheduler_, 1, testSection, j); megabytes(4), scheduler_, 1, testSection, j);