[fold] rename MemoryFactory -> MemDBFactory

This commit is contained in:
Denis Angell
2024-11-11 21:26:27 +01:00
parent febdb6a997
commit 8c00fd10d1
10 changed files with 371 additions and 373 deletions

View File

@@ -538,8 +538,8 @@ target_sources (rippled PRIVATE
subdir: nodestore
#]===============================]
src/ripple/nodestore/backend/CassandraFactory.cpp
src/ripple/nodestore/backend/MemDBFactory.cpp
src/ripple/nodestore/backend/MemoryFactory.cpp
src/ripple/nodestore/backend/TestMemoryFactory.cpp
src/ripple/nodestore/backend/FlatmapFactory.cpp
src/ripple/nodestore/backend/NuDBFactory.cpp
src/ripple/nodestore/backend/NullFactory.cpp

View File

@@ -40,7 +40,7 @@ RelationalDatabase::init(
{
bool use_sqlite = false;
bool use_postgres = false;
bool use_memory = false;
bool use_memdb = false;
bool use_flatmap = false;
if (config.reporting())
@@ -56,9 +56,9 @@ RelationalDatabase::init(
{
use_sqlite = true;
}
else if (boost::iequals(get(rdb_section, "backend"), "memory"))
else if (boost::iequals(get(rdb_section, "backend"), "memdb"))
{
use_memory = true;
use_memdb = true;
}
else if (boost::iequals(get(rdb_section, "backend"), "flatmap"))
{
@@ -85,7 +85,7 @@ RelationalDatabase::init(
{
return getPostgresDatabase(app, config, jobQueue);
}
else if (use_memory)
else if (use_memdb)
{
return getMemoryDatabase(app, config, jobQueue);
}

View File

@@ -357,13 +357,13 @@ public:
static bool const isMem =
(!section(SECTION_RELATIONAL_DB).empty() &&
boost::beast::iequals(
get(section(SECTION_RELATIONAL_DB), "backend"), "memory")) ||
get(section(SECTION_RELATIONAL_DB), "backend"), "memdb")) ||
(!section("node_db").empty() &&
(boost::beast::iequals(
get(section("node_db"), "type"), "memory") ||
get(section("node_db"), "type"), "memdb") ||
boost::beast::iequals(
get(section("node_db"), "type"), "flatmap")));
// RHNOTE: testmemory type is not selected for here because it breaks
// RHNOTE: memory type is not selected for here because it breaks
// tests
return isMem;
}

View File

@@ -0,0 +1,240 @@
#include <ripple/basics/contract.h>
#include <ripple/nodestore/Factory.h>
#include <ripple/nodestore/Manager.h>
#include <ripple/nodestore/impl/DecodedBlob.h>
#include <ripple/nodestore/impl/EncodedBlob.h>
#include <ripple/nodestore/impl/codec.h>
#include <boost/beast/core/string.hpp>
#include <boost/core/ignore_unused.hpp>
#include <boost/unordered/concurrent_flat_map.hpp>
#include <memory>
#include <mutex>
namespace ripple {
namespace NodeStore {
class MemDBBackend : public Backend
{
private:
std::string name_;
beast::Journal journal_;
bool isOpen_{false};
struct base_uint_hasher
{
using result_type = std::size_t;
result_type
operator()(base_uint<256> const& value) const
{
return hardened_hash<>{}(value);
}
};
using DataStore =
std::map<uint256, std::vector<std::uint8_t>>; // Store compressed blob
// data
mutable std::recursive_mutex
mutex_; // Only needed for std::map implementation
DataStore table_;
public:
MemDBBackend(
size_t keyBytes,
Section const& keyValues,
beast::Journal journal)
: name_(get(keyValues, "path")), journal_(journal)
{
boost::ignore_unused(journal_);
if (name_.empty())
name_ = "node_db";
}
~MemDBBackend() override
{
close();
}
std::string
getName() override
{
return name_;
}
void
open(bool createIfMissing) override
{
std::lock_guard lock(mutex_);
if (isOpen_)
Throw<std::runtime_error>("already open");
isOpen_ = true;
}
bool
isOpen() override
{
return isOpen_;
}
void
close() override
{
std::lock_guard lock(mutex_);
table_.clear();
isOpen_ = false;
}
Status
fetch(void const* key, std::shared_ptr<NodeObject>* pObject) override
{
if (!isOpen_)
return notFound;
uint256 const hash(uint256::fromVoid(key));
std::lock_guard lock(mutex_);
auto it = table_.find(hash);
if (it == table_.end())
return notFound;
nudb::detail::buffer bf;
auto const result =
nodeobject_decompress(it->second.data(), it->second.size(), bf);
DecodedBlob decoded(hash.data(), result.first, result.second);
if (!decoded.wasOk())
return dataCorrupt;
*pObject = decoded.createObject();
return ok;
}
std::pair<std::vector<std::shared_ptr<NodeObject>>, Status>
fetchBatch(std::vector<uint256 const*> const& hashes) override
{
std::vector<std::shared_ptr<NodeObject>> results;
results.reserve(hashes.size());
for (auto const& h : hashes)
{
std::shared_ptr<NodeObject> nObj;
Status status = fetch(h->begin(), &nObj);
if (status != ok)
results.push_back({});
else
results.push_back(nObj);
}
return {results, ok};
}
void
store(std::shared_ptr<NodeObject> const& object) override
{
if (!isOpen_)
return;
if (!object)
return;
EncodedBlob encoded(object);
nudb::detail::buffer bf;
auto const result =
nodeobject_compress(encoded.getData(), encoded.getSize(), bf);
std::vector<std::uint8_t> compressed(
static_cast<const std::uint8_t*>(result.first),
static_cast<const std::uint8_t*>(result.first) + result.second);
std::lock_guard lock(mutex_);
table_[object->getHash()] = std::move(compressed);
}
void
storeBatch(Batch const& batch) override
{
for (auto const& e : batch)
store(e);
}
void
sync() override
{
}
void
for_each(std::function<void(std::shared_ptr<NodeObject>)> f) override
{
if (!isOpen_)
return;
std::lock_guard lock(mutex_);
for (const auto& entry : table_)
{
nudb::detail::buffer bf;
auto const result = nodeobject_decompress(
entry.second.data(), entry.second.size(), bf);
DecodedBlob decoded(
entry.first.data(), result.first, result.second);
if (decoded.wasOk())
f(decoded.createObject());
}
}
int
getWriteLoad() override
{
return 0;
}
void
setDeletePath() override
{
close();
}
int
fdRequired() const override
{
return 0;
}
private:
size_t
size() const
{
std::lock_guard lock(mutex_);
return table_.size();
}
};
class MemDBFactory : public Factory
{
public:
MemDBFactory()
{
Manager::instance().insert(*this);
}
~MemDBFactory() override
{
Manager::instance().erase(*this);
}
std::string
getName() const override
{
return "MemDB";
}
std::unique_ptr<Backend>
createInstance(
size_t keyBytes,
Section const& keyValues,
std::size_t burstSize,
Scheduler& scheduler,
beast::Journal journal) override
{
return std::make_unique<MemDBBackend>(keyBytes, keyValues, journal);
}
};
} // namespace NodeStore
} // namespace ripple

View File

@@ -1,43 +1,89 @@
//------------------------------------------------------------------------------
/*
This file is part of rippled: https://github.com/ripple/rippled
Copyright (c) 2012, 2013 Ripple Labs Inc.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include <ripple/basics/contract.h>
#include <ripple/nodestore/Factory.h>
#include <ripple/nodestore/Manager.h>
#include <ripple/nodestore/impl/DecodedBlob.h>
#include <ripple/nodestore/impl/EncodedBlob.h>
#include <ripple/nodestore/impl/codec.h>
#include <boost/beast/core/string.hpp>
#include <boost/core/ignore_unused.hpp>
#include <boost/unordered/concurrent_flat_map.hpp>
#include <map>
#include <memory>
#include <mutex>
namespace ripple {
namespace NodeStore {
struct MemoryDB
{
explicit MemoryDB() = default;
std::mutex mutex;
bool open = false;
std::map<uint256 const, std::shared_ptr<NodeObject>> table;
};
class MemoryFactory : public Factory
{
private:
std::mutex mutex_;
std::map<std::string, MemoryDB, boost::beast::iless> map_;
public:
MemoryFactory();
~MemoryFactory() override;
std::string
getName() const override;
std::unique_ptr<Backend>
createInstance(
size_t keyBytes,
Section const& keyValues,
std::size_t burstSize,
Scheduler& scheduler,
beast::Journal journal) override;
MemoryDB&
open(std::string const& path)
{
std::lock_guard _(mutex_);
auto const result = map_.emplace(
std::piecewise_construct, std::make_tuple(path), std::make_tuple());
MemoryDB& db = result.first->second;
if (db.open)
Throw<std::runtime_error>("already open");
return db;
}
};
static MemoryFactory memoryFactory;
//------------------------------------------------------------------------------
class MemoryBackend : public Backend
{
private:
using Map = std::map<uint256 const, std::shared_ptr<NodeObject>>;
std::string name_;
beast::Journal journal_;
bool isOpen_{false};
struct base_uint_hasher
{
using result_type = std::size_t;
result_type
operator()(base_uint<256> const& value) const
{
return hardened_hash<>{}(value);
}
};
using DataStore =
std::map<uint256, std::vector<std::uint8_t>>; // Store compressed blob
// data
mutable std::recursive_mutex
mutex_; // Only needed for std::map implementation
DataStore table_;
beast::Journal const journal_;
MemoryDB* db_{nullptr};
public:
MemoryBackend(
@@ -46,9 +92,9 @@ public:
beast::Journal journal)
: name_(get(keyValues, "path")), journal_(journal)
{
boost::ignore_unused(journal_);
boost::ignore_unused(journal_); // Keep unused journal_ just in case.
if (name_.empty())
name_ = "node_db";
Throw<std::runtime_error>("Missing path in TestMemory backend");
}
~MemoryBackend() override
@@ -65,46 +111,38 @@ public:
void
open(bool createIfMissing) override
{
std::lock_guard lock(mutex_);
if (isOpen_)
Throw<std::runtime_error>("already open");
isOpen_ = true;
db_ = &memoryFactory.open(name_);
}
bool
isOpen() override
{
return isOpen_;
return static_cast<bool>(db_);
}
void
close() override
{
std::lock_guard lock(mutex_);
table_.clear();
isOpen_ = false;
db_ = nullptr;
}
//--------------------------------------------------------------------------
Status
fetch(void const* key, std::shared_ptr<NodeObject>* pObject) override
{
if (!isOpen_)
return notFound;
assert(db_);
uint256 const hash(uint256::fromVoid(key));
std::lock_guard lock(mutex_);
auto it = table_.find(hash);
if (it == table_.end())
return notFound;
std::lock_guard _(db_->mutex);
nudb::detail::buffer bf;
auto const result =
nodeobject_decompress(it->second.data(), it->second.size(), bf);
DecodedBlob decoded(hash.data(), result.first, result.second);
if (!decoded.wasOk())
return dataCorrupt;
*pObject = decoded.createObject();
Map::iterator iter = db_->table.find(hash);
if (iter == db_->table.end())
{
pObject->reset();
return notFound;
}
*pObject = iter->second;
return ok;
}
@@ -122,29 +160,16 @@ public:
else
results.push_back(nObj);
}
return {results, ok};
}
void
store(std::shared_ptr<NodeObject> const& object) override
{
if (!isOpen_)
return;
if (!object)
return;
EncodedBlob encoded(object);
nudb::detail::buffer bf;
auto const result =
nodeobject_compress(encoded.getData(), encoded.getSize(), bf);
std::vector<std::uint8_t> compressed(
static_cast<const std::uint8_t*>(result.first),
static_cast<const std::uint8_t*>(result.first) + result.second);
std::lock_guard lock(mutex_);
table_[object->getHash()] = std::move(compressed);
assert(db_);
std::lock_guard _(db_->mutex);
db_->table.emplace(object->getHash(), object);
}
void
@@ -162,20 +187,9 @@ public:
void
for_each(std::function<void(std::shared_ptr<NodeObject>)> f) override
{
if (!isOpen_)
return;
std::lock_guard lock(mutex_);
for (const auto& entry : table_)
{
nudb::detail::buffer bf;
auto const result = nodeobject_decompress(
entry.second.data(), entry.second.size(), bf);
DecodedBlob decoded(
entry.first.data(), result.first, result.second);
if (decoded.wasOk())
f(decoded.createObject());
}
assert(db_);
for (auto const& e : db_->table)
f(e.second);
}
int
@@ -187,7 +201,6 @@ public:
void
setDeletePath() override
{
close();
}
int
@@ -195,48 +208,36 @@ public:
{
return 0;
}
private:
size_t
size() const
{
std::lock_guard lock(mutex_);
return table_.size();
}
};
class MemoryFactory : public Factory
//------------------------------------------------------------------------------
MemoryFactory::MemoryFactory()
{
public:
MemoryFactory()
{
Manager::instance().insert(*this);
}
Manager::instance().insert(*this);
}
~MemoryFactory() override
{
Manager::instance().erase(*this);
}
MemoryFactory::~MemoryFactory()
{
Manager::instance().erase(*this);
}
std::string
getName() const override
{
return "Memory";
}
std::string
MemoryFactory::getName() const
{
return "Memory";
}
std::unique_ptr<Backend>
createInstance(
size_t keyBytes,
Section const& keyValues,
std::size_t burstSize,
Scheduler& scheduler,
beast::Journal journal) override
{
return std::make_unique<MemoryBackend>(keyBytes, keyValues, journal);
}
};
static MemoryFactory memoryFactory;
std::unique_ptr<Backend>
MemoryFactory::createInstance(
size_t keyBytes,
Section const& keyValues,
std::size_t,
Scheduler& scheduler,
beast::Journal journal)
{
return std::make_unique<MemoryBackend>(keyBytes, keyValues, journal);
}
} // namespace NodeStore
} // namespace ripple

View File

@@ -1,243 +0,0 @@
//------------------------------------------------------------------------------
/*
This file is part of rippled: https://github.com/ripple/rippled
Copyright (c) 2012, 2013 Ripple Labs Inc.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include <ripple/basics/contract.h>
#include <ripple/nodestore/Factory.h>
#include <ripple/nodestore/Manager.h>
#include <boost/beast/core/string.hpp>
#include <boost/core/ignore_unused.hpp>
#include <map>
#include <memory>
#include <mutex>
namespace ripple {
namespace NodeStore {
struct TestMemoryDB
{
explicit TestMemoryDB() = default;
std::mutex mutex;
bool open = false;
std::map<uint256 const, std::shared_ptr<NodeObject>> table;
};
class TestMemoryFactory : public Factory
{
private:
std::mutex mutex_;
std::map<std::string, TestMemoryDB, boost::beast::iless> map_;
public:
TestMemoryFactory();
~TestMemoryFactory() override;
std::string
getName() const override;
std::unique_ptr<Backend>
createInstance(
size_t keyBytes,
Section const& keyValues,
std::size_t burstSize,
Scheduler& scheduler,
beast::Journal journal) override;
TestMemoryDB&
open(std::string const& path)
{
std::lock_guard _(mutex_);
auto const result = map_.emplace(
std::piecewise_construct, std::make_tuple(path), std::make_tuple());
TestMemoryDB& db = result.first->second;
if (db.open)
Throw<std::runtime_error>("already open");
return db;
}
};
static TestMemoryFactory testMemoryFactory;
//------------------------------------------------------------------------------
class TestMemoryBackend : public Backend
{
private:
using Map = std::map<uint256 const, std::shared_ptr<NodeObject>>;
std::string name_;
beast::Journal const journal_;
TestMemoryDB* db_{nullptr};
public:
TestMemoryBackend(
size_t keyBytes,
Section const& keyValues,
beast::Journal journal)
: name_(get(keyValues, "path")), journal_(journal)
{
boost::ignore_unused(journal_); // Keep unused journal_ just in case.
if (name_.empty())
Throw<std::runtime_error>("Missing path in TestMemory backend");
}
~TestMemoryBackend() override
{
close();
}
std::string
getName() override
{
return name_;
}
void
open(bool createIfMissing) override
{
db_ = &testMemoryFactory.open(name_);
}
bool
isOpen() override
{
return static_cast<bool>(db_);
}
void
close() override
{
db_ = nullptr;
}
//--------------------------------------------------------------------------
Status
fetch(void const* key, std::shared_ptr<NodeObject>* pObject) override
{
assert(db_);
uint256 const hash(uint256::fromVoid(key));
std::lock_guard _(db_->mutex);
Map::iterator iter = db_->table.find(hash);
if (iter == db_->table.end())
{
pObject->reset();
return notFound;
}
*pObject = iter->second;
return ok;
}
std::pair<std::vector<std::shared_ptr<NodeObject>>, Status>
fetchBatch(std::vector<uint256 const*> const& hashes) override
{
std::vector<std::shared_ptr<NodeObject>> results;
results.reserve(hashes.size());
for (auto const& h : hashes)
{
std::shared_ptr<NodeObject> nObj;
Status status = fetch(h->begin(), &nObj);
if (status != ok)
results.push_back({});
else
results.push_back(nObj);
}
return {results, ok};
}
void
store(std::shared_ptr<NodeObject> const& object) override
{
assert(db_);
std::lock_guard _(db_->mutex);
db_->table.emplace(object->getHash(), object);
}
void
storeBatch(Batch const& batch) override
{
for (auto const& e : batch)
store(e);
}
void
sync() override
{
}
void
for_each(std::function<void(std::shared_ptr<NodeObject>)> f) override
{
assert(db_);
for (auto const& e : db_->table)
f(e.second);
}
int
getWriteLoad() override
{
return 0;
}
void
setDeletePath() override
{
}
int
fdRequired() const override
{
return 0;
}
};
//------------------------------------------------------------------------------
TestMemoryFactory::TestMemoryFactory()
{
Manager::instance().insert(*this);
}
TestMemoryFactory::~TestMemoryFactory()
{
Manager::instance().erase(*this);
}
std::string
TestMemoryFactory::getName() const
{
return "TestMemory";
}
std::unique_ptr<Backend>
TestMemoryFactory::createInstance(
size_t keyBytes,
Section const& keyValues,
std::size_t,
Scheduler& scheduler,
beast::Journal journal)
{
return std::make_unique<TestMemoryBackend>(keyBytes, keyValues, journal);
}
} // namespace NodeStore
} // namespace ripple

View File

@@ -49,7 +49,7 @@ setupConfigForUnitTests(Config& cfg)
cfg.FEES.account_reserve = XRP(200).value().xrp().drops();
cfg.FEES.owner_reserve = XRP(50).value().xrp().drops();
cfg.overwrite(ConfigSection::nodeDatabase(), "type", "testmemory");
cfg.overwrite(ConfigSection::nodeDatabase(), "type", "memory");
cfg.overwrite(ConfigSection::nodeDatabase(), "path", "main");
cfg.deprecatedClearSection(ConfigSection::importNodeDatabase());
cfg.legacy("database_path", "");

View File

@@ -107,8 +107,8 @@ public:
{
std::uint64_t const seedValue = 50;
testBackend("testmemory", seedValue);
testBackend("memory", seedValue);
testBackend("memdb", seedValue);
testBackend("nudb", seedValue);
#if RIPPLE_ROCKSDB_AVAILABLE

View File

@@ -659,10 +659,10 @@ public:
testConfig();
testNodeStore("testmemory", false, seedValue);
testNodeStore("memory", false, seedValue);
testNodeStore("memdb", false, seedValue);
// Persistent backend tests
{
testNodeStore("nudb", true, seedValue);

View File

@@ -57,7 +57,7 @@ public:
, j_(j)
{
Section testSection;
testSection.set("type", "testmemory");
testSection.set("type", "memory");
testSection.set("path", "SHAMap_test");
db_ = NodeStore::Manager::instance().make_Database(
megabytes(4), scheduler_, 1, testSection, j);