This commit is contained in:
Richard Holland
2024-08-18 16:46:46 +10:00
parent fbfd8c1e0a
commit 224e78ac81
2 changed files with 720 additions and 674 deletions

View File

@@ -24,13 +24,13 @@
#include <ripple/nodestore/impl/EncodedBlob.h> #include <ripple/nodestore/impl/EncodedBlob.h>
#include <ripple/nodestore/impl/codec.h> #include <ripple/nodestore/impl/codec.h>
#include <boost/filesystem.hpp> #include <boost/filesystem.hpp>
#include "snug.hpp"
#include <cassert> #include <cassert>
#include <chrono> #include <chrono>
#include <cstdint> #include <cstdint>
#include <cstdio> #include <cstdio>
#include <exception> #include <exception>
#include <memory> #include <memory>
#include "snug.hpp"
namespace ripple { namespace ripple {
namespace NodeStore { namespace NodeStore {
@@ -38,7 +38,8 @@ namespace NodeStore {
class SnugDBBackend : public Backend class SnugDBBackend : public Backend
{ {
private: private:
static constexpr uint64_t BUFFER_SIZE = 256ULL*1024ULL*1024ULL; // 256 Mib read buffer per thread static constexpr uint64_t BUFFER_SIZE =
256ULL * 1024ULL * 1024ULL; // 256 Mib read buffer per thread
public: public:
beast::Journal const j_; beast::Journal const j_;
std::string const name_; std::string const name_;
@@ -49,14 +50,11 @@ public:
Section const& keyValues, Section const& keyValues,
Scheduler& scheduler, Scheduler& scheduler,
beast::Journal journal) beast::Journal journal)
: j_(journal) : j_(journal), name_(get(keyValues, "path")), scheduler_(scheduler)
, name_(get(keyValues, "path"))
, scheduler_(scheduler)
{ {
if (name_.empty()) if (name_.empty())
throw std::runtime_error( throw std::runtime_error(
"nodestore: Missing path in SnugDB backend"); "nodestore: Missing path in SnugDB backend");
} }
~SnugDBBackend() override ~SnugDBBackend() override
@@ -68,8 +66,7 @@ public:
} }
catch (std::exception const& e) catch (std::exception const& e)
{ {
JLOG(j_.warn()) JLOG(j_.warn()) << "SnugDB threw on destruction: " << e.what();
<< "SnugDB threw on destruction: " << e.what();
// Don't allow exceptions to propagate out of destructors. // Don't allow exceptions to propagate out of destructors.
} }
} }
@@ -91,10 +88,8 @@ public:
return; return;
} }
std::string path = name_ + "/" + std::string path = name_ + "/" + std::to_string(uid) + "-" +
std::to_string(uid) + "-" + std::to_string(appType) + "-" + std::to_string(salt);
std::to_string(appType) + "-" +
std::to_string(salt);
boost::filesystem::create_directories(path); boost::filesystem::create_directories(path);
db_ = std::make_unique<snug::SnugDB>(path); db_ = std::make_unique<snug::SnugDB>(path);
@@ -126,32 +121,34 @@ public:
pno->reset(); pno->reset();
static thread_local std::unique_ptr<uint8_t[]> static thread_local std::unique_ptr<uint8_t[]> thread_buffer =
thread_buffer = std::make_unique<uint8_t[]>(BUFFER_SIZE); std::make_unique<uint8_t[]>(BUFFER_SIZE);
uint8_t* ptr = &(thread_buffer[0]); uint8_t* ptr = &(thread_buffer[0]);
uint64_t len = BUFFER_SIZE; uint64_t len = BUFFER_SIZE;
int result = db_->read_entry( int result = db_->read_entry(
static_cast<uint8_t*>(const_cast<void*>(key)), static_cast<uint8_t*>(const_cast<void*>(key)), ptr, &len);
ptr,
&len);
if (0) if (0)
{ {
std::stringstream ss; std::stringstream ss;
const unsigned char* bytes = static_cast<const unsigned char*>(key); const unsigned char* bytes = static_cast<const unsigned char*>(key);
for (int i = 0; i < 32; ++i) { for (int i = 0; i < 32; ++i)
ss << std::setfill('0') << std::setw(2) << std::hex << static_cast<int>(bytes[i]); {
ss << std::setfill('0') << std::setw(2) << std::hex
<< static_cast<int>(bytes[i]);
} }
std::string key_hex = ss.str(); std::string key_hex = ss.str();
// Print the result using printf // Print the result using printf
printf("snug fetch: len=%zu result=%zu key=%s\n", len, result, key_hex.c_str()); printf(
"snug fetch: len=%zu result=%zu key=%s\n",
len,
result,
key_hex.c_str());
} }
if (result == 1) if (result == 1)
return notFound; return notFound;
@@ -190,24 +187,28 @@ public:
do_insert(std::shared_ptr<NodeObject> const& no) do_insert(std::shared_ptr<NodeObject> const& no)
{ {
EncodedBlob e(no); EncodedBlob e(no);
if (0) if (0)
{ {
std::stringstream ss; std::stringstream ss;
const unsigned char* bytes = static_cast<const unsigned char*>(const_cast<void*>(e.getKey())); const unsigned char* bytes = static_cast<const unsigned char*>(
const_cast<void*>(e.getKey()));
for (int i = 0; i < 32; ++i) for (int i = 0; i < 32; ++i)
ss << std::setfill('0') << std::setw(2) << std::hex << static_cast<int>(bytes[i]); ss << std::setfill('0') << std::setw(2) << std::hex
<< static_cast<int>(bytes[i]);
std::string key_hex = ss.str(); std::string key_hex = ss.str();
std::cout << "snugdb write: len=" << e.getSize()
std::cout << "snugdb write: len=" << e.getSize() << ", key=" << key_hex << "\n"; << ", key=" << key_hex << "\n";
} }
int out = db_->write_entry( int out = db_->write_entry(
static_cast<uint8_t*>(const_cast<void*>(e.getKey())), static_cast<uint8_t*>(const_cast<void*>(e.getKey())),
static_cast<uint8_t*>(const_cast<void*>(e.getData())), static_cast<uint8_t*>(const_cast<void*>(e.getData())),
e.getSize()); e.getSize());
if (out != 0) if (out != 0)
throw std::runtime_error("SnugDB could not write entry. Disk full? error" + std::to_string(out)); throw std::runtime_error(
"SnugDB could not write entry. Disk full? error" +
std::to_string(out));
} }
void void
@@ -243,19 +244,22 @@ public:
void void
for_each(std::function<void(std::shared_ptr<NodeObject>)> f) override for_each(std::function<void(std::shared_ptr<NodeObject>)> f) override
{ {
db_->visit_all([](uint8_t* key, uint8_t* data, uint64_t len, void* fp) -> void db_->visit_all(
{ [](uint8_t* key, uint8_t* data, uint64_t len, void* fp) -> void {
DecodedBlob decoded(key, data, len); DecodedBlob decoded(key, data, len);
if (!decoded.wasOk()) if (!decoded.wasOk())
{ {
throw std::runtime_error("Missing or corrupted data in snugdb"); throw std::runtime_error(
return; "Missing or corrupted data in snugdb");
} return;
}
std::function<void(std::shared_ptr<NodeObject>)> f = std::function<void(std::shared_ptr<NodeObject>)> f =
*(reinterpret_cast<std::function<void(std::shared_ptr<NodeObject>)>*>(fp)); *(reinterpret_cast<
f(decoded.createObject()); std::function<void(std::shared_ptr<NodeObject>)>*>(fp));
}, reinterpret_cast<void*>(&f)); f(decoded.createObject());
},
reinterpret_cast<void*>(&f));
} }
int int

File diff suppressed because it is too large Load Diff