mirror of
https://github.com/XRPLF/rippled.git
synced 2025-11-04 11:15:56 +00:00
Revert support for deterministic shards:
Commit 4dc08f8202 introduced support for
deterministic shards, which makes the sharding functionality provided
by rippled more useful.
After merging, several opportunities for further improvements to the
deterministic sharding implementation were identified and a significant
increase int memory usage during shard finalization was detected.
Because of these issues, the commit is being reverted and the feature is
being rolled back. It will be reintroduced in a future release.
This commit is contained in:
manojsdoshi
committed by
Nik Bougalis
Nik Bougalis
parent
12c0e8148b
commit
dbd5f0073e
@@ -515,7 +515,6 @@ target_sources (rippled PRIVATE
|
||||
src/ripple/nodestore/impl/DatabaseNodeImp.cpp
|
||||
src/ripple/nodestore/impl/DatabaseRotatingImp.cpp
|
||||
src/ripple/nodestore/impl/DatabaseShardImp.cpp
|
||||
src/ripple/nodestore/impl/DeterministicShard.cpp
|
||||
src/ripple/nodestore/impl/DecodedBlob.cpp
|
||||
src/ripple/nodestore/impl/DummyScheduler.cpp
|
||||
src/ripple/nodestore/impl/EncodedBlob.cpp
|
||||
|
||||
@@ -58,25 +58,6 @@ public:
|
||||
virtual void
|
||||
open(bool createIfMissing = true) = 0;
|
||||
|
||||
/** Open the backend.
|
||||
@param createIfMissing Create the database files if necessary.
|
||||
@param appType Deterministic appType used to create a backend.
|
||||
@param uid Deterministic uid used to create a backend.
|
||||
@param salt Deterministic salt used to create a backend.
|
||||
This allows the caller to catch exceptions.
|
||||
*/
|
||||
virtual void
|
||||
open(
|
||||
bool createIfMissing,
|
||||
boost::optional<uint64_t> appType,
|
||||
boost::optional<uint64_t> uid,
|
||||
boost::optional<uint64_t> salt)
|
||||
{
|
||||
Throw<std::runtime_error>(std::string(
|
||||
"Deterministic appType/uid/salt not supported by backend " +
|
||||
getName()));
|
||||
}
|
||||
|
||||
/** Close the backend.
|
||||
This allows the caller to catch exceptions.
|
||||
*/
|
||||
|
||||
@@ -1,103 +0,0 @@
|
||||
# Deterministic Database Shards
|
||||
|
||||
This doc describes the standard way to assemble the database shard. A shard assembled using this approach becomes deterministic i.e. if two independent sides assemble the shard consists of the same ledgers, accounts and transactions, then they will obtain the same shard files `nudb.dat` and `nudb.key`. The approach deals with the `NuDB` database format only, refer to `https://github.com/vinniefalco/NuDB`.
|
||||
|
||||
|
||||
## Headers
|
||||
|
||||
Due to NuDB database definition, the following headers are used for database files:
|
||||
|
||||
nudb.key:
|
||||
```
|
||||
char[8] Type The characters "nudb.key"
|
||||
uint16 Version Holds the version number
|
||||
uint64 UID Unique ID generated on creation
|
||||
uint64 Appnum Application defined constant
|
||||
uint16 KeySize Key size in bytes
|
||||
uint64 Salt A random seed
|
||||
uint64 Pepper The salt hashed
|
||||
uint16 BlockSize Size of a file block in bytes
|
||||
uint16 LoadFactor Target fraction in 65536ths
|
||||
uint8[56] Reserved Zeroes
|
||||
uint8[] Reserved Zero-pad to block size
|
||||
```
|
||||
|
||||
nudb.dat:
|
||||
```
|
||||
char[8] Type The characters "nudb.dat"
|
||||
uint16 Version Holds the version number
|
||||
uint64 UID Unique ID generated on creation
|
||||
uint64 Appnum Application defined constant
|
||||
uint16 KeySize Key size in bytes
|
||||
uint8[64] (reserved) Zeroes
|
||||
```
|
||||
there all fields are saved using network byte order (most significant byte first).
|
||||
|
||||
To make the shard deterministic the following parameters are used as values of header field both for `nudb.key` and `nudb.dat` files.
|
||||
```
|
||||
Version 2
|
||||
UID digest(0)
|
||||
Appnum digest(2) | 0x5348524400000000 /* 'SHRD' */
|
||||
KeySize 32
|
||||
Salt digest(1)
|
||||
Pepper XXH64(Salt)
|
||||
BlockSize 0x1000 (4096 bytes)
|
||||
LoadFactor 0.5 (numeric 0x8000)
|
||||
```
|
||||
Note: XXH64() is well-known hash algorithm.
|
||||
|
||||
The `digest(i)` mentioned above defined as the follows:
|
||||
|
||||
First, RIPEMD160 hash `H` calculated of the following structure
|
||||
```
|
||||
uint256 lastHash Hash of last ledger in shard
|
||||
uint32 index Index of the shard
|
||||
uint32 firstSeq Sequence number of first ledger in the shard
|
||||
uint32 lastSeq Sequence number of last ledger in the shard
|
||||
uint32 version Version of shard, 2 at the present
|
||||
```
|
||||
there all 32-bit integers are hashed in network byte order.
|
||||
|
||||
Then, `digest(i)` is defined as the following portion of the above hash `H`:
|
||||
```
|
||||
digest(0) = H[0] << 56 | H[2] << 48 | ... | H[14] << 0,
|
||||
digest(1) = H[1] << 56 | H[3] << 48 | ... | H[15] << 0,
|
||||
digest(2) = H[19] << 24 | H[18] << 16 | ... | H[16] << 0,
|
||||
```
|
||||
where `H[i]` denotes `i`-th byte of hash `H`.
|
||||
|
||||
|
||||
## Contents
|
||||
|
||||
After deterministic shard is created using the above mentioned headers, it filled with objects. First, all objects of the shard are collected and sorted in according to their hashes. Here the objects are: ledgers, SHAmap tree nodes including accounts and transactions, and final key object with hash 0. Objects are sorted by increasing of their hashes, precisely, by increasing of hex representations of hashes in lexicographic order.
|
||||
|
||||
For example, the following is an example of sorted hashes in their hex representation:
|
||||
```
|
||||
0000000000000000000000000000000000000000000000000000000000000000
|
||||
154F29A919B30F50443A241C466691B046677C923EE7905AB97A4DBE8A5C2423
|
||||
2231553FC01D37A66C61BBEEACBB8C460994493E5659D118E19A8DDBB1444273
|
||||
272DCBFD8E4D5D786CF11A5444B30FB35435933B5DE6C660AA46E68CF0F5C447
|
||||
3C062FD9F0BCDCA31ACEBCD8E530D0BDAD1F1D1257B89C435616506A3EE6CB9E
|
||||
58A0E5AE427CDDC1C7C06448E8C3E4BF718DE036D827881624B20465C3E1334F
|
||||
...
|
||||
```
|
||||
|
||||
Finally, objects added to the shard one by one in the sorted order from low to high hashes.
|
||||
|
||||
|
||||
## Tests
|
||||
|
||||
To perform test to deterministic shards implementation one can enter the following command:
|
||||
```
|
||||
rippled --unittest ripple.NodeStore.DatabaseShard
|
||||
```
|
||||
|
||||
The following is the right output of deterministic shards test:
|
||||
```
|
||||
ripple.NodeStore.DatabaseShard DatabaseShard deterministic_shard with backend nudb
|
||||
Iteration 0: RIPEMD160[nudb.key] = 4CFA8985836B549EC99D2E9705707F488DC91E4E
|
||||
Iteration 0: RIPEMD160[nudb.dat] = 8CC61F503C36339803F8C2FC652C1102DDB889F1
|
||||
Iteration 1: RIPEMD160[nudb.key] = 4CFA8985836B549EC99D2E9705707F488DC91E4E
|
||||
Iteration 1: RIPEMD160[nudb.dat] = 8CC61F503C36339803F8C2FC652C1102DDB889F1
|
||||
```
|
||||
|
||||
@@ -38,10 +38,7 @@ namespace NodeStore {
|
||||
class NuDBBackend : public Backend
|
||||
{
|
||||
public:
|
||||
static constexpr std::uint64_t currentType = 1;
|
||||
static constexpr std::uint64_t deterministicType = 0x5348524400000000ull;
|
||||
/* "SHRD" in ASCII */
|
||||
static constexpr std::uint64_t deterministicMask = 0xFFFFFFFF00000000ull;
|
||||
static constexpr std::size_t currentType = 1;
|
||||
|
||||
beast::Journal const j_;
|
||||
size_t const keyBytes_;
|
||||
@@ -96,11 +93,7 @@ public:
|
||||
}
|
||||
|
||||
void
|
||||
open(
|
||||
bool createIfMissing,
|
||||
boost::optional<uint64_t> appType,
|
||||
boost::optional<uint64_t> uid,
|
||||
boost::optional<uint64_t> salt) override
|
||||
open(bool createIfMissing) override
|
||||
{
|
||||
using namespace boost::filesystem;
|
||||
if (db_.is_open())
|
||||
@@ -121,9 +114,8 @@ public:
|
||||
dp,
|
||||
kp,
|
||||
lp,
|
||||
appType.value_or(currentType),
|
||||
uid.value_or(nudb::make_uid()),
|
||||
salt.value_or(nudb::make_salt()),
|
||||
currentType,
|
||||
nudb::make_salt(),
|
||||
keyBytes_,
|
||||
nudb::block_size(kp),
|
||||
0.50,
|
||||
@@ -136,27 +128,10 @@ public:
|
||||
db_.open(dp, kp, lp, ec);
|
||||
if (ec)
|
||||
Throw<nudb::system_error>(ec);
|
||||
|
||||
/** Old value currentType is accepted for appnum in traditional
|
||||
* databases, new value is used for deterministic shard databases.
|
||||
* New 64-bit value is constructed from fixed and random parts.
|
||||
* Fixed part is bounded by bitmask deterministicMask,
|
||||
* and the value of fixed part is deterministicType.
|
||||
* Random part depends on the contents of the shard and may be any.
|
||||
* The contents of appnum field should match either old or new rule.
|
||||
*/
|
||||
if (db_.appnum() != appType.value_or(currentType) &&
|
||||
(appType ||
|
||||
(db_.appnum() & deterministicMask) != deterministicType))
|
||||
if (db_.appnum() != currentType)
|
||||
Throw<std::runtime_error>("nodestore: unknown appnum");
|
||||
}
|
||||
|
||||
void
|
||||
open(bool createIfMissing) override
|
||||
{
|
||||
open(createIfMissing, boost::none, boost::none, boost::none);
|
||||
}
|
||||
|
||||
void
|
||||
close() override
|
||||
{
|
||||
|
||||
@@ -1,206 +0,0 @@
|
||||
//------------------------------------------------------------------------------
|
||||
/*
|
||||
This file is part of rippled: https://github.com/ripple/rippled
|
||||
Copyright (c) 2020 Ripple Labs Inc.
|
||||
|
||||
Permission to use, copy, modify, and/or distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#include <ripple/app/main/Application.h>
|
||||
#include <ripple/beast/hash/hash_append.h>
|
||||
#include <ripple/core/ConfigSections.h>
|
||||
#include <ripple/nodestore/Manager.h>
|
||||
#include <ripple/nodestore/impl/DeterministicShard.h>
|
||||
#include <ripple/nodestore/impl/Shard.h>
|
||||
#include <ripple/protocol/digest.h>
|
||||
#include <fstream>
|
||||
#include <nudb/detail/format.hpp>
|
||||
#include <nudb/nudb.hpp>
|
||||
#include <openssl/ripemd.h>
|
||||
|
||||
namespace ripple {
|
||||
namespace NodeStore {
|
||||
|
||||
DeterministicShard::DeterministicShard(
|
||||
Application& app,
|
||||
DatabaseShard const& db,
|
||||
std::uint32_t index,
|
||||
uint256 const& lastHash,
|
||||
beast::Journal j)
|
||||
: inited_(false)
|
||||
, nodeset_([](nodeptr l, nodeptr r) { return l->getHash() < r->getHash(); })
|
||||
, app_(app)
|
||||
, db_(db)
|
||||
, index_(index)
|
||||
, hash_(hash(lastHash))
|
||||
, tempdir_(db.getRootDir() / (std::to_string(index_) + ".tmp"))
|
||||
, finaldir_(db.getRootDir() / std::to_string(index_))
|
||||
, ctx_(std::make_unique<nudb::context>())
|
||||
, j_(j)
|
||||
{
|
||||
}
|
||||
|
||||
DeterministicShard::~DeterministicShard()
|
||||
{
|
||||
close(true);
|
||||
}
|
||||
|
||||
uint160
|
||||
DeterministicShard::hash(uint256 const& lastHash) const
|
||||
{
|
||||
using beast::hash_append;
|
||||
ripemd160_hasher h;
|
||||
|
||||
hash_append(h, lastHash);
|
||||
hash_append(h, index_);
|
||||
hash_append(h, db_.firstLedgerSeq(index_));
|
||||
hash_append(h, db_.lastLedgerSeq(index_));
|
||||
hash_append(h, Shard::version);
|
||||
|
||||
auto const result = static_cast<ripemd160_hasher::result_type>(h);
|
||||
return uint160::fromVoid(result.data());
|
||||
}
|
||||
|
||||
std::uint64_t
|
||||
DeterministicShard::digest(int n) const
|
||||
{
|
||||
auto const data = hash_.data();
|
||||
|
||||
if (n == 2)
|
||||
{ // Extract 32 bits:
|
||||
return (static_cast<std::uint64_t>(data[19]) << 24) +
|
||||
(static_cast<std::uint64_t>(data[18]) << 16) +
|
||||
(static_cast<std::uint64_t>(data[17]) << 8) +
|
||||
(static_cast<std::uint64_t>(data[16]));
|
||||
}
|
||||
|
||||
std::uint64_t ret = 0;
|
||||
|
||||
if (n == 0 || n == 1)
|
||||
{ // Extract 64 bits
|
||||
for (int i = n; i < 16; i += 2)
|
||||
ret = (ret << 8) + data[i];
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool
|
||||
DeterministicShard::init()
|
||||
{
|
||||
if (index_ < db_.earliestShardIndex())
|
||||
{
|
||||
JLOG(j_.error()) << "shard " << index_ << " is illegal";
|
||||
return false;
|
||||
}
|
||||
|
||||
Config const& config{app_.config()};
|
||||
|
||||
Section section{config.section(ConfigSection::shardDatabase())};
|
||||
std::string const type{get<std::string>(section, "type", "nudb")};
|
||||
|
||||
if (type != "nudb")
|
||||
{
|
||||
JLOG(j_.error()) << "shard " << index_ << " backend type " << type
|
||||
<< " not supported";
|
||||
return false;
|
||||
}
|
||||
|
||||
auto factory{Manager::instance().find(type)};
|
||||
if (!factory)
|
||||
{
|
||||
JLOG(j_.error()) << "shard " << index_
|
||||
<< " failed to create factory for backend type "
|
||||
<< type;
|
||||
return false;
|
||||
}
|
||||
|
||||
ctx_->start();
|
||||
|
||||
section.set("path", tempdir_.string());
|
||||
backend_ = factory->createInstance(
|
||||
NodeObject::keyBytes, section, scheduler_, *ctx_, j_);
|
||||
|
||||
if (!backend_)
|
||||
{
|
||||
JLOG(j_.error()) << "shard " << index_
|
||||
<< " failed to create backend type " << type;
|
||||
return false;
|
||||
}
|
||||
|
||||
// Open or create the NuDB key/value store
|
||||
bool preexist = exists(tempdir_);
|
||||
if (preexist)
|
||||
{
|
||||
remove_all(tempdir_);
|
||||
preexist = false;
|
||||
}
|
||||
|
||||
backend_->open(
|
||||
!preexist,
|
||||
digest(2) | 0x5348524400000000ll, /* appType */
|
||||
digest(0), /* uid */
|
||||
digest(1) /* salt */
|
||||
);
|
||||
|
||||
inited_ = true;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void
|
||||
DeterministicShard::close(bool cancel)
|
||||
{
|
||||
if (!inited_)
|
||||
return;
|
||||
|
||||
backend_->close();
|
||||
if (cancel)
|
||||
{
|
||||
remove_all(tempdir_);
|
||||
}
|
||||
else
|
||||
{
|
||||
flush();
|
||||
remove_all(finaldir_);
|
||||
rename(tempdir_, finaldir_);
|
||||
}
|
||||
inited_ = false;
|
||||
}
|
||||
|
||||
void
|
||||
DeterministicShard::store(nodeptr nObj)
|
||||
{
|
||||
if (!inited_)
|
||||
return;
|
||||
|
||||
nodeset_.insert(nObj);
|
||||
}
|
||||
|
||||
void
|
||||
DeterministicShard::flush()
|
||||
{
|
||||
if (!inited_)
|
||||
return;
|
||||
|
||||
for (auto nObj : nodeset_)
|
||||
{
|
||||
backend_->store(nObj);
|
||||
}
|
||||
|
||||
nodeset_.clear();
|
||||
}
|
||||
|
||||
} // namespace NodeStore
|
||||
} // namespace ripple
|
||||
@@ -1,144 +0,0 @@
|
||||
//------------------------------------------------------------------------------
|
||||
/*
|
||||
This file is part of rippled: https://github.com/ripple/rippled
|
||||
Copyright (c) 2020 Ripple Labs Inc.
|
||||
|
||||
Permission to use, copy, modify, and/or distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#ifndef RIPPLE_NODESTORE_DETERMINISTICSHARD_H_INCLUDED
|
||||
#define RIPPLE_NODESTORE_DETERMINISTICSHARD_H_INCLUDED
|
||||
|
||||
#include <ripple/nodestore/DatabaseShard.h>
|
||||
#include <ripple/nodestore/DummyScheduler.h>
|
||||
#include <nudb/nudb.hpp>
|
||||
#include <set>
|
||||
|
||||
namespace ripple {
|
||||
namespace NodeStore {
|
||||
|
||||
/** DeterministicShard class.
|
||||
*
|
||||
* 1. The init() method creates temporary folder tempdir_,
|
||||
* and the deterministic shard is initialized in that folder.
|
||||
* 2. The store() method adds object to memory pool.
|
||||
* 3. The flush() method stores all objects from memory pool to the shard
|
||||
* located in tempdir_ in sorted order.
|
||||
* 4. The close(true) method finalizes the shard and moves it from tempdir_
|
||||
* temporary folder to filandir_ permanent folder,
|
||||
* deleting old (non-deterministic) shard located in finaldir_.
|
||||
*/
|
||||
class DeterministicShard
|
||||
{
|
||||
public:
|
||||
using nodeptr = std::shared_ptr<NodeObject>;
|
||||
|
||||
DeterministicShard(DeterministicShard const&) = delete;
|
||||
DeterministicShard&
|
||||
operator=(DeterministicShard const&) = delete;
|
||||
|
||||
/** Creates the object for shard database
|
||||
*
|
||||
* @param app Application object
|
||||
* @param db Shard Database which deterministic shard belongs to
|
||||
* @param index Index of the shard
|
||||
* @param lastHash Hash of last ledger in the shard
|
||||
* @param j Journal to logging
|
||||
*/
|
||||
DeterministicShard(
|
||||
Application& app,
|
||||
DatabaseShard const& db,
|
||||
std::uint32_t index,
|
||||
uint256 const& lastHash,
|
||||
beast::Journal j);
|
||||
|
||||
~DeterministicShard();
|
||||
|
||||
/** Initializes the deterministic shard.
|
||||
*
|
||||
* @return true is success, false if errored
|
||||
*/
|
||||
bool
|
||||
init();
|
||||
|
||||
/** Finalizes and closes the shard.
|
||||
*
|
||||
* @param cancel True if reject the shard and delete all files,
|
||||
* false if finalize the shard and store them
|
||||
*/
|
||||
void
|
||||
close(bool cancel = false);
|
||||
|
||||
/** Store the object into memory pool
|
||||
*
|
||||
* @param nobj Object to store.
|
||||
*/
|
||||
void
|
||||
store(nodeptr nobj);
|
||||
|
||||
/** Flush all objects from memory pool to shard
|
||||
*/
|
||||
void
|
||||
flush();
|
||||
|
||||
private:
|
||||
// Count hash of shard parameters: lashHash, firstSeq, lastSeq, index
|
||||
uint160
|
||||
hash(const uint256& lastHash) const;
|
||||
|
||||
// Get n-th 64-bit portion of shard parameters's hash
|
||||
std::uint64_t
|
||||
digest(int n) const;
|
||||
|
||||
// If database inited
|
||||
bool inited_;
|
||||
|
||||
// Sorted set of stored and not flushed objects
|
||||
std::set<nodeptr, std::function<bool(nodeptr, nodeptr)>> nodeset_;
|
||||
|
||||
// Application reference
|
||||
Application& app_;
|
||||
|
||||
// Shard database
|
||||
DatabaseShard const& db_;
|
||||
|
||||
// Shard Index
|
||||
std::uint32_t const index_;
|
||||
|
||||
// Hash used for digests
|
||||
uint160 const hash_;
|
||||
|
||||
// Path to temporary database files
|
||||
boost::filesystem::path const tempdir_;
|
||||
|
||||
// Path to final database files
|
||||
boost::filesystem::path const finaldir_;
|
||||
|
||||
// Dummy scheduler for deterministic write
|
||||
DummyScheduler scheduler_;
|
||||
|
||||
// NuDB context
|
||||
std::unique_ptr<nudb::context> ctx_;
|
||||
|
||||
// NuDB key/value store for node objects
|
||||
std::shared_ptr<Backend> backend_;
|
||||
|
||||
// Journal
|
||||
beast::Journal const j_;
|
||||
};
|
||||
|
||||
} // namespace NodeStore
|
||||
} // namespace ripple
|
||||
|
||||
#endif
|
||||
@@ -40,7 +40,6 @@ Shard::Shard(
|
||||
std::uint32_t index,
|
||||
beast::Journal j)
|
||||
: app_(app)
|
||||
, db_(db)
|
||||
, index_(index)
|
||||
, firstSeq_(db.firstLedgerSeq(index))
|
||||
, lastSeq_(std::max(firstSeq_, db.lastLedgerSeq(index)))
|
||||
@@ -393,8 +392,7 @@ Shard::isLegacy() const
|
||||
bool
|
||||
Shard::finalize(
|
||||
bool const writeSQLite,
|
||||
boost::optional<uint256> const& expectedHash,
|
||||
const bool writeDeterministicShard)
|
||||
boost::optional<uint256> const& expectedHash)
|
||||
{
|
||||
assert(backend_);
|
||||
|
||||
@@ -508,17 +506,6 @@ Shard::finalize(
|
||||
std::shared_ptr<Ledger const> next;
|
||||
auto const lastLedgerHash{hash};
|
||||
|
||||
std::shared_ptr<DeterministicShard> dsh;
|
||||
if (writeDeterministicShard)
|
||||
{
|
||||
dsh = std::make_shared<DeterministicShard>(
|
||||
app_, db_, index_, lastLedgerHash, j_);
|
||||
if (!dsh->init())
|
||||
{
|
||||
return fail("can't create deterministic shard");
|
||||
}
|
||||
}
|
||||
|
||||
// Start with the last ledger in the shard and walk backwards from
|
||||
// child to parent until we reach the first ledger
|
||||
seq = lastSeq_;
|
||||
@@ -555,11 +542,8 @@ Shard::finalize(
|
||||
return fail("missing root TXN node");
|
||||
}
|
||||
|
||||
if (dsh)
|
||||
dsh->store(nObj);
|
||||
|
||||
if (!verifyLedger(ledger, next, dsh))
|
||||
return fail("verification check failed");
|
||||
if (!valLedger(ledger, next))
|
||||
return fail("failed to validate ledger");
|
||||
|
||||
if (writeSQLite)
|
||||
{
|
||||
@@ -620,12 +604,6 @@ Shard::finalize(
|
||||
{
|
||||
backend_->store(nObj);
|
||||
|
||||
if (dsh)
|
||||
{
|
||||
dsh->store(nObj);
|
||||
dsh->flush();
|
||||
}
|
||||
|
||||
std::lock_guard lock(mutex_);
|
||||
final_ = true;
|
||||
|
||||
@@ -645,23 +623,6 @@ Shard::finalize(
|
||||
std::string("exception ") + e.what() + " in function " + __func__);
|
||||
}
|
||||
|
||||
if (dsh)
|
||||
{
|
||||
/* Close non-deterministic shard database. */
|
||||
backend_->close();
|
||||
/* Replace non-deterministic shard by deterministic one. */
|
||||
dsh->close();
|
||||
/* Re-open deterministic shard database. */
|
||||
backend_->open(false);
|
||||
/** The finalize() function verifies the shard and, if third parameter
|
||||
* is true, then replaces the shard by deterministic copy of the shard.
|
||||
* After deterministic shard is created it verifies again,
|
||||
* the finalize() function called here to verify deterministic shard,
|
||||
* third parameter is false.
|
||||
*/
|
||||
return finalize(false, expectedHash, false);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -964,10 +925,9 @@ Shard::setFileStats(std::lock_guard<std::recursive_mutex> const&)
|
||||
}
|
||||
|
||||
bool
|
||||
Shard::verifyLedger(
|
||||
Shard::valLedger(
|
||||
std::shared_ptr<Ledger const> const& ledger,
|
||||
std::shared_ptr<Ledger const> const& next,
|
||||
std::shared_ptr<DeterministicShard> dsh) const
|
||||
std::shared_ptr<Ledger const> const& next) const
|
||||
{
|
||||
auto fail = [j = j_, index = index_, &ledger](std::string const& msg) {
|
||||
JLOG(j.fatal()) << "shard " << index << ". " << msg
|
||||
@@ -986,14 +946,11 @@ Shard::verifyLedger(
|
||||
return fail("Invalid ledger account hash");
|
||||
|
||||
bool error{false};
|
||||
auto visit = [this, &error, dsh](SHAMapAbstractNode& node) {
|
||||
auto visit = [this, &error](SHAMapAbstractNode& node) {
|
||||
if (stop_)
|
||||
return false;
|
||||
auto nObj = valFetch(node.getNodeHash().as_uint256());
|
||||
if (!nObj)
|
||||
if (!valFetch(node.getNodeHash().as_uint256()))
|
||||
error = true;
|
||||
else if (dsh)
|
||||
dsh->store(nObj);
|
||||
return !error;
|
||||
};
|
||||
|
||||
|
||||
@@ -26,7 +26,6 @@
|
||||
#include <ripple/core/DatabaseCon.h>
|
||||
#include <ripple/nodestore/NodeObject.h>
|
||||
#include <ripple/nodestore/Scheduler.h>
|
||||
#include <ripple/nodestore/impl/DeterministicShard.h>
|
||||
|
||||
#include <boost/filesystem.hpp>
|
||||
#include <nudb/nudb.hpp>
|
||||
@@ -129,14 +128,11 @@ public:
|
||||
verified backend data.
|
||||
@param referenceHash If present, this hash must match the hash
|
||||
of the last ledger in the shard.
|
||||
@param writeDeterministicShard If true, shard will be rewritten
|
||||
deterministically.
|
||||
*/
|
||||
bool
|
||||
finalize(
|
||||
bool const writeSQLite,
|
||||
boost::optional<uint256> const& referenceHash,
|
||||
const bool writeDeterministicShard = true);
|
||||
boost::optional<uint256> const& referenceHash);
|
||||
|
||||
void
|
||||
stop()
|
||||
@@ -174,8 +170,6 @@ private:
|
||||
Application& app_;
|
||||
mutable std::recursive_mutex mutex_;
|
||||
|
||||
DatabaseShard const& db_;
|
||||
|
||||
// Shard Index
|
||||
std::uint32_t const index_;
|
||||
|
||||
@@ -258,12 +252,10 @@ private:
|
||||
setFileStats(std::lock_guard<std::recursive_mutex> const& lock);
|
||||
|
||||
// Validate this ledger by walking its SHAMaps and verifying Merkle trees
|
||||
// If dsh != NULL then save all walking SHAMaps to deterministic shard dsh
|
||||
bool
|
||||
verifyLedger(
|
||||
valLedger(
|
||||
std::shared_ptr<Ledger const> const& ledger,
|
||||
std::shared_ptr<Ledger const> const& next,
|
||||
std::shared_ptr<DeterministicShard> dsh = {}) const;
|
||||
std::shared_ptr<Ledger const> const& next) const;
|
||||
|
||||
// Fetches from backend and log errors based on status codes
|
||||
std::shared_ptr<NodeObject>
|
||||
|
||||
@@ -19,7 +19,6 @@
|
||||
|
||||
#include <ripple/app/ledger/LedgerMaster.h>
|
||||
#include <ripple/app/ledger/LedgerToJson.h>
|
||||
#include <ripple/beast/hash/hash_append.h>
|
||||
#include <ripple/beast/utility/temp_dir.h>
|
||||
#include <ripple/core/ConfigSections.h>
|
||||
#include <ripple/nodestore/DatabaseShard.h>
|
||||
@@ -28,135 +27,13 @@
|
||||
#include <ripple/nodestore/impl/DecodedBlob.h>
|
||||
#include <ripple/nodestore/impl/EncodedBlob.h>
|
||||
#include <ripple/nodestore/impl/Shard.h>
|
||||
#include <ripple/protocol/digest.h>
|
||||
#include <boost/algorithm/hex.hpp>
|
||||
#include <chrono>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <openssl/ripemd.h>
|
||||
#include <test/jtx.h>
|
||||
#include <test/nodestore/TestBase.h>
|
||||
|
||||
namespace ripple {
|
||||
namespace NodeStore {
|
||||
|
||||
/** std::uniform_int_distribution is platform dependent.
|
||||
* Unit test for deterministic shards is the following: it generates
|
||||
* predictable accounts and transactions, packs them into ledgers
|
||||
* and makes the shard. The hash of this shard should be equal to the
|
||||
* given value. On different platforms (precisely, Linux and Mac)
|
||||
* hashes of the resulting shard was different. It was unvestigated
|
||||
* that the problem is in the class std::uniform_int_distribution
|
||||
* which generates different pseudorandom sequences on different
|
||||
* platforms, but we need predictable sequence.
|
||||
*/
|
||||
template <class IntType = int>
|
||||
struct uniformIntDistribution
|
||||
{
|
||||
using resultType = IntType;
|
||||
|
||||
const resultType A, B;
|
||||
|
||||
struct paramType
|
||||
{
|
||||
const resultType A, B;
|
||||
|
||||
paramType(resultType aa, resultType bb) : A(aa), B(bb)
|
||||
{
|
||||
}
|
||||
};
|
||||
|
||||
explicit uniformIntDistribution(
|
||||
const resultType a = 0,
|
||||
const resultType b = std::numeric_limits<resultType>::max())
|
||||
: A(a), B(b)
|
||||
{
|
||||
}
|
||||
|
||||
explicit uniformIntDistribution(const paramType& params)
|
||||
: A(params.A), B(params.B)
|
||||
{
|
||||
}
|
||||
|
||||
template <class Generator>
|
||||
resultType
|
||||
operator()(Generator& g) const
|
||||
{
|
||||
return rnd(g, A, B);
|
||||
}
|
||||
|
||||
template <class Generator>
|
||||
resultType
|
||||
operator()(Generator& g, const paramType& params) const
|
||||
{
|
||||
return rnd(g, params.A, params.B);
|
||||
}
|
||||
|
||||
resultType
|
||||
a() const
|
||||
{
|
||||
return A;
|
||||
}
|
||||
|
||||
resultType
|
||||
b() const
|
||||
{
|
||||
return B;
|
||||
}
|
||||
|
||||
resultType
|
||||
min() const
|
||||
{
|
||||
return A;
|
||||
}
|
||||
|
||||
resultType
|
||||
max() const
|
||||
{
|
||||
return B;
|
||||
}
|
||||
|
||||
private:
|
||||
template <class Generator>
|
||||
resultType
|
||||
rnd(Generator& g, const resultType a, const resultType b) const
|
||||
{
|
||||
static_assert(
|
||||
std::is_convertible<typename Generator::result_type, resultType>::
|
||||
value,
|
||||
"Ups...");
|
||||
static_assert(
|
||||
Generator::min() == 0, "If non-zero we have handle the offset");
|
||||
const resultType range = b - a + 1;
|
||||
assert(Generator::max() >= range); // Just for safety
|
||||
const resultType rejectLim = g.max() % range;
|
||||
resultType n;
|
||||
do
|
||||
n = g();
|
||||
while (n <= rejectLim);
|
||||
return (n % range) + a;
|
||||
}
|
||||
};
|
||||
|
||||
template <class Engine, class Integral>
|
||||
Integral
|
||||
randInt(Engine& engine, Integral min, Integral max)
|
||||
{
|
||||
assert(max > min);
|
||||
|
||||
// This should have no state and constructing it should
|
||||
// be very cheap. If that turns out not to be the case
|
||||
// it could be hand-optimized.
|
||||
return uniformIntDistribution<Integral>(min, max)(engine);
|
||||
}
|
||||
|
||||
template <class Engine, class Integral>
|
||||
Integral
|
||||
randInt(Engine& engine, Integral max)
|
||||
{
|
||||
return randInt(engine, Integral(0), max);
|
||||
}
|
||||
|
||||
// Tests DatabaseShard class
|
||||
//
|
||||
class DatabaseShard_test : public TestBase
|
||||
@@ -210,7 +87,7 @@ class DatabaseShard_test : public TestBase
|
||||
{
|
||||
int p;
|
||||
if (n >= 2)
|
||||
p = randInt(rng_, 2 * dataSize);
|
||||
p = rand_int(rng_, 2 * dataSize);
|
||||
else
|
||||
p = 0;
|
||||
|
||||
@@ -222,27 +99,27 @@ class DatabaseShard_test : public TestBase
|
||||
int from, to;
|
||||
do
|
||||
{
|
||||
from = randInt(rng_, n - 1);
|
||||
to = randInt(rng_, n - 1);
|
||||
from = rand_int(rng_, n - 1);
|
||||
to = rand_int(rng_, n - 1);
|
||||
} while (from == to);
|
||||
|
||||
pay.push_back(std::make_pair(from, to));
|
||||
}
|
||||
|
||||
n += !randInt(rng_, nLedgers / dataSize);
|
||||
n += !rand_int(rng_, nLedgers / dataSize);
|
||||
|
||||
if (n > accounts_.size())
|
||||
{
|
||||
char str[9];
|
||||
for (int j = 0; j < 8; ++j)
|
||||
str[j] = 'a' + randInt(rng_, 'z' - 'a');
|
||||
str[j] = 'a' + rand_int(rng_, 'z' - 'a');
|
||||
str[8] = 0;
|
||||
accounts_.emplace_back(str);
|
||||
}
|
||||
|
||||
nAccounts_.push_back(n);
|
||||
payAccounts_.push_back(std::move(pay));
|
||||
xrpAmount_.push_back(randInt(rng_, 90) + 10);
|
||||
xrpAmount_.push_back(rand_int(rng_, 90) + 10);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -618,7 +495,7 @@ class DatabaseShard_test : public TestBase
|
||||
}
|
||||
|
||||
std::optional<int>
|
||||
createShard(TestData& data, DatabaseShard& db, int maxShardNumber = 1)
|
||||
createShard(TestData& data, DatabaseShard& db, int maxShardNumber)
|
||||
{
|
||||
int shardNumber = -1;
|
||||
|
||||
@@ -792,7 +669,7 @@ class DatabaseShard_test : public TestBase
|
||||
|
||||
for (std::uint32_t i = 0; i < nTestShards * 2; ++i)
|
||||
{
|
||||
std::uint32_t n = randInt(data.rng_, nTestShards - 1) + 1;
|
||||
std::uint32_t n = rand_int(data.rng_, nTestShards - 1) + 1;
|
||||
if (bitMask & (1ll << n))
|
||||
{
|
||||
db->removePreShard(n);
|
||||
@@ -1101,90 +978,6 @@ class DatabaseShard_test : public TestBase
|
||||
}
|
||||
}
|
||||
|
||||
std::string
|
||||
ripemd160File(std::string filename)
|
||||
{
|
||||
using beast::hash_append;
|
||||
std::ifstream input(filename, std::ios::in | std::ios::binary);
|
||||
char buf[4096];
|
||||
ripemd160_hasher h;
|
||||
|
||||
while (input.read(buf, 4096), input.gcount() > 0)
|
||||
hash_append(h, buf, input.gcount());
|
||||
|
||||
auto const binResult = static_cast<ripemd160_hasher::result_type>(h);
|
||||
const auto charDigest = binResult.data();
|
||||
std::string result;
|
||||
boost::algorithm::hex(
|
||||
charDigest,
|
||||
charDigest + sizeof(binResult),
|
||||
std::back_inserter(result));
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
void
|
||||
testDeterministicShard(
|
||||
std::string const& backendType,
|
||||
std::uint64_t const seedValue)
|
||||
{
|
||||
using namespace test::jtx;
|
||||
|
||||
std::string ripemd160Key("4CFA8985836B549EC99D2E9705707F488DC91E4E"),
|
||||
ripemd160Dat("8CC61F503C36339803F8C2FC652C1102DDB889F1");
|
||||
|
||||
for (int i = 0; i < 2; i++)
|
||||
{
|
||||
beast::temp_dir shardDir;
|
||||
{
|
||||
Env env{
|
||||
*this,
|
||||
testConfig(
|
||||
(i ? "" : "deterministicShard"),
|
||||
backendType,
|
||||
shardDir.path())};
|
||||
DatabaseShard* db = env.app().getShardStore();
|
||||
BEAST_EXPECT(db);
|
||||
|
||||
TestData data(seedValue, 4);
|
||||
if (!BEAST_EXPECT(data.makeLedgers(env)))
|
||||
return;
|
||||
|
||||
if (createShard(data, *db) < 0)
|
||||
return;
|
||||
}
|
||||
{
|
||||
Env env{*this, testConfig("", backendType, shardDir.path())};
|
||||
DatabaseShard* db = env.app().getShardStore();
|
||||
BEAST_EXPECT(db);
|
||||
|
||||
TestData data(seedValue, 4);
|
||||
if (!BEAST_EXPECT(data.makeLedgers(env)))
|
||||
return;
|
||||
|
||||
waitShard(*db, 1);
|
||||
|
||||
for (std::uint32_t j = 0; j < ledgersPerShard; ++j)
|
||||
checkLedger(data, *db, *data.ledgers_[j]);
|
||||
}
|
||||
|
||||
boost::filesystem::path path(shardDir.path());
|
||||
path /= "1";
|
||||
boost::filesystem::path keypath = path / (backendType + ".key");
|
||||
std::string key = ripemd160File(keypath.string());
|
||||
boost::filesystem::path datpath = path / (backendType + ".dat");
|
||||
std::string dat = ripemd160File(datpath.string());
|
||||
|
||||
std::cerr << "Iteration " << i << ": RIPEMD160[" << backendType
|
||||
<< ".key] = " << key << std::endl;
|
||||
std::cerr << "Iteration " << i << ": RIPEMD160[" << backendType
|
||||
<< ".dat] = " << dat << std::endl;
|
||||
|
||||
BEAST_EXPECT(key == ripemd160Key);
|
||||
BEAST_EXPECT(dat == ripemd160Dat);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
testAll(std::string const& backendType)
|
||||
{
|
||||
@@ -1198,7 +991,6 @@ class DatabaseShard_test : public TestBase
|
||||
testCorruptedDatabase(backendType, seedValue + 40);
|
||||
testIllegalFinalKey(backendType, seedValue + 50);
|
||||
testImport(backendType, seedValue + 60);
|
||||
testDeterministicShard(backendType, seedValue + 70);
|
||||
}
|
||||
|
||||
public:
|
||||
|
||||
Reference in New Issue
Block a user