diff --git a/Builds/VisualStudio2013/RippleD.vcxproj b/Builds/VisualStudio2013/RippleD.vcxproj index 97043cb15..e8987be73 100644 --- a/Builds/VisualStudio2013/RippleD.vcxproj +++ b/Builds/VisualStudio2013/RippleD.vcxproj @@ -794,6 +794,8 @@ True + + @@ -916,7 +918,8 @@ - + + @@ -4064,4 +4067,4 @@ - + diff --git a/Builds/VisualStudio2013/RippleD.vcxproj.filters b/Builds/VisualStudio2013/RippleD.vcxproj.filters index 097acb284..72ee25c31 100644 --- a/Builds/VisualStudio2013/RippleD.vcxproj.filters +++ b/Builds/VisualStudio2013/RippleD.vcxproj.filters @@ -163,6 +163,9 @@ {EADD6FA3-A535-01B1-8B05-B6363E6AE41E} + + {94B0990A-9ABE-B1EC-C220-83FD8C2F529F} + {C8013957-E624-4A24-C0F8-CBAAC144AF09} @@ -1431,6 +1434,9 @@ beast\net\tests + + beast\random + beast @@ -1596,6 +1602,9 @@ beast\unit_test + + beast\unit_test + beast\utility @@ -5443,9 +5452,4 @@ websocket\src - - - beast\unit_test - - - + diff --git a/SConstruct b/SConstruct index ee282cb8c..015bfa48c 100644 --- a/SConstruct +++ b/SConstruct @@ -277,6 +277,8 @@ def config_env(toolchain, variant, env): '-std=c++11', '-Wno-invalid-offsetof']) + env.Append(CPPDEFINES=['_FILE_OFFSET_BITS=64']) + if Beast.system.osx: env.Append(CPPDEFINES={ 'BEAST_COMPILE_OBJECTIVE_CPP': 1, @@ -304,11 +306,10 @@ def config_env(toolchain, variant, env): 'boost_program_options', 'boost_regex', 'boost_system', + 'boost_thread' ] # We prefer static libraries for boost if env.get('BOOST_ROOT'): - # Need to add boost_thread. Not needed when dynamic linking is used. - boost_libs += ['boost_thread'] static_libs = ['%s/stage/lib/lib%s.a' % (env['BOOST_ROOT'], l) for l in boost_libs] if all(os.path.exists(f) for f in static_libs): diff --git a/src/ripple/app/main/Main.cpp b/src/ripple/app/main/Main.cpp index 4b2186bf1..dfd0e84bf 100644 --- a/src/ripple/app/main/Main.cpp +++ b/src/ripple/app/main/Main.cpp @@ -149,7 +149,7 @@ static void setupConfigForUnitTests (Config* config) { - config->nodeDatabase = parseDelimitedKeyValueString ("type=memory"); + config->nodeDatabase = parseDelimitedKeyValueString ("type=memory|path=main"); config->ephemeralNodeDatabase = beast::StringPairArray (); config->importNodeDatabase = beast::StringPairArray (); } diff --git a/src/ripple/app/node/SqliteFactory.cpp b/src/ripple/app/node/SqliteFactory.cpp index 2610ac125..56be1a26b 100644 --- a/src/ripple/app/node/SqliteFactory.cpp +++ b/src/ripple/app/node/SqliteFactory.cpp @@ -73,6 +73,13 @@ public: return m_name; } + void + close() override + { + // VFALCO how do we do this? + assert(false); + } + //-------------------------------------------------------------------------- NodeStore::Status fetch (void const* key, NodeObject::Ptr* pObject) @@ -221,6 +228,11 @@ public: return type; } + void + verify() override + { + } + private: std::string const m_name; std::unique_ptr m_db; diff --git a/src/ripple/basics/BasicConfig.h b/src/ripple/basics/BasicConfig.h index 617374ace..358927a65 100644 --- a/src/ripple/basics/BasicConfig.h +++ b/src/ripple/basics/BasicConfig.h @@ -20,6 +20,7 @@ #ifndef RIPPLE_BASICS_BASICCONFIG_H_INCLUDED #define RIPPLE_BASICS_BASICCONFIG_H_INCLUDED +#include #include #include #include @@ -37,16 +38,18 @@ using IniFileSections = std::map>; A configuration file contains zero or more sections. */ class Section + : public beast::const_container < + std::map > { private: std::string name_; std::vector lines_; std::vector values_; - std::map map_; public: /** Create an empty section. */ - Section (std::string const& name); + explicit + Section (std::string const& name = ""); /** Returns the name of this section. */ std::string const& @@ -55,13 +58,6 @@ public: return name_; } - /** Returns the number of key/value pairs. */ - std::size_t - keys() const - { - return map_.size(); - } - /** Returns all the lines in the section. This includes everything. */ diff --git a/src/ripple/basics/impl/BasicConfig.cpp b/src/ripple/basics/impl/BasicConfig.cpp index bedf4a2f0..7d2de96ce 100644 --- a/src/ripple/basics/impl/BasicConfig.cpp +++ b/src/ripple/basics/impl/BasicConfig.cpp @@ -32,7 +32,7 @@ Section::Section (std::string const& name) void Section::set (std::string const& key, std::string const& value) { - auto const result = map_.emplace (key, value); + auto const result = cont().emplace (key, value); if (! result.second) result.first->second = value; } @@ -68,14 +68,14 @@ Section::append (std::vector const& lines) bool Section::exists (std::string const& name) const { - return map_.find (name) != map_.end(); + return cont().find (name) != cont().end(); } std::pair Section::find (std::string const& name) const { - auto const iter = map_.find (name); - if (iter == map_.end()) + auto const iter = cont().find (name); + if (iter == cont().end()) return {{}, false}; return {iter->second, true}; } @@ -83,7 +83,7 @@ Section::find (std::string const& name) const std::ostream& operator<< (std::ostream& os, Section const& section) { - for (auto const& kv : section.map_) + for (auto const& kv : section.cont()) os << kv.first << "=" << kv.second << "\n"; return os; } @@ -113,7 +113,7 @@ BasicConfig::remap (std::string const& legacy_section, auto const iter = map_.find (legacy_section); if (iter == map_.end()) return; - if (iter->second.keys() != 0) + if (iter->second.size() != 0) return; if (iter->second.lines().size() != 1) return; diff --git a/src/ripple/nodestore/Backend.h b/src/ripple/nodestore/Backend.h index 3b9ab4b54..f8b126e35 100644 --- a/src/ripple/nodestore/Backend.h +++ b/src/ripple/nodestore/Backend.h @@ -50,6 +50,11 @@ public: */ virtual std::string getName() = 0; + /** Close the backend. + This allows the caller to catch exceptions. + */ + virtual void close() = 0; + /** Fetch a single object. If the object is not found or an error is encountered, the result will indicate the condition. @@ -87,6 +92,9 @@ public: /** Remove contents on disk upon destruction. */ virtual void setDeletePath() = 0; + + /** Perform consistency checks on database .*/ + virtual void verify() = 0; }; } diff --git a/src/ripple/nodestore/Database.h b/src/ripple/nodestore/Database.h index 51a3b2ff9..0e60c1be6 100644 --- a/src/ripple/nodestore/Database.h +++ b/src/ripple/nodestore/Database.h @@ -54,6 +54,11 @@ public: or paths used by the underlying backend. */ virtual std::string getName () const = 0; + + /** Close the database. + This allows the caller to catch exceptions. + */ + virtual void close() = 0; /** Fetch an object. If the object is known to be not in the database, isn't found in the diff --git a/src/ripple/nodestore/Manager.h b/src/ripple/nodestore/Manager.h index d7d4976c8..0367799f7 100644 --- a/src/ripple/nodestore/Manager.h +++ b/src/ripple/nodestore/Manager.h @@ -22,6 +22,8 @@ #include #include +#include +#include namespace ripple { namespace NodeStore { @@ -98,6 +100,13 @@ public: beast::Journal journal) = 0; }; +//------------------------------------------------------------------------------ + +/** Create a Backend. */ +std::unique_ptr +make_Backend (Section const& config, + Scheduler& scheduler, beast::Journal journal); + } } diff --git a/src/ripple/nodestore/backend/HyperDBFactory.cpp b/src/ripple/nodestore/backend/HyperDBFactory.cpp index c13edb633..32d9d574c 100644 --- a/src/ripple/nodestore/backend/HyperDBFactory.cpp +++ b/src/ripple/nodestore/backend/HyperDBFactory.cpp @@ -101,12 +101,7 @@ public: ~HyperDBBackend () { - if (m_deletePath) - { - m_db.reset(); - boost::filesystem::path dir = m_name; - boost::filesystem::remove_all (dir); - } + close(); } std::string @@ -115,6 +110,20 @@ public: return m_name; } + void + close() override + { + if (m_db) + { + m_db.reset(); + if (m_deletePath) + { + boost::filesystem::path dir = m_name; + boost::filesystem::remove_all (dir); + } + } + } + //-------------------------------------------------------------------------- Status @@ -251,6 +260,11 @@ public: { storeBatch (batch); } + + void + verify() override + { + } }; //------------------------------------------------------------------------------ diff --git a/src/ripple/nodestore/backend/LevelDBFactory.cpp b/src/ripple/nodestore/backend/LevelDBFactory.cpp index 89ce6d6ec..435d84af2 100644 --- a/src/ripple/nodestore/backend/LevelDBFactory.cpp +++ b/src/ripple/nodestore/backend/LevelDBFactory.cpp @@ -108,12 +108,7 @@ public: ~LevelDBBackend() { - if (m_deletePath) - { - m_db.reset(); - boost::filesystem::path dir = m_name; - boost::filesystem::remove_all (dir); - } + close(); } std::string @@ -122,6 +117,20 @@ public: return m_name; } + void + close() override + { + if (m_db) + { + m_db.reset(); + if (m_deletePath) + { + boost::filesystem::path dir = m_name; + boost::filesystem::remove_all (dir); + } + } + } + //-------------------------------------------------------------------------- Status @@ -258,6 +267,11 @@ public: { storeBatch (batch); } + + void + verify() override + { + } }; //------------------------------------------------------------------------------ diff --git a/src/ripple/nodestore/backend/MemoryFactory.cpp b/src/ripple/nodestore/backend/MemoryFactory.cpp index 96921ca9d..14525d7a6 100644 --- a/src/ripple/nodestore/backend/MemoryFactory.cpp +++ b/src/ripple/nodestore/backend/MemoryFactory.cpp @@ -20,37 +20,93 @@ #include #include #include +#include #include // #include +#include namespace ripple { namespace NodeStore { +struct MemoryDB +{ + std::mutex mutex; + bool open = false; + std::map table; +}; + +class MemoryFactory : public Factory +{ +private: + std::mutex mutex_; + std::map map_; + +public: + MemoryFactory(); + ~MemoryFactory(); + + std::string + getName() const; + + std::unique_ptr + createInstance ( + size_t keyBytes, + Parameters const& keyValues, + Scheduler& scheduler, + beast::Journal journal); + + MemoryDB& + open (std::string const& path) + { + std::lock_guard _(mutex_); + auto const result = map_.emplace (std::piecewise_construct, + std::make_tuple(path), std::make_tuple()); + MemoryDB& db = result.first->second; + if (db.open) + throw std::runtime_error("already open"); + return db; + } +}; + +static MemoryFactory memoryFactory; + +//------------------------------------------------------------------------------ + class MemoryBackend : public Backend { -public: - typedef std::map Map; - beast::Journal m_journal; - size_t const m_keyBytes; - Map m_map; - Scheduler& m_scheduler; +private: + using Map = std::map ; + std::string name_; + beast::Journal journal_; + MemoryDB* db_; + +public: MemoryBackend (size_t keyBytes, Parameters const& keyValues, Scheduler& scheduler, beast::Journal journal) - : m_journal (journal) - , m_keyBytes (keyBytes) - , m_scheduler (scheduler) + : name_ (keyValues ["path"].toStdString ()) + , journal_ (journal) { + if (name_.empty()) + throw std::runtime_error ("Missing path in Memory backend"); + db_ = &memoryFactory.open(name_); } ~MemoryBackend () { + close(); } std::string getName () { - return "memory"; + return name_; + } + + void + close() override + { + db_ = nullptr; } //-------------------------------------------------------------------------- @@ -60,29 +116,23 @@ public: { uint256 const hash (uint256::fromVoid (key)); - Map::iterator iter = m_map.find (hash); + std::lock_guard _(db_->mutex); - if (iter != m_map.end ()) + Map::iterator iter = db_->table.find (hash); + if (iter == db_->table.end()) { - *pObject = iter->second; + pObject->reset(); + return notFound; } - else - { - pObject->reset (); - } - + *pObject = iter->second; return ok; } void store (NodeObject::ref object) { - Map::iterator iter = m_map.find (object->getHash ()); - - if (iter == m_map.end ()) - { - m_map.insert (std::make_pair (object->getHash (), object)); - } + std::lock_guard _(db_->mutex); + db_->table.emplace (object->getHash(), object); } void @@ -95,56 +145,55 @@ public: void for_each (std::function f) { - for (auto const& e : m_map) + for (auto const& e : db_->table) f (e.second); } int - getWriteLoad () + getWriteLoad() { return 0; } void - setDeletePath() override {} + setDeletePath() override + { + } + + void + verify() override + { + } }; //------------------------------------------------------------------------------ -class MemoryFactory : public Factory +MemoryFactory::MemoryFactory() { -public: - MemoryFactory() - { - Manager::instance().insert(*this); - } + Manager::instance().insert(*this); +} - ~MemoryFactory() - { - Manager::instance().erase(*this); - } +MemoryFactory::~MemoryFactory() +{ + Manager::instance().erase(*this); +} - std::string - getName () const - { - return "Memory"; - } +std::string +MemoryFactory::getName() const +{ + return "Memory"; +} - std::unique_ptr - createInstance ( - size_t keyBytes, - Parameters const& keyValues, - Scheduler& scheduler, - beast::Journal journal) - { - return std::make_unique ( - keyBytes, keyValues, scheduler, journal); - } -}; - -//------------------------------------------------------------------------------ - -static MemoryFactory memoryFactory; +std::unique_ptr +MemoryFactory::createInstance ( + size_t keyBytes, + Parameters const& keyValues, + Scheduler& scheduler, + beast::Journal journal) +{ + return std::make_unique ( + keyBytes, keyValues, scheduler, journal); +} } } diff --git a/src/ripple/nodestore/backend/NullFactory.cpp b/src/ripple/nodestore/backend/NullFactory.cpp index c607503da..b2571afcd 100644 --- a/src/ripple/nodestore/backend/NullFactory.cpp +++ b/src/ripple/nodestore/backend/NullFactory.cpp @@ -42,6 +42,11 @@ public: return std::string (); } + void + close() override + { + } + Status fetch (void const*, NodeObject::Ptr*) { @@ -70,7 +75,14 @@ public: } void - setDeletePath() override {} + setDeletePath() override + { + } + + void + verify() override + { + } private: }; diff --git a/src/ripple/nodestore/backend/RocksDBFactory.cpp b/src/ripple/nodestore/backend/RocksDBFactory.cpp index e72a30edc..4ded1a457 100644 --- a/src/ripple/nodestore/backend/RocksDBFactory.cpp +++ b/src/ripple/nodestore/backend/RocksDBFactory.cpp @@ -174,12 +174,7 @@ public: ~RocksDBBackend () { - if (m_deletePath) - { - m_db.reset(); - boost::filesystem::path dir = m_name; - boost::filesystem::remove_all (dir); - } + close(); } std::string @@ -188,6 +183,20 @@ public: return m_name; } + void + close() override + { + if (m_db) + { + m_db.reset(); + if (m_deletePath) + { + boost::filesystem::path dir = m_name; + boost::filesystem::remove_all (dir); + } + } + } + //-------------------------------------------------------------------------- Status @@ -330,6 +339,11 @@ public: { storeBatch (batch); } + + void + verify() override + { + } }; //------------------------------------------------------------------------------ diff --git a/src/ripple/nodestore/impl/DatabaseImp.h b/src/ripple/nodestore/impl/DatabaseImp.h index 34a22237a..7775f9f12 100644 --- a/src/ripple/nodestore/impl/DatabaseImp.h +++ b/src/ripple/nodestore/impl/DatabaseImp.h @@ -109,6 +109,21 @@ public: return m_backend->getName (); } + void + close() override + { + if (m_backend) + { + m_backend->close(); + m_backend = nullptr; + } + if (m_fastBackend) + { + m_fastBackend->close(); + m_fastBackend = nullptr; + } + } + //------------------------------------------------------------------------------ bool asyncFetch (uint256 const& hash, NodeObject::pointer& object) diff --git a/src/ripple/nodestore/impl/DatabaseRotatingImp.h b/src/ripple/nodestore/impl/DatabaseRotatingImp.h index 6f3977378..493b84a54 100644 --- a/src/ripple/nodestore/impl/DatabaseRotatingImp.h +++ b/src/ripple/nodestore/impl/DatabaseRotatingImp.h @@ -85,6 +85,13 @@ public: return getWritableBackend()->getName(); } + void + close() override + { + // VFALCO TODO How do we close everything? + assert(false); + } + std::int32_t getWriteLoad() const override { return getWritableBackend()->getWriteLoad(); diff --git a/src/ripple/nodestore/impl/ManagerImp.cpp b/src/ripple/nodestore/impl/ManagerImp.cpp index f45da06ea..a985f7a34 100644 --- a/src/ripple/nodestore/impl/ManagerImp.cpp +++ b/src/ripple/nodestore/impl/ManagerImp.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include // #include @@ -160,5 +161,18 @@ Manager::instance() return ManagerImp::instance(); } +//------------------------------------------------------------------------------ + +std::unique_ptr +make_Backend (Section const& config, + Scheduler& scheduler, beast::Journal journal) +{ + beast::StringPairArray v; + for (auto const& _ : config) + v.set (_.first, _.second); + return Manager::instance().make_Backend ( + v, scheduler, journal); +} + } } diff --git a/src/ripple/nodestore/impl/NodeObject.cpp b/src/ripple/nodestore/impl/NodeObject.cpp index 9ee8dc34f..d760c4a55 100644 --- a/src/ripple/nodestore/impl/NodeObject.cpp +++ b/src/ripple/nodestore/impl/NodeObject.cpp @@ -38,7 +38,8 @@ NodeObject::NodeObject ( mData = std::move (data); } -NodeObject::Ptr NodeObject::createObject ( +NodeObject::Ptr +NodeObject::createObject ( NodeObjectType type, LedgerIndex ledgerIndex, Blob&& data, diff --git a/src/ripple/nodestore/tests/Backend.test.cpp b/src/ripple/nodestore/tests/Backend.test.cpp index 692599104..6209712ce 100644 --- a/src/ripple/nodestore/tests/Backend.test.cpp +++ b/src/ripple/nodestore/tests/Backend.test.cpp @@ -100,7 +100,7 @@ public: testBackend ("hyperleveldb", seedValue); #endif -#if RIPPLE_ROCKSDB_AVAILABLE + #if RIPPLE_ROCKSDB_AVAILABLE testBackend ("rocksdb", seedValue); #endif diff --git a/src/ripple/nodestore/tests/Timing.test.cpp b/src/ripple/nodestore/tests/Timing.test.cpp index 06820139a..179187deb 100644 --- a/src/ripple/nodestore/tests/Timing.test.cpp +++ b/src/ripple/nodestore/tests/Timing.test.cpp @@ -21,390 +21,735 @@ #include #include #include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include #include #include +#include +#include +#include +#include +#include // +#include + +#ifndef NODESTORE_TIMING_DO_VERIFY +#define NODESTORE_TIMING_DO_VERIFY 0 +#endif namespace ripple { namespace NodeStore { -class NodeStoreTiming_test : public TestBase +// Fill memory with random bits +template +static +void +rngcpy (void* buffer, std::size_t bytes, Generator& g) { -public: - // Simple and fast RNG based on: - // http://xorshift.di.unimi.it/xorshift128plus.c - // does not accept seed==0 - class XORShiftEngine{ - public: - using result_type = std::uint64_t; - - static const result_type default_seed = 1977u; - - explicit XORShiftEngine(result_type val = default_seed) { seed(val); } - - void seed(result_type const seed) - { - if (seed==0) - throw std::range_error("zero seed supplied"); - s[0] = murmurhash3(seed); - s[1] = murmurhash3(s[0]); - } - - result_type operator()() - { - result_type s1 = s[0]; - const result_type s0 = s[1]; - s[0] = s0; - s1 ^= s1 << 23; - return (s[1] = (s1 ^ s0 ^ (s1 >> 17) ^ (s0 >> 26))) + s0; - } - - static BEAST_CONSTEXPR result_type min() - { - return std::numeric_limits::min(); - } - - static BEAST_CONSTEXPR result_type max() - { - return std::numeric_limits::max(); - } - - private: - result_type s[2]; - - static result_type murmurhash3(result_type x) - { - x ^= x >> 33; - x *= 0xff51afd7ed558ccdULL; - x ^= x >> 33; - x *= 0xc4ceb9fe1a85ec53ULL; - return x ^= x >> 33; - } - }; - - class NodeFactory + using result_type = typename Generator::result_type; + while (bytes >= sizeof(result_type)) { - enum - { - minLedger = 1, - maxLedger = 10000000, - minValueLength = 128, // Must be a multiple of 8 - maxValueLength = 256 // Will be multiplied by 8 - }; - - public : NodeFactory(std::int64_t seed, - std::int64_t numObjects, - std::int64_t minKey, - std::int64_t maxKey) - : seed_(seed), - numObjects_(numObjects), - count_(0), - rng_(seed), - key_(minKey+1, maxKey+1), - value_(minValueLength, maxValueLength), - type_(hotLEDGER, hotTRANSACTION_NODE), - ledger_(minLedger, maxLedger) - { - } - - NodeObject::Ptr next() - { - // Stop when done - if (count_ == numObjects_) - return nullptr; - count_++; - - // Seed from range between minKey and maxKey to ensure repeatability - r_.seed(key_(rng_)); - - uint256 hash; - std::generate_n(reinterpret_cast(hash.begin()), - hash.size() / sizeof(std::uint64_t), - std::bind(filler_, r_)); - - Blob data(value_(r_)*8); - std::generate_n(reinterpret_cast(data.data()), - data.size() / sizeof(std::uint64_t), - std::bind(filler_, r_)); - - NodeObjectType nodeType(static_cast(type_(r_))); - return NodeObject::createObject(nodeType, ledger_(r_), - std::move(data), hash); - } - - bool fillBatch(Batch& batch,std::int64_t batchSize) - { - batch.clear(); - for (std::uint64_t i = 0; i < batchSize; i++) - { - auto node = next(); - if (!node) - return false; - batch.emplace_back(node); - } - return true; - } - - void reset() - { - count_ = 0; - rng_.seed(seed_); - } - - private: - std::int64_t seed_; - std::int64_t numObjects_; - std::int64_t count_; - std::mt19937_64 rng_; - XORShiftEngine r_; - std::uniform_int_distribution key_; - std::uniform_int_distribution value_; - std::uniform_int_distribution type_; - std::uniform_int_distribution ledger_; - std::uniform_int_distribution filler_; - }; // end NodeFactory - - // Checks NodeFactory - void testNodeFactory(std::int64_t const seedValue) - { - testcase("repeatableObject"); - - NodeFactory factory(seedValue, 10000, 0, 99); - - std::set out; - - while (auto node = factory.next()) - { - auto it = out.find(node); - if (it == out.end()) - { - out.insert(node); - } - else - { - expect(it->get()->isCloneOf(node), "Should be clones"); - } - } - expect(out.size() == 100, "Too many objects created"); + auto const v = g(); + memcpy(buffer, &v, sizeof(v)); + buffer = reinterpret_cast(buffer) + sizeof(v); + bytes -= sizeof(v); } - class Stopwatch + if (bytes > 0) { - public: - Stopwatch() {} - - void start() { m_startTime = beast::Time::getHighResolutionTicks(); } - - double getElapsed() - { - std::int64_t const now = beast::Time::getHighResolutionTicks(); - - return beast::Time::highResolutionTicksToSeconds(now - m_startTime); - } - - private: - std::int64_t m_startTime; - }; - - //-------------------------------------------------------------------------- + auto const v = g(); + memcpy(buffer, &v, bytes); + } +} +// Instance of node factory produces a deterministic sequence +// of random NodeObjects within the given +class Sequence +{ +private: enum { - batchSize = 128 + minLedger = 1, + maxLedger = 1000000, + minSize = 250, + maxSize = 1250 }; - using check_func = std::function; - using backend_ptr = std::unique_ptr; - using result_type = std::vector>; + beast::xor_shift_engine gen_; + std::uint8_t prefix_; + std::uniform_int_distribution d_seq_; + std::uniform_int_distribution d_type_; + std::uniform_int_distribution d_size_; - static bool checkNotFound(Status const status) +public: + explicit + Sequence(std::uint8_t prefix) + : prefix_ (prefix) + , d_seq_ (minLedger, maxLedger) + , d_type_ (hotLEDGER, hotTRANSACTION_NODE) + , d_size_ (minSize, maxSize) { - return status == notFound; + } + + // Returns the n-th key + uint256 + key (std::size_t n) + { + gen_.seed(n+1); + uint256 result; + rngcpy (&*result.begin(), result.size(), gen_); + return result; + } + + // Returns the n-th complete NodeObject + NodeObject::Ptr + obj (std::size_t n) + { + gen_.seed(n+1); + uint256 key; + auto const data = + static_cast(&*key.begin()); + *data = prefix_; + rngcpy (data + 1, key.size() - 1, gen_); + Blob value(d_size_(gen_)); + rngcpy (&value[0], value.size(), gen_); + return NodeObject::createObject ( + static_cast(d_type_(gen_)), + d_seq_(gen_), std::move(value), key); + } + + // returns a batch of NodeObjects starting at n + void + batch (std::size_t n, Batch& b, std::size_t size) + { + b.clear(); + b.reserve (size); + while(size--) + b.emplace_back(obj(n++)); + } +}; + +//---------------------------------------------------------------------------------- + +class Timing_test : public beast::unit_test::suite +{ +public: + enum + { + // percent of fetches for missing nodes + missingNodePercent = 20 }; - static bool checkOk(Status const status) { return status == ok; }; + std::size_t const default_repeat = 1; +#ifndef NDEBUG + std::size_t const default_items = 10000; +#else + std::size_t const default_items = 100000; // release +#endif - static bool checkOkOrNotFound(Status const status) + using clock_type = std::chrono::steady_clock; + using duration_type = std::chrono::milliseconds; + + struct Params { - return (status == ok) || (status == notFound); + std::size_t items; + std::size_t threads; }; - void testFetch(backend_ptr& backend, NodeFactory& factory, - check_func f) + static + std::string + to_string (Section const& config) { - factory.reset(); - while (auto expected = factory.next()) - { - NodeObject::Ptr got; - - Status const status = - backend->fetch(expected->getHash().cbegin(), &got); - expect(f(status), - "Wrong status for: " + to_string(expected->getHash())); - if (status == ok) - { - expect(got != nullptr, "Should not be null"); - expect(got->isCloneOf(expected), "Should be clones"); - } - } + std::string s; + for (auto iter = config.begin(); iter != config.end(); ++iter) + s += (iter != config.begin() ? "," : "") + + iter->first + "=" + iter->second; + return s; } - static void testInsert(backend_ptr& backend, NodeFactory& factory) + static + std::string + to_string (duration_type const& d) { - factory.reset(); - while (auto node = factory.next()) backend->store(node); + std::stringstream ss; + ss << std::fixed << std::setprecision(3) << + (d.count() / 1000.) << "s"; + return ss.str(); } - static void testBatchInsert(backend_ptr& backend, NodeFactory& factory) + static + Section + parse (std::string s) { - factory.reset(); - Batch batch; - while (factory.fillBatch(batch, batchSize)) backend->storeBatch(batch); - } - - result_type benchmarkBackend(beast::StringPairArray const& params, - std::int64_t const seedValue, - std::int64_t const numObjects) - { - Stopwatch t; - result_type results; - - DummyScheduler scheduler; - beast::Journal j; - - auto backend = Manager::instance().make_Backend(params, scheduler, j); - - NodeFactory insertFactory(seedValue, numObjects, 0, numObjects); - NodeFactory batchFactory(seedValue, numObjects, numObjects * 10, - numObjects * 11); - - // Twice the range of insert - NodeFactory mixedFactory(seedValue, numObjects, numObjects, - numObjects * 2); - // Same as batch, different order - NodeFactory randomFactory(seedValue + 1, numObjects, numObjects * 10, - numObjects * 11); - // Don't exist - NodeFactory missingFactory(seedValue, numObjects, numObjects * 3, - numObjects * 4); - - t.start(); - testInsert(backend, insertFactory); - results.emplace_back("Inserts", t.getElapsed()); - - t.start(); - testBatchInsert(backend, batchFactory); - results.emplace_back("Batch Insert", t.getElapsed()); - - t.start(); - testFetch(backend, mixedFactory, checkOkOrNotFound); - results.emplace_back("Fetch 50/50", t.getElapsed()); - - t.start(); - testFetch(backend, insertFactory, checkOk); - results.emplace_back("Ordered Fetch", t.getElapsed()); - - t.start(); - testFetch(backend, randomFactory, checkOkOrNotFound); - results.emplace_back("Fetch Random", t.getElapsed()); - - t.start(); - testFetch(backend, missingFactory, checkNotFound); - results.emplace_back("Fetch Missing", t.getElapsed()); - - return results; + Section section; + std::vector v; + boost::split (v, s, + boost::algorithm::is_any_of (",")); + section.append(v); + return section; } //-------------------------------------------------------------------------- - void run () + // Workaround for GCC's parameter pack expansion in lambdas + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47226 + template + class parallel_for_lambda { - int const seedValue = 50; + private: + std::size_t const n_; + std::atomic& c_; + + public: + parallel_for_lambda (std::size_t n, + std::atomic& c) + : n_ (n) + , c_ (c) + { + } + + template + void + operator()(Args&&... args) + { + Body body(args...); + for(;;) + { + auto const i = c_++; + if (i >= n_) + break; + body (i); + } + } + }; + + /* Execute parallel-for loop. + + Constructs `number_of_threads` instances of `Body` + with `args...` parameters and runs them on individual threads + with unique loop indexes in the range [0, n). + */ + template + void + parallel_for (std::size_t const n, + std::size_t number_of_threads, Args const&... args) + { + std::atomic c(0); + std::vector t; + t.reserve(number_of_threads); + for (std::size_t id = 0; id < number_of_threads; ++id) + t.emplace_back(*this, + parallel_for_lambda(n, c), + args...); + for (auto& _ : t) + _.join(); + } + + template + void + parallel_for_id (std::size_t const n, + std::size_t number_of_threads, Args const&... args) + { + std::atomic c(0); + std::vector t; + t.reserve(number_of_threads); + for (std::size_t id = 0; id < number_of_threads; ++id) + t.emplace_back(*this, + parallel_for_lambda(n, c), + id, args...); + for (auto& _ : t) + _.join(); + } + + //-------------------------------------------------------------------------- + + // Insert only + void + do_insert (Section const& config, Params const& params) + { + beast::Journal journal; + DummyScheduler scheduler; + auto backend = make_Backend (config, scheduler, journal); + expect (backend != nullptr); + + class Body + { + private: + suite& suite_; + Backend& backend_; + Sequence seq_; + + public: + explicit + Body (suite& s, Backend& backend) + : suite_ (s) + , backend_ (backend) + , seq_(1) + { + } + + void + operator()(std::size_t i) + { + try + { + backend_.store(seq_.obj(i)); + } + catch(std::exception const& e) + { + suite_.fail(e.what()); + } + } + }; + + try + { + parallel_for(params.items, + params.threads, std::ref(*this), std::ref(*backend)); + } + catch(...) + { + #if NODESTORE_TIMING_DO_VERIFY + backend->verify(); + #endif + throw; + } + backend->close(); + } + + // Fetch existing keys + void + do_fetch (Section const& config, Params const& params) + { + beast::Journal journal; + DummyScheduler scheduler; + auto backend = make_Backend (config, scheduler, journal); + expect (backend != nullptr); + + class Body + { + private: + suite& suite_; + Backend& backend_; + Sequence seq1_; + beast::xor_shift_engine gen_; + std::uniform_int_distribution dist_; + + public: + Body (std::size_t id, suite& s, + Params const& params, Backend& backend) + : suite_(s) + , backend_ (backend) + , seq1_ (1) + , gen_ (id + 1) + , dist_ (0, params.items - 1) + { + } + + void + operator()(std::size_t i) + { + try + { + NodeObject::Ptr obj; + NodeObject::Ptr result; + obj = seq1_.obj(dist_(gen_)); + backend_.fetch(obj->getHash().data(), &result); + suite_.expect (result && result->isCloneOf(obj)); + } + catch(std::exception const& e) + { + suite_.fail(e.what()); + } + } + }; + try + { + parallel_for_id(params.items, params.threads, + std::ref(*this), std::ref(params), std::ref(*backend)); + } + catch(...) + { + #if NODESTORE_TIMING_DO_VERIFY + backend->verify(); + #endif + throw; + } + backend->close(); + } + + // Perform lookups of non-existent keys + void + do_missing (Section const& config, Params const& params) + { + beast::Journal journal; + DummyScheduler scheduler; + auto backend = make_Backend (config, scheduler, journal); + expect (backend != nullptr); + + class Body + { + private: + suite& suite_; + //Params const& params_; + Backend& backend_; + Sequence seq2_; + beast::xor_shift_engine gen_; + std::uniform_int_distribution dist_; + + public: + Body (std::size_t id, suite& s, + Params const& params, Backend& backend) + : suite_ (s) + //, params_ (params) + , backend_ (backend) + , seq2_ (2) + , gen_ (id + 1) + , dist_ (0, params.items - 1) + { + } + + void + operator()(std::size_t i) + { + try + { + auto const key = seq2_.key(i); + NodeObject::Ptr result; + backend_.fetch(key.data(), &result); + suite_.expect (! result); + } + catch(std::exception const& e) + { + suite_.fail(e.what()); + } + } + }; + + try + { + parallel_for_id(params.items, params.threads, + std::ref(*this), std::ref(params), std::ref(*backend)); + } + catch(...) + { + #if NODESTORE_TIMING_DO_VERIFY + backend->verify(); + #endif + throw; + } + backend->close(); + } + + // Fetch with present and missing keys + void + do_mixed (Section const& config, Params const& params) + { + beast::Journal journal; + DummyScheduler scheduler; + auto backend = make_Backend (config, scheduler, journal); + expect (backend != nullptr); + + class Body + { + private: + suite& suite_; + //Params const& params_; + Backend& backend_; + Sequence seq1_; + Sequence seq2_; + beast::xor_shift_engine gen_; + std::uniform_int_distribution rand_; + std::uniform_int_distribution dist_; + + public: + Body (std::size_t id, suite& s, + Params const& params, Backend& backend) + : suite_ (s) + //, params_ (params) + , backend_ (backend) + , seq1_ (1) + , seq2_ (2) + , gen_ (id + 1) + , rand_ (0, 99) + , dist_ (0, params.items - 1) + { + } + + void + operator()(std::size_t i) + { + try + { + if (rand_(gen_) < missingNodePercent) + { + auto const key = seq2_.key(dist_(gen_)); + NodeObject::Ptr result; + backend_.fetch(key.data(), &result); + suite_.expect (! result); + } + else + { + NodeObject::Ptr obj; + NodeObject::Ptr result; + obj = seq1_.obj(dist_(gen_)); + backend_.fetch(obj->getHash().data(), &result); + suite_.expect (result && result->isCloneOf(obj)); + } + } + catch(std::exception const& e) + { + suite_.fail(e.what()); + } + } + }; - testNodeFactory(seedValue); - - // Expects a semi-colon delimited list of backend configurations. - // Each configuration is a comma delimited list of key-value pairs. - // Each pair is separated by a '='. - // 'type' defaults to 'rocksdb' - // 'num_objects' defaults to '100000' - // 'num_runs' defaults to '3' - // defaultArguments serves as an example. - - std::string defaultArguments = - "type=rocksdb,open_files=2000,filter_bits=12,cache_mb=256" - "file_size_mb=8,file_size_mult=2,num_objects=100000,num_runs=3;" - "type=hyperleveldb,num_objects=100000,num_runs=3"; - - auto args = arg(); - - if (args.empty()) args = defaultArguments; - - std::vector configs; - boost::split (configs, args, boost::algorithm::is_any_of (";")); - - std::map> results; - - for (auto& config : configs) + try { - auto params = parseDelimitedKeyValueString(config, ','); + parallel_for_id(params.items, params.threads, + std::ref(*this), std::ref(params), std::ref(*backend)); + } + catch(...) + { + #if NODESTORE_TIMING_DO_VERIFY + backend->verify(); + #endif + throw; + } + backend->close(); + } - // Defaults - std::int64_t numRuns = 3; - std::int64_t numObjects = 100000; + // Simulate a rippled workload: + // Each thread randomly: + // inserts a new key + // fetches an old key + // fetches recent, possibly non existent data + void + do_work (Section const& config, Params const& params) + { + beast::Journal journal; + DummyScheduler scheduler; + auto backend = make_Backend (config, scheduler, journal); + backend->setDeletePath(); + expect (backend != nullptr); - if (!params["num_objects"].isEmpty()) - numObjects = params["num_objects"].getIntValue(); + class Body + { + private: + suite& suite_; + Params const& params_; + Backend& backend_; + Sequence seq1_; + beast::xor_shift_engine gen_; + std::uniform_int_distribution rand_; + std::uniform_int_distribution recent_; + std::uniform_int_distribution older_; - if (!params["num_runs"].isEmpty()) - numRuns = params["num_runs"].getIntValue(); - - if (params["type"].isEmpty()) - params.set("type", "rocksdb"); - - for (std::int64_t i = 0; i < numRuns; i++) + public: + Body (std::size_t id, suite& s, + Params const& params, Backend& backend) + : suite_ (s) + , params_ (params) + , backend_ (backend) + , seq1_ (1) + , gen_ (id + 1) + , rand_ (0, 99) + , recent_ (params.items, params.items * 2 - 1) + , older_ (0, params.items - 1) { - beast::UnitTestUtilities::TempDirectory path("node_db"); - params.set("path", path.getFullPathName()); - results[config].emplace_back( - benchmarkBackend(params, seedValue + i, numObjects)); } + + void + operator()(std::size_t i) + { + try + { + if (rand_(gen_) < 200) + { + // historical lookup + NodeObject::Ptr obj; + NodeObject::Ptr result; + auto const j = older_(gen_); + obj = seq1_.obj(j); + NodeObject::Ptr result1; + backend_.fetch(obj->getHash().data(), &result); + suite_.expect (result != nullptr, + "object " + std::to_string(j) + " missing"); + suite_.expect (result->isCloneOf(obj), + "object " + std::to_string(j) + " not a clone"); + } + + char p[2]; + p[0] = rand_(gen_) < 50 ? 0 : 1; + p[1] = 1 - p[0]; + for (int q = 0; q < 2; ++q) + { + switch (p[q]) + { + case 0: + { + // fetch recent + NodeObject::Ptr obj; + NodeObject::Ptr result; + auto const j = recent_(gen_); + obj = seq1_.obj(j); + backend_.fetch(obj->getHash().data(), &result); + suite_.expect(! result || result->isCloneOf(obj)); + break; + } + + case 1: + { + // insert new + auto const j = i + params_.items; + backend_.store(seq1_.obj(j)); + break; + } + } + } + } + catch(std::exception const& e) + { + suite_.fail(e.what()); + } + } + }; + + try + { + parallel_for_id(params.items, params.threads, + std::ref(*this), std::ref(params), std::ref(*backend)); + } + catch(...) + { + #if NODESTORE_TIMING_DO_VERIFY + backend->verify(); + #endif + throw; + } + backend->close(); + } + + //-------------------------------------------------------------------------- + + using test_func = void (Timing_test::*)(Section const&, Params const&); + using test_list = std::vector >; + + duration_type + do_test (test_func f, + Section const& config, Params const& params) + { + auto const start = clock_type::now(); + (this->*f)(config, params); + return std::chrono::duration_cast ( + clock_type::now() - start); + } + + void + do_tests (std::size_t threads, test_list const& tests, + std::vector const& config_strings) + { + using std::setw; + int w = 8; + for (auto const& test : tests) + if (w < test.first.size()) + w = test.first.size(); + log << + "\n" << + threads << " Thread" << (threads > 1 ? "s" : "") << ", " << + default_items << " Objects"; + { + std::stringstream ss; + ss << std::left << setw(10) << "Backend" << std::right; + for (auto const& test : tests) + ss << " " << setw(w) << test.first; + log << ss.str(); } - std::stringstream header; - std::stringstream stats; - std::stringstream legend; - - auto firstRun = results.begin()->second.begin(); - header << std::setw(7) << "Config" << std::setw(4) << "Run"; - for (auto const& title : *firstRun) - header << std::setw(14) << title.first; - - stats << std::setprecision(2) << std::fixed; - - std::int64_t resultCount = 0; - for (auto const& result : results) + for (auto const& config_string : config_strings) { - std::int64_t runCount = 0; - for (auto const& run : result.second) + Params params; + params.items = default_items; + params.threads = threads; + for (auto i = default_repeat; i--;) { - stats << std::setw(7) << resultCount << std::setw(4) - << runCount; - for (auto const& item : run) - stats << std::setw(14) << item.second; - runCount++; - stats << std::endl; - + Section config = parse(config_string); + config.set ("path", + beast::UnitTestUtilities::TempDirectory( + "test_db").getFullPathName().toStdString()); + std::stringstream ss; + ss << std::left << setw(10) << + get(config, "type", std::string()) << std::right; + for (auto const& test : tests) + ss << " " << setw(w) << to_string( + do_test (test.second, config, params)); + ss << " " << to_string(config); + log << ss.str(); } - legend << std::setw(2) << resultCount << ": " << result.first - << std::endl; - resultCount++; } - log << header.str() << std::endl << stats.str() << std::endl - << "Configs:" << std::endl << legend.str(); + } + + void + run() override + { + testcase ("Timing", suite::abort_on_fail); + + /* Parameters: + + repeat Number of times to repeat each test + items Number of objects to create in the database + + */ + std::string default_args = + #ifdef _MSC_VER + "type=leveldb" + #endif + //"type=nudb" + #if RIPPLE_ROCKSDB_AVAILABLE + ";type=rocksdb,open_files=2000,filter_bits=12,cache_mb=256," + "file_size_mb=8,file_size_mult=2" + #endif + #if 0 + ";type=memory|path=NodeStore" + #endif + ; + + test_list const tests = + { + { "Insert", &Timing_test::do_insert } + ,{ "Fetch", &Timing_test::do_fetch } + ,{ "Missing", &Timing_test::do_missing } + ,{ "Mixed", &Timing_test::do_mixed } + ,{ "Work", &Timing_test::do_work } + }; + + auto args = arg().empty() ? default_args : arg(); + std::vector config_strings; + boost::split (config_strings, args, + boost::algorithm::is_any_of (";")); + for (auto iter = config_strings.begin(); + iter != config_strings.end();) + if (iter->empty()) + iter = config_strings.erase (iter); + else + ++iter; + + do_tests ( 1, tests, config_strings); + do_tests ( 4, tests, config_strings); + do_tests ( 8, tests, config_strings); + //do_tests (16, tests, config_strings); } }; -BEAST_DEFINE_TESTSUITE_MANUAL(NodeStoreTiming,bench,ripple); -} // namespace Nodestore +BEAST_DEFINE_TESTSUITE_MANUAL(Timing,NodeStore,ripple); + } +} + diff --git a/src/ripple/shamap/tests/FetchPack.test.cpp b/src/ripple/shamap/tests/FetchPack.test.cpp index 527cb36f4..579c78ab7 100644 --- a/src/ripple/shamap/tests/FetchPack.test.cpp +++ b/src/ripple/shamap/tests/FetchPack.test.cpp @@ -121,7 +121,7 @@ public: TreeNodeCache treeNodeCache ("test.tree_node_cache", 65536, 60, clock, j); NodeStore::DummyScheduler scheduler; auto db = NodeStore::Manager::instance().make_Database ( - "test", scheduler, j, 0, parseDelimitedKeyValueString("type=memory")); + "test", scheduler, j, 0, parseDelimitedKeyValueString("type=memory|path=FetchPack")); std::shared_ptr t1 (std::make_shared
( smtFREE, fullBelowCache, treeNodeCache, *db, Handler(), beast::Journal())); diff --git a/src/ripple/shamap/tests/SHAMap.test.cpp b/src/ripple/shamap/tests/SHAMap.test.cpp index fab01be99..75a6eccd5 100644 --- a/src/ripple/shamap/tests/SHAMap.test.cpp +++ b/src/ripple/shamap/tests/SHAMap.test.cpp @@ -62,7 +62,7 @@ public: TreeNodeCache treeNodeCache ("test.tree_node_cache", 65536, 60, clock, j); NodeStore::DummyScheduler scheduler; auto db = NodeStore::Manager::instance().make_Database ( - "test", scheduler, j, 0, parseDelimitedKeyValueString("type=memory")); + "test", scheduler, j, 0, parseDelimitedKeyValueString("type=memory|Path=SHAMap_test")); // h3 and h4 differ only in the leaf, same terminal node (level 19) uint256 h1, h2, h3, h4, h5; diff --git a/src/ripple/shamap/tests/SHAMapSync.test.cpp b/src/ripple/shamap/tests/SHAMapSync.test.cpp index 03e05792e..81c3df4be 100644 --- a/src/ripple/shamap/tests/SHAMapSync.test.cpp +++ b/src/ripple/shamap/tests/SHAMapSync.test.cpp @@ -114,7 +114,7 @@ public: TreeNodeCache treeNodeCache ("test.tree_node_cache", 65536, 60, clock, j); NodeStore::DummyScheduler scheduler; auto db = NodeStore::Manager::instance().make_Database ( - "test", scheduler, j, 1, parseDelimitedKeyValueString("type=memory")); + "test", scheduler, j, 1, parseDelimitedKeyValueString("type=memory|path=SHAMapSync_test")); SHAMap source (smtFREE, fullBelowCache, treeNodeCache, *db, Handler(), beast::Journal()); diff --git a/src/ripple/unity/nodestore.cpp b/src/ripple/unity/nodestore.cpp index dca1b71d9..f2cba0b8a 100644 --- a/src/ripple/unity/nodestore.cpp +++ b/src/ripple/unity/nodestore.cpp @@ -38,3 +38,5 @@ #include #include #include + + diff --git a/test/config-example.js b/test/config-example.js index 3bfe59460..16201de6d 100644 --- a/test/config-example.js +++ b/test/config-example.js @@ -56,7 +56,8 @@ exports.servers = { 'admin = allow', 'protocol = ws'), - 'node_db': 'type=memory' + 'node_db': lines('type=memory', + 'path=integration') } };