From 38e99e01f9ff23e892a69bec3b11c6e45b117430 Mon Sep 17 00:00:00 2001 From: Donovan Hide Date: Sun, 2 Nov 2014 00:54:26 +0000 Subject: [PATCH] Improve nodestore benchmarking: * Use more succinct while loops on NodeFactory. * Better formatting of multiple test results. * Updated benchmarks. * Use simpler and faster RNG to generate test data. --- src/ripple/nodestore/Benchmarks.md | 35 ++-- src/ripple/nodestore/tests/TimingTests.cpp | 194 ++++++++++++++------- 2 files changed, 147 insertions(+), 82 deletions(-) diff --git a/src/ripple/nodestore/Benchmarks.md b/src/ripple/nodestore/Benchmarks.md index 80edebbdb..1b5473a69 100644 --- a/src/ripple/nodestore/Benchmarks.md +++ b/src/ripple/nodestore/Benchmarks.md @@ -1,31 +1,20 @@ #Benchmarks ``` -$rippled --unittest=NodeStoreTiming --unittest-arg="type=rocksdbquick,style=level,num_objects=2000000" - +$rippled --unittest=NodeStoreTiming --unittest-arg="type=rocksdb,num_objects=2000000,open_files=2000,filter_bits=12,cache_mb=256,file_size_mb=8,file_size_mult=2;type=rocksdbquick,num_objects=2000000" +2014-Nov-01 21:49:02 Validators:NFO Validators constructed (info) ripple.bench.NodeStoreTiming repeatableObject - Batch Insert Fetch 50/50 Fetch Missing Fetch Random Inserts Ordered Fetch - 59.53 12.67 6.04 11.33 25.55 52.15 type=rocksdbquick,style=level,num_objects=2000000 + Config Run Inserts Batch Insert Fetch 50/50 Ordered Fetch Fetch Random Fetch Missing + 0 0 160.57 699.08 50.88 51.17 29.99 14.05 + 0 1 406.70 797.47 32.53 60.18 46.63 14.94 + 0 2 408.81 743.89 42.79 72.99 49.03 14.93 + 1 0 111.03 151.06 28.89 53.44 31.88 18.46 + 1 1 92.63 160.75 19.64 41.60 28.17 10.40 + 1 2 101.31 122.83 30.66 55.65 32.69 16.15 -$rippled --unittest=NodeStoreTiming --unittest-arg="type=rocksdbquick,style=level,num_objects=2000000" - -ripple.bench.NodeStoreTiming repeatableObject - Batch Insert Fetch 50/50 Fetch Missing Fetch Random Inserts Ordered Fetch - 44.29 27.45 5.95 20.47 23.58 53.60 type=rocksdbquick,style=level,num_objects=2000000 -``` - -``` -$rippled --unittest=NodeStoreTiming --unittest-arg="type=rocksdb,num_objects=2000000,open_files=2000,filter_bits=12,cache_mb=256,file_size_mb=8,file_size_mult=2" - -ripple.bench.NodeStoreTiming repeatableObject - Batch Insert Fetch 50/50 Fetch Missing Fetch Random Inserts Ordered Fetch - 377.61 30.62 10.05 17.41 201.73 64.46 type=rocksdb,num_objects=2000000,open_files=2000,filter_bits=12,cache_mb=256,file_size_mb=8,file_size_mult=2 - -$rippled --unittest=NodeStoreTiming --unittest-arg="type=rocksdb,num_objects=2000000,open_files=2000,filter_bits=12,cache_mb=256,file_size_mb=8,file_size_mult=2" - -ripple.bench.NodeStoreTiming repeatableObject - Batch Insert Fetch 50/50 Fetch Missing Fetch Random Inserts Ordered Fetch - 405.83 29.48 11.29 25.81 209.05 55.75 type=rocksdb,num_objects=2000000,open_files=2000,filter_bits=12,cache_mb=256,file_size_mb=8,file_size_mult=2 +Configs: + 0: type=rocksdb,num_objects=2000000,open_files=2000,filter_bits=12,cache_mb=256,file_size_mb=8,file_size_mult=2 + 1: type=rocksdbquick,num_objects=2000000 ``` ##Discussion diff --git a/src/ripple/nodestore/tests/TimingTests.cpp b/src/ripple/nodestore/tests/TimingTests.cpp index 5219d8e94..ee4d3da20 100644 --- a/src/ripple/nodestore/tests/TimingTests.cpp +++ b/src/ripple/nodestore/tests/TimingTests.cpp @@ -17,12 +17,66 @@ */ //============================================================================== +#include +#include + namespace ripple { namespace NodeStore { class NodeStoreTiming_test : public TestBase { public: + // Simple and fast RNG based on: + // http://xorshift.di.unimi.it/xorshift128plus.c + // does not accept seed==0 + class XORShiftEngine{ + public: + using result_type = std::uint64_t; + + static const result_type default_seed = 1977u; + + explicit XORShiftEngine(result_type val = default_seed) { seed(val); } + + void seed(result_type const seed) + { + if (seed==0) + throw std::range_error("zero seed supplied"); + s[0] = murmurhash3(seed); + s[1] = murmurhash3(s[0]); + } + + result_type operator()() + { + result_type s1 = s[0]; + const result_type s0 = s[1]; + s[0] = s0; + s1 ^= s1 << 23; + return (s[1] = (s1 ^ s0 ^ (s1 >> 17) ^ (s0 >> 26))) + s0; + } + + static BEAST_CONSTEXPR result_type min() + { + return std::numeric_limits::min(); + } + + static BEAST_CONSTEXPR result_type max() + { + return std::numeric_limits::max(); + } + + private: + result_type s[2]; + + static result_type murmurhash3(result_type x) + { + x ^= x >> 33; + x *= 0xff51afd7ed558ccdULL; + x ^= x >> 33; + x *= 0xc4ceb9fe1a85ec53ULL; + return x ^= x >> 33; + } + }; + class NodeFactory { enum @@ -41,7 +95,7 @@ public: numObjects_(numObjects), count_(0), rng_(seed), - key_(minKey, maxKey), + key_(minKey+1, maxKey+1), value_(minValueLength, maxValueLength), type_(hotLEDGER, hotTRANSACTION_NODE), ledger_(minLedger, maxLedger) @@ -51,7 +105,8 @@ public: NodeObject::Ptr next() { // Stop when done - if (count_==numObjects_) return nullptr; + if (count_ == numObjects_) + return nullptr; count_++; // Seed from range between minKey and maxKey to ensure repeatability @@ -96,7 +151,7 @@ public: std::int64_t numObjects_; std::int64_t count_; std::mt19937_64 rng_; - std::mt19937_64 r_; + XORShiftEngine r_; std::uniform_int_distribution key_; std::uniform_int_distribution value_; std::uniform_int_distribution type_; @@ -113,7 +168,7 @@ public: std::set out; - for (auto node = factory.next(); node; node = factory.next()) + while (auto node = factory.next()) { auto it = out.find(node); if (it == out.end()) @@ -131,21 +186,16 @@ public: class Stopwatch { public: - Stopwatch () - { - } + Stopwatch() {} - void start () - { - m_startTime = beast::Time::getHighResolutionTicks (); - } + void start() { m_startTime = beast::Time::getHighResolutionTicks(); } - double getElapsed () - { - std::int64_t const now = beast::Time::getHighResolutionTicks(); + double getElapsed() + { + std::int64_t const now = beast::Time::getHighResolutionTicks(); - return beast::Time::highResolutionTicksToSeconds (now - m_startTime); - } + return beast::Time::highResolutionTicksToSeconds(now - m_startTime); + } private: std::int64_t m_startTime; @@ -161,7 +211,7 @@ public: using check_func = std::function; using backend_ptr = std::unique_ptr; using manager_ptr = std::unique_ptr; - using result_type = std::map; + using result_type = std::vector>; static bool checkNotFound(Status const status) { @@ -179,13 +229,14 @@ public: check_func f) { factory.reset(); - for (auto expected = factory.next(); expected; expected = factory.next()) + while (auto expected = factory.next()) { NodeObject::Ptr got; Status const status = backend->fetch(expected->getHash().cbegin(), &got); - expect(f(status), "Wrong status"); + expect(f(status), + "Wrong status for: " + to_string(expected->getHash())); if (status == ok) { expect(got != nullptr, "Should not be null"); @@ -197,33 +248,24 @@ public: static void testInsert(backend_ptr& backend, NodeFactory& factory) { factory.reset(); - while (auto node = factory.next()) - backend->store(node); + while (auto node = factory.next()) backend->store(node); } static void testBatchInsert(backend_ptr& backend, NodeFactory& factory) { factory.reset(); Batch batch; - for (; factory.fillBatch(batch, batchSize);) - backend->storeBatch(batch); + while (factory.fillBatch(batch, batchSize)) backend->storeBatch(batch); } - result_type benchmarkBackend(std::string const& config, - std::int64_t const seedValue) + result_type benchmarkBackend(beast::StringPairArray const& params, + std::int64_t const seedValue, + std::int64_t const numObjects) { Stopwatch t; result_type results; - auto params = parseDelimitedKeyValueString(config, ','); - - std::int64_t numObjects = params["num_objects"].getIntValue(); - params.remove("num_objects"); - auto manager = make_Manager(); - - beast::UnitTestUtilities::TempDirectory path ("node_db"); - params.set("path", path.getFullPathName()); DummyScheduler scheduler; beast::Journal j; @@ -245,27 +287,27 @@ public: t.start(); testInsert(backend, insertFactory); - results["Inserts"] = t.getElapsed(); + results.emplace_back("Inserts", t.getElapsed()); t.start(); testBatchInsert(backend, batchFactory); - results["Batch Insert"] = t.getElapsed(); + results.emplace_back("Batch Insert", t.getElapsed()); t.start(); testFetch(backend, mixedFactory, checkOkOrNotFound); - results["Fetch 50/50"] = t.getElapsed(); + results.emplace_back("Fetch 50/50", t.getElapsed()); t.start(); testFetch(backend, insertFactory, checkOk); - results["Ordered Fetch"] = t.getElapsed(); + results.emplace_back("Ordered Fetch", t.getElapsed()); t.start(); testFetch(backend, randomFactory, checkOkOrNotFound); - results["Fetch Random"] = t.getElapsed(); + results.emplace_back("Fetch Random", t.getElapsed()); t.start(); testFetch(backend, missingFactory, checkNotFound); - results["Fetch Missing"] = t.getElapsed(); + results.emplace_back("Fetch Missing", t.getElapsed()); return results; } @@ -282,12 +324,14 @@ public: // Each configuration is a comma delimited list of key-value pairs. // Each pair is separated by a '='. // 'type' defaults to 'rocksdb' - // 'num_objects' defaults to '100000' + // 'num_objects' defaults to '100000' + // 'num_runs' defaults to '3' // defaultArguments serves as an example. std::string defaultArguments = - "type=rocksdb,open_files=2000,filter_bits=12,cache_mb=256,file_size_mb=8,file_size_mult=2,num_objects=100000;" - "type=hyperleveldb,num_objects=100000"; + "type=rocksdb,open_files=2000,filter_bits=12,cache_mb=256" + "file_size_mb=8,file_size_mult=2,num_objects=100000,num_runs=3;" + "type=hyperleveldb,num_objects=100000,num_runs=3"; auto args = arg(); @@ -296,33 +340,65 @@ public: std::vector configs; boost::split (configs, args, boost::algorithm::is_any_of (";")); - std::map results; + std::map> results; for (auto& config : configs) { - // Trim trailing comma if exists - boost::trim_right_if(config, boost::algorithm::is_any_of(",")); + auto params = parseDelimitedKeyValueString(config, ','); // Defaults - if (config.find("type=") == std::string::npos) - config += ",type=rocksdb"; - if (config.find("num_objects") == std::string::npos) - config += ",num_objects=100000"; - results[config] = benchmarkBackend(config, seedValue); + std::int64_t numRuns = 3; + std::int64_t numObjects = 100000; + + if (!params["num_objects"].isEmpty()) + numObjects = params["num_objects"].getIntValue(); + + if (!params["num_runs"].isEmpty()) + numRuns = params["num_runs"].getIntValue(); + + if (params["type"].isEmpty()) + params.set("type", "rocksdb"); + + for (std::int64_t i = 0; i < numRuns; i++) + { + beast::UnitTestUtilities::TempDirectory path("node_db"); + params.set("path", path.getFullPathName()); + results[config].emplace_back( + benchmarkBackend(params, seedValue + i, numObjects)); + } } - std::stringstream ss; - ss << std::setprecision(2) << std::fixed; - for (auto const& header : results.begin()->second) - ss << std::setw(14) << header.first << " "; - ss << std::endl; + std::stringstream header; + std::stringstream stats; + std::stringstream legend; + + auto firstRun = results.begin()->second.begin(); + header << std::setw(7) << "Config" << std::setw(4) << "Run"; + for (auto const& title : *firstRun) + header << std::setw(14) << title.first; + + stats << std::setprecision(2) << std::fixed; + + std::int64_t resultCount = 0; for (auto const& result : results) { - for (auto const item : result.second) - ss << std::setw(14) << item.second << " "; - ss << result.first << std::endl; + std::int64_t runCount = 0; + for (auto const& run : result.second) + { + stats << std::setw(7) << resultCount << std::setw(4) + << runCount; + for (auto const& item : run) + stats << std::setw(14) << item.second; + runCount++; + stats << std::endl; + + } + legend << std::setw(2) << resultCount << ": " << result.first + << std::endl; + resultCount++; } - log << ss.str(); + log << header.str() << std::endl << stats.str() << std::endl + << "Configs:" << std::endl << legend.str(); } }; BEAST_DEFINE_TESTSUITE_MANUAL(NodeStoreTiming,bench,ripple);