From 94629edb9ba6e4dd1f960ae5d25d09348adfe613 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Mon, 12 Jan 2015 13:07:38 -0800 Subject: [PATCH] Add NuDB backend: The NuDB database backend is a high performance key/value store presented as an alternative to RocksDB on Mac and Linux deployments, and the preferred backend option for Windows deployments. The LevelDB backend is deprecated for all platforms. This includes these changes: * Add Backend::verify API for doing consistency checks * Add Database::close so caller can catch exceptions * Improved Timing test for NodeStore creates a simulated workload --- Builds/VisualStudio2013/RippleD.vcxproj | 72 ++- .../VisualStudio2013/RippleD.vcxproj.filters | 102 ++++ SConstruct | 8 +- doc/rippled-example.cfg | 9 +- src/ripple/nodestore/backend/NuDBFactory.cpp | 438 ++++++++++++++++++ src/ripple/nodestore/tests/Backend.test.cpp | 2 + src/ripple/nodestore/tests/Database.test.cpp | 10 +- src/ripple/nodestore/tests/Timing.test.cpp | 7 +- src/ripple/unity/beast.cpp | 1 + src/ripple/unity/nodestore.cpp | 5 +- 10 files changed, 635 insertions(+), 19 deletions(-) create mode 100644 src/ripple/nodestore/backend/NuDBFactory.cpp diff --git a/Builds/VisualStudio2013/RippleD.vcxproj b/Builds/VisualStudio2013/RippleD.vcxproj index 4a1824069..2ff2aa02b 100644 --- a/Builds/VisualStudio2013/RippleD.vcxproj +++ b/Builds/VisualStudio2013/RippleD.vcxproj @@ -807,6 +807,71 @@ True + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + True + + + + + + + + + True + + + + + + + True + + + True + + + True + + + + + @@ -2450,6 +2515,9 @@ True + + True + True @@ -3324,8 +3392,8 @@ - ..\..\src\leveldb\include;..\..\src\rocksdb2\include;%(AdditionalIncludeDirectories) - ..\..\src\leveldb\include;..\..\src\rocksdb2\include;%(AdditionalIncludeDirectories) + ..\..\src\leveldb\include;..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories) + ..\..\src\leveldb\include;..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories) diff --git a/Builds/VisualStudio2013/RippleD.vcxproj.filters b/Builds/VisualStudio2013/RippleD.vcxproj.filters index 51d40a277..7650cebaf 100644 --- a/Builds/VisualStudio2013/RippleD.vcxproj.filters +++ b/Builds/VisualStudio2013/RippleD.vcxproj.filters @@ -169,6 +169,15 @@ {EADD6FA3-A535-01B1-8B05-B6363E6AE41E} + + {D20F5803-9150-FD61-2E8F-973EB53E8674} + + + {5CE9DE06-3D47-E45D-4D84-C2FEF2E1D821} + + + {ACEF9E32-BEA2-86E3-BDE3-E772564EA55B} + {94B0990A-9ABE-B1EC-C220-83FD8C2F529F} @@ -1461,6 +1470,96 @@ beast\net\tests + + beast + + + beast\nudb + + + beast\nudb\detail + + + beast\nudb\detail + + + beast\nudb\detail + + + beast\nudb\detail + + + beast\nudb\detail + + + beast\nudb\detail + + + beast\nudb\detail + + + beast\nudb\detail + + + beast\nudb\detail + + + beast\nudb\detail + + + beast\nudb\detail + + + beast\nudb\detail + + + beast\nudb\detail + + + beast\nudb + + + beast\nudb + + + beast\nudb + + + beast\nudb + + + beast\nudb + + + beast\nudb + + + beast\nudb + + + beast\nudb\tests + + + beast\nudb\tests + + + beast\nudb\tests + + + beast\nudb\tests + + + beast\nudb\tests + + + beast\nudb\tests + + + beast\nudb + + + beast\nudb + beast\random @@ -3498,6 +3597,9 @@ ripple\nodestore\backend + + ripple\nodestore\backend + ripple\nodestore\backend diff --git a/SConstruct b/SConstruct index 586ca0cd6..fdbb7714c 100644 --- a/SConstruct +++ b/SConstruct @@ -616,8 +616,9 @@ for tu_style in ['classic', 'unity']: *list_sources('src/ripple/nodestore', '.cpp'), CPPPATH=[ 'src/leveldb/include', - #'src/hyperleveldb/include', # hyper 'src/rocksdb2/include', + 'src/snappy/snappy', + 'src/snappy/config', ]) else: object_builder.add_source_files( @@ -645,9 +646,10 @@ for tu_style in ['classic', 'unity']: object_builder.add_source_files( 'src/ripple/unity/nodestore.cpp', CPPPATH=[ - 'src/leveldb/include', - #'src/hyperleveldb/include', # hyper + 'src/leveldb/include', 'src/rocksdb2/include', + 'src/snappy/snappy', + 'src/snappy/config', ]) git_commit_tag = {} diff --git a/doc/rippled-example.cfg b/doc/rippled-example.cfg index 2f6fee77b..c8894379b 100644 --- a/doc/rippled-example.cfg +++ b/doc/rippled-example.cfg @@ -665,10 +665,11 @@ # # Choices for 'type' (not case-sensitive) # RocksDB Use Facebook's RocksDB database (preferred) -# HyperLevelDB Use an improved version of LevelDB -# SQLite Use SQLite -# LevelDB Use Google's LevelDB database (deprecated) -# none Use no backend +# NuDB Use Ripple Labs' NuDB (Windows preferred) +# HyperLevelDB (Deprecated) +# SQLite (Deprecated) +# LevelDB (Deprecated) +# none (No backend) # # Required keys: # path Location to store the database (all types) diff --git a/src/ripple/nodestore/backend/NuDBFactory.cpp b/src/ripple/nodestore/backend/NuDBFactory.cpp new file mode 100644 index 000000000..193197d58 --- /dev/null +++ b/src/ripple/nodestore/backend/NuDBFactory.cpp @@ -0,0 +1,438 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ripple { +namespace NodeStore { + +class NuDBBackend + : public Backend +{ +public: + enum + { + // This needs to be tuned for the + // distribution of data sizes. + arena_alloc_size = 16 * 1024 * 1024, + + // Version 1 + // No compression + // + typeOne = 1, + + // Version 2 + // Snappy compression + typeTwo = 2, + + + + currentType = typeTwo + }; + + beast::Journal journal_; + size_t const keyBytes_; + std::string const name_; + beast::nudb::store db_; + std::atomic deletePath_; + Scheduler& scheduler_; + + NuDBBackend (int keyBytes, Parameters const& keyValues, + Scheduler& scheduler, beast::Journal journal) + : journal_ (journal) + , keyBytes_ (keyBytes) + , name_ (keyValues ["path"].toStdString ()) + , deletePath_(false) + , scheduler_ (scheduler) + { + if (name_.empty()) + throw std::runtime_error ( + "nodestore: Missing path in NuDB backend"); + auto const folder = boost::filesystem::path (name_); + boost::filesystem::create_directories (folder); + auto const dp = (folder / "nudb.dat").string(); + auto const kp = (folder / "nudb.key").string (); + auto const lp = (folder / "nudb.log").string (); + using beast::nudb::make_salt; + beast::nudb::create (dp, kp, lp, + currentType, make_salt(), keyBytes, + beast::nudb::block_size(kp), + 0.50); + try + { + if (! db_.open (dp, kp, lp, + arena_alloc_size)) + throw std::runtime_error( + "nodestore: open failed"); + if (db_.appnum() != currentType) + throw std::runtime_error( + "nodestore: unknown appnum"); + } + catch (std::exception const& e) + { + // log and terminate? + std::cerr << e.what(); + std::terminate(); + } + } + + ~NuDBBackend () + { + close(); + } + + std::string + getName() + { + return name_; + } + + void + close() override + { + if (db_.is_open()) + { + db_.close(); + if (deletePath_) + { + boost::filesystem::remove_all (name_); + } + } + } + + //-------------------------------------------------------------------------- + + class Buffer + { + private: + std::size_t size_ = 0; + std::size_t capacity_ = 0; + std::unique_ptr buf_; + + public: + Buffer() = default; + Buffer (Buffer const&) = delete; + Buffer& operator= (Buffer const&) = delete; + + explicit + Buffer (std::size_t n) + { + resize (n); + } + + std::size_t + size() const + { + return size_; + } + + std::size_t + capacity() const + { + return capacity_; + } + + void* + get() + { + return buf_.get(); + } + + void + resize (std::size_t n) + { + if (capacity_ < n) + { + capacity_ = beast::nudb::detail::ceil_pow2(n); + buf_.reset (new std::uint8_t[capacity_]); + } + size_ = n; + } + + // Meet the requirements of BufferFactory + void* + operator() (std::size_t n) + { + resize(n); + return get(); + } + }; + + //-------------------------------------------------------------------------- + + Status + fetch1 (void const* key, + std::shared_ptr * pno) + { + pno->reset(); + std::size_t bytes; + std::unique_ptr data; + if (! db_.fetch (key, + [&data, &bytes](std::size_t n) + { + bytes = n; + data.reset(new std::uint8_t[bytes]); + return data.get(); + })) + return notFound; + DecodedBlob decoded (key, data.get(), bytes); + if (! decoded.wasOk ()) + return dataCorrupt; + *pno = decoded.createObject(); + return ok; + } + + void + insert1 (void const* key, void const* data, + std::size_t size) + { + db_.insert (key, data, size); + } + + //-------------------------------------------------------------------------- + + Status + fetch2 (void const* key, + std::shared_ptr * pno) + { + pno->reset(); + std::size_t actual; + std::unique_ptr compressed; + if (! db_.fetch (key, + [&](std::size_t n) + { + actual = n; + compressed.reset( + new char[n]); + return compressed.get(); + })) + return notFound; + std::size_t size; + if (! snappy::GetUncompressedLength( + (char const*)compressed.get(), + actual, &size)) + return dataCorrupt; + std::unique_ptr data (new char[size]); + snappy::RawUncompress (compressed.get(), + actual, data.get()); + DecodedBlob decoded (key, data.get(), size); + if (! decoded.wasOk ()) + return dataCorrupt; + *pno = decoded.createObject(); + return ok; + } + + void + insert2 (void const* key, void const* data, + std::size_t size) + { + std::unique_ptr buf ( + new char[snappy::MaxCompressedLength(size)]); + std::size_t actual; + snappy::RawCompress ((char const*)data, size, + buf.get(), &actual); + db_.insert (key, buf.get(), actual); + } + + //-------------------------------------------------------------------------- + + Status + fetch (void const* key, NodeObject::Ptr* pno) + { + Buffer b1; + if (! db_.fetch (key, b1)) + return notFound; + + switch (db_.appnum()) + { + case typeOne: return fetch1 (key, pno); + case typeTwo: return fetch2 (key, pno); + } + throw std::runtime_error( + "nodestore: unknown appnum"); + return notFound; + } + + void + do_insert (std::shared_ptr const& no) + { + EncodedBlob e; + e.prepare (no); + switch (db_.appnum()) + { + case typeOne: return insert1 (e.getKey(), e.getData(), e.getSize()); + case typeTwo: return insert2 (e.getKey(), e.getData(), e.getSize()); + } + throw std::runtime_error( + "nodestore: unknown appnum"); + } + + void + store (std::shared_ptr const& no) override + { + BatchWriteReport report; + report.writeCount = 1; + auto const start = + std::chrono::steady_clock::now(); + do_insert (no); + report.elapsed = std::chrono::duration_cast < + std::chrono::milliseconds>( + std::chrono::steady_clock::now() - start); + scheduler_.onBatchWrite (report); + } + + void + storeBatch (Batch const& batch) override + { + BatchWriteReport report; + EncodedBlob encoded; + report.writeCount = batch.size(); + auto const start = + std::chrono::steady_clock::now(); + for (auto const& e : batch) + do_insert (e); + report.elapsed = std::chrono::duration_cast < + std::chrono::milliseconds>( + std::chrono::steady_clock::now() - start); + scheduler_.onBatchWrite (report); + } + + void + for_each (std::function f) + { + auto const dp = db_.dat_path(); + auto const kp = db_.key_path(); + auto const lp = db_.log_path(); + auto const appnum = db_.appnum(); + db_.close(); + beast::nudb::visit (dp, + [&]( + void const* key, std::size_t key_bytes, + void const* data, std::size_t size) + { + switch (appnum) + { + case typeOne: + { + DecodedBlob decoded (key, data, size); + if (! decoded.wasOk ()) + return false; + f (decoded.createObject()); + break; + } + case typeTwo: + { + std::size_t actual; + if (! snappy::GetUncompressedLength( + (char const*)data, size, &actual)) + return false; + std::unique_ptr buf (new char[actual]); + if (! snappy::RawUncompress ((char const*)data, + size, buf.get())) + return false; + DecodedBlob decoded (key, buf.get(), actual); + if (! decoded.wasOk ()) + return false; + f (decoded.createObject()); + break; + } + } + return true; + }); + db_.open (dp, kp, lp, + arena_alloc_size); + } + + int + getWriteLoad () + { + return 0; + } + + void + setDeletePath() override + { + deletePath_ = true; + } + + void + verify() override + { + auto const dp = db_.dat_path(); + auto const kp = db_.key_path(); + auto const lp = db_.log_path(); + db_.close(); + beast::nudb::verify (dp, kp); + db_.open (dp, kp, lp, + arena_alloc_size); + } +}; + +//------------------------------------------------------------------------------ + +class NuDBFactory : public Factory +{ +public: + NuDBFactory() + { + Manager::instance().insert(*this); + } + + ~NuDBFactory() + { + Manager::instance().erase(*this); + } + + std::string + getName() const + { + return "NuDB"; + } + + std::unique_ptr + createInstance ( + size_t keyBytes, + Parameters const& keyValues, + Scheduler& scheduler, + beast::Journal journal) + { + return std::make_unique ( + keyBytes, keyValues, scheduler, journal); + } +}; + +static NuDBFactory nuDBFactory; + +} +} diff --git a/src/ripple/nodestore/tests/Backend.test.cpp b/src/ripple/nodestore/tests/Backend.test.cpp index 6209712ce..900569d38 100644 --- a/src/ripple/nodestore/tests/Backend.test.cpp +++ b/src/ripple/nodestore/tests/Backend.test.cpp @@ -94,6 +94,8 @@ public: { int const seedValue = 50; + testBackend ("nudb", seedValue); + testBackend ("leveldb", seedValue); #if RIPPLE_HYPERLEVELDB_AVAILABLE diff --git a/src/ripple/nodestore/tests/Database.test.cpp b/src/ripple/nodestore/tests/Database.test.cpp index bb7e94b72..33576d906 100644 --- a/src/ripple/nodestore/tests/Database.test.cpp +++ b/src/ripple/nodestore/tests/Database.test.cpp @@ -26,7 +26,7 @@ namespace ripple { namespace NodeStore { -class NodeStoreDatabase_test : public TestBase +class Database_test : public TestBase { public: void testImport (std::string const& destBackendType, @@ -182,6 +182,8 @@ public: void runBackendTests (bool useEphemeralDatabase, std::int64_t const seedValue) { + testNodeStore ("nudb", useEphemeralDatabase, true, seedValue); + testNodeStore ("leveldb", useEphemeralDatabase, true, seedValue); #if RIPPLE_HYPERLEVELDB_AVAILABLE @@ -201,8 +203,10 @@ public: void runImportTests (std::int64_t const seedValue) { - testImport ("leveldb", "leveldb", seedValue); + testImport ("nudb", "nudb", seedValue); + testImport ("leveldb", "leveldb", seedValue); + #if RIPPLE_HYPERLEVELDB_AVAILABLE testImport ("hyperleveldb", "hyperleveldb", seedValue); #endif @@ -232,7 +236,7 @@ public: } }; -BEAST_DEFINE_TESTSUITE(NodeStoreDatabase,ripple_core,ripple); +BEAST_DEFINE_TESTSUITE(Database,NodeStore,ripple); } } diff --git a/src/ripple/nodestore/tests/Timing.test.cpp b/src/ripple/nodestore/tests/Timing.test.cpp index 179187deb..644a13b2f 100644 --- a/src/ripple/nodestore/tests/Timing.test.cpp +++ b/src/ripple/nodestore/tests/Timing.test.cpp @@ -147,7 +147,7 @@ public: missingNodePercent = 20 }; - std::size_t const default_repeat = 1; + std::size_t const default_repeat = 3; #ifndef NDEBUG std::size_t const default_items = 10000; #else @@ -708,10 +708,7 @@ public: */ std::string default_args = - #ifdef _MSC_VER - "type=leveldb" - #endif - //"type=nudb" + "type=nudb" #if RIPPLE_ROCKSDB_AVAILABLE ";type=rocksdb,open_files=2000,filter_bits=12,cache_mb=256," "file_size_mb=8,file_size_mult=2" diff --git a/src/ripple/unity/beast.cpp b/src/ripple/unity/beast.cpp index d2aedbe47..046e41ab9 100644 --- a/src/ripple/unity/beast.cpp +++ b/src/ripple/unity/beast.cpp @@ -40,6 +40,7 @@ #include #include #include +//#include #include #include #include diff --git a/src/ripple/unity/nodestore.cpp b/src/ripple/unity/nodestore.cpp index f2cba0b8a..ee032dee3 100644 --- a/src/ripple/unity/nodestore.cpp +++ b/src/ripple/unity/nodestore.cpp @@ -19,9 +19,12 @@ #include +#include + #include #include #include +#include #include #include @@ -38,5 +41,3 @@ #include #include #include - -