diff --git a/Builds/VisualStudio2013/RippleD.vcxproj b/Builds/VisualStudio2013/RippleD.vcxproj
index 4a1824069..2ff2aa02b 100644
--- a/Builds/VisualStudio2013/RippleD.vcxproj
+++ b/Builds/VisualStudio2013/RippleD.vcxproj
@@ -807,6 +807,71 @@
True
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ True
+
+
+
+
+
+
+
+
+ True
+
+
+
+
+
+
+ True
+
+
+ True
+
+
+ True
+
+
+
+
+
@@ -2450,6 +2515,9 @@
True
+
+ True
+
True
@@ -3324,8 +3392,8 @@
- ..\..\src\leveldb\include;..\..\src\rocksdb2\include;%(AdditionalIncludeDirectories)
- ..\..\src\leveldb\include;..\..\src\rocksdb2\include;%(AdditionalIncludeDirectories)
+ ..\..\src\leveldb\include;..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)
+ ..\..\src\leveldb\include;..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)
diff --git a/Builds/VisualStudio2013/RippleD.vcxproj.filters b/Builds/VisualStudio2013/RippleD.vcxproj.filters
index 51d40a277..7650cebaf 100644
--- a/Builds/VisualStudio2013/RippleD.vcxproj.filters
+++ b/Builds/VisualStudio2013/RippleD.vcxproj.filters
@@ -169,6 +169,15 @@
{EADD6FA3-A535-01B1-8B05-B6363E6AE41E}
+
+ {D20F5803-9150-FD61-2E8F-973EB53E8674}
+
+
+ {5CE9DE06-3D47-E45D-4D84-C2FEF2E1D821}
+
+
+ {ACEF9E32-BEA2-86E3-BDE3-E772564EA55B}
+
{94B0990A-9ABE-B1EC-C220-83FD8C2F529F}
@@ -1461,6 +1470,96 @@
beast\net\tests
+
+ beast
+
+
+ beast\nudb
+
+
+ beast\nudb\detail
+
+
+ beast\nudb\detail
+
+
+ beast\nudb\detail
+
+
+ beast\nudb\detail
+
+
+ beast\nudb\detail
+
+
+ beast\nudb\detail
+
+
+ beast\nudb\detail
+
+
+ beast\nudb\detail
+
+
+ beast\nudb\detail
+
+
+ beast\nudb\detail
+
+
+ beast\nudb\detail
+
+
+ beast\nudb\detail
+
+
+ beast\nudb\detail
+
+
+ beast\nudb
+
+
+ beast\nudb
+
+
+ beast\nudb
+
+
+ beast\nudb
+
+
+ beast\nudb
+
+
+ beast\nudb
+
+
+ beast\nudb
+
+
+ beast\nudb\tests
+
+
+ beast\nudb\tests
+
+
+ beast\nudb\tests
+
+
+ beast\nudb\tests
+
+
+ beast\nudb\tests
+
+
+ beast\nudb\tests
+
+
+ beast\nudb
+
+
+ beast\nudb
+
beast\random
@@ -3498,6 +3597,9 @@
ripple\nodestore\backend
+
+ ripple\nodestore\backend
+
ripple\nodestore\backend
diff --git a/SConstruct b/SConstruct
index 586ca0cd6..fdbb7714c 100644
--- a/SConstruct
+++ b/SConstruct
@@ -616,8 +616,9 @@ for tu_style in ['classic', 'unity']:
*list_sources('src/ripple/nodestore', '.cpp'),
CPPPATH=[
'src/leveldb/include',
- #'src/hyperleveldb/include', # hyper
'src/rocksdb2/include',
+ 'src/snappy/snappy',
+ 'src/snappy/config',
])
else:
object_builder.add_source_files(
@@ -645,9 +646,10 @@ for tu_style in ['classic', 'unity']:
object_builder.add_source_files(
'src/ripple/unity/nodestore.cpp',
CPPPATH=[
- 'src/leveldb/include',
- #'src/hyperleveldb/include', # hyper
+ 'src/leveldb/include',
'src/rocksdb2/include',
+ 'src/snappy/snappy',
+ 'src/snappy/config',
])
git_commit_tag = {}
diff --git a/doc/rippled-example.cfg b/doc/rippled-example.cfg
index 2f6fee77b..c8894379b 100644
--- a/doc/rippled-example.cfg
+++ b/doc/rippled-example.cfg
@@ -665,10 +665,11 @@
#
# Choices for 'type' (not case-sensitive)
# RocksDB Use Facebook's RocksDB database (preferred)
-# HyperLevelDB Use an improved version of LevelDB
-# SQLite Use SQLite
-# LevelDB Use Google's LevelDB database (deprecated)
-# none Use no backend
+# NuDB Use Ripple Labs' NuDB (Windows preferred)
+# HyperLevelDB (Deprecated)
+# SQLite (Deprecated)
+# LevelDB (Deprecated)
+# none (No backend)
#
# Required keys:
# path Location to store the database (all types)
diff --git a/src/ripple/nodestore/backend/NuDBFactory.cpp b/src/ripple/nodestore/backend/NuDBFactory.cpp
new file mode 100644
index 000000000..193197d58
--- /dev/null
+++ b/src/ripple/nodestore/backend/NuDBFactory.cpp
@@ -0,0 +1,438 @@
+//------------------------------------------------------------------------------
+/*
+ This file is part of rippled: https://github.com/ripple/rippled
+ Copyright (c) 2012, 2013 Ripple Labs Inc.
+
+ Permission to use, copy, modify, and/or distribute this software for any
+ purpose with or without fee is hereby granted, provided that the above
+ copyright notice and this permission notice appear in all copies.
+
+ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+namespace ripple {
+namespace NodeStore {
+
+class NuDBBackend
+ : public Backend
+{
+public:
+ enum
+ {
+ // This needs to be tuned for the
+ // distribution of data sizes.
+ arena_alloc_size = 16 * 1024 * 1024,
+
+ // Version 1
+ // No compression
+ //
+ typeOne = 1,
+
+ // Version 2
+ // Snappy compression
+ typeTwo = 2,
+
+
+
+ currentType = typeTwo
+ };
+
+ beast::Journal journal_;
+ size_t const keyBytes_;
+ std::string const name_;
+ beast::nudb::store db_;
+ std::atomic deletePath_;
+ Scheduler& scheduler_;
+
+ NuDBBackend (int keyBytes, Parameters const& keyValues,
+ Scheduler& scheduler, beast::Journal journal)
+ : journal_ (journal)
+ , keyBytes_ (keyBytes)
+ , name_ (keyValues ["path"].toStdString ())
+ , deletePath_(false)
+ , scheduler_ (scheduler)
+ {
+ if (name_.empty())
+ throw std::runtime_error (
+ "nodestore: Missing path in NuDB backend");
+ auto const folder = boost::filesystem::path (name_);
+ boost::filesystem::create_directories (folder);
+ auto const dp = (folder / "nudb.dat").string();
+ auto const kp = (folder / "nudb.key").string ();
+ auto const lp = (folder / "nudb.log").string ();
+ using beast::nudb::make_salt;
+ beast::nudb::create (dp, kp, lp,
+ currentType, make_salt(), keyBytes,
+ beast::nudb::block_size(kp),
+ 0.50);
+ try
+ {
+ if (! db_.open (dp, kp, lp,
+ arena_alloc_size))
+ throw std::runtime_error(
+ "nodestore: open failed");
+ if (db_.appnum() != currentType)
+ throw std::runtime_error(
+ "nodestore: unknown appnum");
+ }
+ catch (std::exception const& e)
+ {
+ // log and terminate?
+ std::cerr << e.what();
+ std::terminate();
+ }
+ }
+
+ ~NuDBBackend ()
+ {
+ close();
+ }
+
+ std::string
+ getName()
+ {
+ return name_;
+ }
+
+ void
+ close() override
+ {
+ if (db_.is_open())
+ {
+ db_.close();
+ if (deletePath_)
+ {
+ boost::filesystem::remove_all (name_);
+ }
+ }
+ }
+
+ //--------------------------------------------------------------------------
+
+ class Buffer
+ {
+ private:
+ std::size_t size_ = 0;
+ std::size_t capacity_ = 0;
+ std::unique_ptr buf_;
+
+ public:
+ Buffer() = default;
+ Buffer (Buffer const&) = delete;
+ Buffer& operator= (Buffer const&) = delete;
+
+ explicit
+ Buffer (std::size_t n)
+ {
+ resize (n);
+ }
+
+ std::size_t
+ size() const
+ {
+ return size_;
+ }
+
+ std::size_t
+ capacity() const
+ {
+ return capacity_;
+ }
+
+ void*
+ get()
+ {
+ return buf_.get();
+ }
+
+ void
+ resize (std::size_t n)
+ {
+ if (capacity_ < n)
+ {
+ capacity_ = beast::nudb::detail::ceil_pow2(n);
+ buf_.reset (new std::uint8_t[capacity_]);
+ }
+ size_ = n;
+ }
+
+ // Meet the requirements of BufferFactory
+ void*
+ operator() (std::size_t n)
+ {
+ resize(n);
+ return get();
+ }
+ };
+
+ //--------------------------------------------------------------------------
+
+ Status
+ fetch1 (void const* key,
+ std::shared_ptr * pno)
+ {
+ pno->reset();
+ std::size_t bytes;
+ std::unique_ptr data;
+ if (! db_.fetch (key,
+ [&data, &bytes](std::size_t n)
+ {
+ bytes = n;
+ data.reset(new std::uint8_t[bytes]);
+ return data.get();
+ }))
+ return notFound;
+ DecodedBlob decoded (key, data.get(), bytes);
+ if (! decoded.wasOk ())
+ return dataCorrupt;
+ *pno = decoded.createObject();
+ return ok;
+ }
+
+ void
+ insert1 (void const* key, void const* data,
+ std::size_t size)
+ {
+ db_.insert (key, data, size);
+ }
+
+ //--------------------------------------------------------------------------
+
+ Status
+ fetch2 (void const* key,
+ std::shared_ptr * pno)
+ {
+ pno->reset();
+ std::size_t actual;
+ std::unique_ptr compressed;
+ if (! db_.fetch (key,
+ [&](std::size_t n)
+ {
+ actual = n;
+ compressed.reset(
+ new char[n]);
+ return compressed.get();
+ }))
+ return notFound;
+ std::size_t size;
+ if (! snappy::GetUncompressedLength(
+ (char const*)compressed.get(),
+ actual, &size))
+ return dataCorrupt;
+ std::unique_ptr data (new char[size]);
+ snappy::RawUncompress (compressed.get(),
+ actual, data.get());
+ DecodedBlob decoded (key, data.get(), size);
+ if (! decoded.wasOk ())
+ return dataCorrupt;
+ *pno = decoded.createObject();
+ return ok;
+ }
+
+ void
+ insert2 (void const* key, void const* data,
+ std::size_t size)
+ {
+ std::unique_ptr buf (
+ new char[snappy::MaxCompressedLength(size)]);
+ std::size_t actual;
+ snappy::RawCompress ((char const*)data, size,
+ buf.get(), &actual);
+ db_.insert (key, buf.get(), actual);
+ }
+
+ //--------------------------------------------------------------------------
+
+ Status
+ fetch (void const* key, NodeObject::Ptr* pno)
+ {
+ Buffer b1;
+ if (! db_.fetch (key, b1))
+ return notFound;
+
+ switch (db_.appnum())
+ {
+ case typeOne: return fetch1 (key, pno);
+ case typeTwo: return fetch2 (key, pno);
+ }
+ throw std::runtime_error(
+ "nodestore: unknown appnum");
+ return notFound;
+ }
+
+ void
+ do_insert (std::shared_ptr const& no)
+ {
+ EncodedBlob e;
+ e.prepare (no);
+ switch (db_.appnum())
+ {
+ case typeOne: return insert1 (e.getKey(), e.getData(), e.getSize());
+ case typeTwo: return insert2 (e.getKey(), e.getData(), e.getSize());
+ }
+ throw std::runtime_error(
+ "nodestore: unknown appnum");
+ }
+
+ void
+ store (std::shared_ptr const& no) override
+ {
+ BatchWriteReport report;
+ report.writeCount = 1;
+ auto const start =
+ std::chrono::steady_clock::now();
+ do_insert (no);
+ report.elapsed = std::chrono::duration_cast <
+ std::chrono::milliseconds>(
+ std::chrono::steady_clock::now() - start);
+ scheduler_.onBatchWrite (report);
+ }
+
+ void
+ storeBatch (Batch const& batch) override
+ {
+ BatchWriteReport report;
+ EncodedBlob encoded;
+ report.writeCount = batch.size();
+ auto const start =
+ std::chrono::steady_clock::now();
+ for (auto const& e : batch)
+ do_insert (e);
+ report.elapsed = std::chrono::duration_cast <
+ std::chrono::milliseconds>(
+ std::chrono::steady_clock::now() - start);
+ scheduler_.onBatchWrite (report);
+ }
+
+ void
+ for_each (std::function f)
+ {
+ auto const dp = db_.dat_path();
+ auto const kp = db_.key_path();
+ auto const lp = db_.log_path();
+ auto const appnum = db_.appnum();
+ db_.close();
+ beast::nudb::visit (dp,
+ [&](
+ void const* key, std::size_t key_bytes,
+ void const* data, std::size_t size)
+ {
+ switch (appnum)
+ {
+ case typeOne:
+ {
+ DecodedBlob decoded (key, data, size);
+ if (! decoded.wasOk ())
+ return false;
+ f (decoded.createObject());
+ break;
+ }
+ case typeTwo:
+ {
+ std::size_t actual;
+ if (! snappy::GetUncompressedLength(
+ (char const*)data, size, &actual))
+ return false;
+ std::unique_ptr buf (new char[actual]);
+ if (! snappy::RawUncompress ((char const*)data,
+ size, buf.get()))
+ return false;
+ DecodedBlob decoded (key, buf.get(), actual);
+ if (! decoded.wasOk ())
+ return false;
+ f (decoded.createObject());
+ break;
+ }
+ }
+ return true;
+ });
+ db_.open (dp, kp, lp,
+ arena_alloc_size);
+ }
+
+ int
+ getWriteLoad ()
+ {
+ return 0;
+ }
+
+ void
+ setDeletePath() override
+ {
+ deletePath_ = true;
+ }
+
+ void
+ verify() override
+ {
+ auto const dp = db_.dat_path();
+ auto const kp = db_.key_path();
+ auto const lp = db_.log_path();
+ db_.close();
+ beast::nudb::verify (dp, kp);
+ db_.open (dp, kp, lp,
+ arena_alloc_size);
+ }
+};
+
+//------------------------------------------------------------------------------
+
+class NuDBFactory : public Factory
+{
+public:
+ NuDBFactory()
+ {
+ Manager::instance().insert(*this);
+ }
+
+ ~NuDBFactory()
+ {
+ Manager::instance().erase(*this);
+ }
+
+ std::string
+ getName() const
+ {
+ return "NuDB";
+ }
+
+ std::unique_ptr
+ createInstance (
+ size_t keyBytes,
+ Parameters const& keyValues,
+ Scheduler& scheduler,
+ beast::Journal journal)
+ {
+ return std::make_unique (
+ keyBytes, keyValues, scheduler, journal);
+ }
+};
+
+static NuDBFactory nuDBFactory;
+
+}
+}
diff --git a/src/ripple/nodestore/tests/Backend.test.cpp b/src/ripple/nodestore/tests/Backend.test.cpp
index 6209712ce..900569d38 100644
--- a/src/ripple/nodestore/tests/Backend.test.cpp
+++ b/src/ripple/nodestore/tests/Backend.test.cpp
@@ -94,6 +94,8 @@ public:
{
int const seedValue = 50;
+ testBackend ("nudb", seedValue);
+
testBackend ("leveldb", seedValue);
#if RIPPLE_HYPERLEVELDB_AVAILABLE
diff --git a/src/ripple/nodestore/tests/Database.test.cpp b/src/ripple/nodestore/tests/Database.test.cpp
index bb7e94b72..33576d906 100644
--- a/src/ripple/nodestore/tests/Database.test.cpp
+++ b/src/ripple/nodestore/tests/Database.test.cpp
@@ -26,7 +26,7 @@
namespace ripple {
namespace NodeStore {
-class NodeStoreDatabase_test : public TestBase
+class Database_test : public TestBase
{
public:
void testImport (std::string const& destBackendType,
@@ -182,6 +182,8 @@ public:
void runBackendTests (bool useEphemeralDatabase, std::int64_t const seedValue)
{
+ testNodeStore ("nudb", useEphemeralDatabase, true, seedValue);
+
testNodeStore ("leveldb", useEphemeralDatabase, true, seedValue);
#if RIPPLE_HYPERLEVELDB_AVAILABLE
@@ -201,8 +203,10 @@ public:
void runImportTests (std::int64_t const seedValue)
{
- testImport ("leveldb", "leveldb", seedValue);
+ testImport ("nudb", "nudb", seedValue);
+ testImport ("leveldb", "leveldb", seedValue);
+
#if RIPPLE_HYPERLEVELDB_AVAILABLE
testImport ("hyperleveldb", "hyperleveldb", seedValue);
#endif
@@ -232,7 +236,7 @@ public:
}
};
-BEAST_DEFINE_TESTSUITE(NodeStoreDatabase,ripple_core,ripple);
+BEAST_DEFINE_TESTSUITE(Database,NodeStore,ripple);
}
}
diff --git a/src/ripple/nodestore/tests/Timing.test.cpp b/src/ripple/nodestore/tests/Timing.test.cpp
index 179187deb..644a13b2f 100644
--- a/src/ripple/nodestore/tests/Timing.test.cpp
+++ b/src/ripple/nodestore/tests/Timing.test.cpp
@@ -147,7 +147,7 @@ public:
missingNodePercent = 20
};
- std::size_t const default_repeat = 1;
+ std::size_t const default_repeat = 3;
#ifndef NDEBUG
std::size_t const default_items = 10000;
#else
@@ -708,10 +708,7 @@ public:
*/
std::string default_args =
- #ifdef _MSC_VER
- "type=leveldb"
- #endif
- //"type=nudb"
+ "type=nudb"
#if RIPPLE_ROCKSDB_AVAILABLE
";type=rocksdb,open_files=2000,filter_bits=12,cache_mb=256,"
"file_size_mb=8,file_size_mult=2"
diff --git a/src/ripple/unity/beast.cpp b/src/ripple/unity/beast.cpp
index d2aedbe47..046e41ab9 100644
--- a/src/ripple/unity/beast.cpp
+++ b/src/ripple/unity/beast.cpp
@@ -40,6 +40,7 @@
#include
#include
#include
+//#include
#include
#include
#include
diff --git a/src/ripple/unity/nodestore.cpp b/src/ripple/unity/nodestore.cpp
index f2cba0b8a..ee032dee3 100644
--- a/src/ripple/unity/nodestore.cpp
+++ b/src/ripple/unity/nodestore.cpp
@@ -19,9 +19,12 @@
#include
+#include
+
#include
#include
#include
+#include
#include
#include
@@ -38,5 +41,3 @@
#include
#include
#include
-
-