From 740115b65a6953c8f73c44c98cbf63eb69026c01 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Mon, 15 Jul 2013 08:35:58 -0700 Subject: [PATCH] Add HyperLevelDB backend --- Builds/QtCreator/rippled.pro | 1 + Builds/VisualStudio2012/RippleD.vcxproj | 2 + .../VisualStudio2012/RippleD.vcxproj.filters | 9 ++ .../ripple_HyperLevelDBBackendFactory.cpp | 152 ++++++++++++++++++ .../node/ripple_HyperLevelDBBackendFactory.h | 31 ++++ .../node/ripple_MdbBackendFactory.h | 4 + modules/ripple_app/node/ripple_NodeStore.h | 14 +- modules/ripple_app/ripple_app.cpp | 4 + modules/ripple_data/ripple_data.h | 3 - .../ripple_hyperleveldb.cpp | 4 +- .../ripple_hyperleveldb/ripple_hyperleveldb.h | 22 ++- src/cpp/ripple/ripple_Main.cpp | 3 + 12 files changed, 233 insertions(+), 16 deletions(-) create mode 100644 modules/ripple_app/node/ripple_HyperLevelDBBackendFactory.cpp create mode 100644 modules/ripple_app/node/ripple_HyperLevelDBBackendFactory.h diff --git a/Builds/QtCreator/rippled.pro b/Builds/QtCreator/rippled.pro index 5d46a2102..946376b4a 100644 --- a/Builds/QtCreator/rippled.pro +++ b/Builds/QtCreator/rippled.pro @@ -75,6 +75,7 @@ SOURCES += \ ../../modules/ripple_core/ripple_core.cpp \ ../../modules/ripple_client/ripple_client.cpp \ ../../modules/ripple_data/ripple_data.cpp \ + ../../modules/ripple_hyperleveldb/ripple_hyperleveldb.cpp \ ../../modules/ripple_json/ripple_json.cpp \ ../../modules/ripple_leveldb/ripple_leveldb.cpp \ ../../modules/ripple_mdb/ripple_mdb.c \ diff --git a/Builds/VisualStudio2012/RippleD.vcxproj b/Builds/VisualStudio2012/RippleD.vcxproj index 3a215b3d3..ba9ba1223 100644 --- a/Builds/VisualStudio2012/RippleD.vcxproj +++ b/Builds/VisualStudio2012/RippleD.vcxproj @@ -374,6 +374,7 @@ true + true true @@ -1418,6 +1419,7 @@ + diff --git a/Builds/VisualStudio2012/RippleD.vcxproj.filters b/Builds/VisualStudio2012/RippleD.vcxproj.filters index 8593b0116..1487bad87 100644 --- a/Builds/VisualStudio2012/RippleD.vcxproj.filters +++ b/Builds/VisualStudio2012/RippleD.vcxproj.filters @@ -151,6 +151,9 @@ {6a769530-8edf-4836-afc8-8836fe315603} + + {571acd5b-065c-4202-8de3-8692735171dc} + @@ -870,6 +873,9 @@ [1] Ripple\ripple_mdb + + [1] Ripple\ripple_hyperleveldb + @@ -1623,6 +1629,9 @@ [1] Ripple\ripple_mdb + + [1] Ripple\ripple_hyperleveldb + diff --git a/modules/ripple_app/node/ripple_HyperLevelDBBackendFactory.cpp b/modules/ripple_app/node/ripple_HyperLevelDBBackendFactory.cpp new file mode 100644 index 000000000..12a389237 --- /dev/null +++ b/modules/ripple_app/node/ripple_HyperLevelDBBackendFactory.cpp @@ -0,0 +1,152 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#if RIPPLE_HYPERLEVELDB_AVAILABLE + +class HyperLevelDBBackendFactory::Backend : public NodeStore::Backend +{ +public: + Backend (StringPairArray const& keyValues) + : mName(keyValues ["path"].toStdString ()) + , mDB(NULL) + { + if (mName.empty()) + throw std::runtime_error ("Missing path in LevelDB backend"); + + hyperleveldb::Options options; + options.create_if_missing = true; + + if (keyValues["cache_mb"].isEmpty()) + options.block_cache = hyperleveldb::NewLRUCache (theConfig.getSize (siHashNodeDBCache) * 1024 * 1024); + else + options.block_cache = hyperleveldb::NewLRUCache (keyValues["cache_mb"].getIntValue() * 1024L * 1024L); + + if (keyValues["filter_bits"].isEmpty()) + { + if (theConfig.NODE_SIZE >= 2) + options.filter_policy = hyperleveldb::NewBloomFilterPolicy (10); + } + else if (keyValues["filter_bits"].getIntValue() != 0) + options.filter_policy = hyperleveldb::NewBloomFilterPolicy (keyValues["filter_bits"].getIntValue()); + + if (!keyValues["open_files"].isEmpty()) + options.max_open_files = keyValues["open_files"].getIntValue(); + + hyperleveldb::Status status = hyperleveldb::DB::Open (options, mName, &mDB); + if (!status.ok () || !mDB) + throw (std::runtime_error (std::string("Unable to open/create leveldb: ") + status.ToString())); + } + + ~Backend () + { + delete mDB; + } + + std::string getDataBaseName() + { + return mName; + } + + bool bulkStore (const std::vector< NodeObject::pointer >& objs) + { + hyperleveldb::WriteBatch batch; + + BOOST_FOREACH (NodeObject::ref obj, objs) + { + Blob blob (toBlob (obj)); + batch.Put ( + hyperleveldb::Slice (reinterpret_cast(obj->getHash ().begin ()), 256 / 8), + hyperleveldb::Slice (reinterpret_cast(&blob.front ()), blob.size ())); + } + return mDB->Write (hyperleveldb::WriteOptions (), &batch).ok (); + } + + NodeObject::pointer retrieve (uint256 const& hash) + { + std::string sData; + if (!mDB->Get (hyperleveldb::ReadOptions (), + hyperleveldb::Slice (reinterpret_cast(hash.begin ()), 256 / 8), &sData).ok ()) + { + return NodeObject::pointer(); + } + return fromBinary(hash, &sData[0], sData.size ()); + } + + void visitAll (FUNCTION_TYPE func) + { + hyperleveldb::Iterator* it = mDB->NewIterator (hyperleveldb::ReadOptions ()); + for (it->SeekToFirst (); it->Valid (); it->Next ()) + { + if (it->key ().size () == 256 / 8) + { + uint256 hash; + memcpy(hash.begin(), it->key ().data(), 256 / 8); + func (fromBinary (hash, it->value ().data (), it->value ().size ())); + } + } + } + + Blob toBlob(NodeObject::ref obj) + { + Blob rawData (9 + obj->getData ().size ()); + unsigned char* bufPtr = &rawData.front(); + + *reinterpret_cast (bufPtr + 0) = ntohl (obj->getIndex ()); + *reinterpret_cast (bufPtr + 4) = ntohl (obj->getIndex ()); + * (bufPtr + 8) = static_cast (obj->getType ()); + memcpy (bufPtr + 9, &obj->getData ().front (), obj->getData ().size ()); + + return rawData; + } + + NodeObject::pointer fromBinary(uint256 const& hash, + char const* data, int size) + { + if (size < 9) + throw std::runtime_error ("undersized object"); + + uint32 index = htonl (*reinterpret_cast (data)); + int htype = data[8]; + + return boost::make_shared (static_cast (htype), index, + data + 9, size - 9, hash); + } + +private: + std::string mName; + hyperleveldb::DB* mDB; +}; + +//------------------------------------------------------------------------------ + +HyperLevelDBBackendFactory::HyperLevelDBBackendFactory () +{ +} + +HyperLevelDBBackendFactory::~HyperLevelDBBackendFactory () +{ +} + +HyperLevelDBBackendFactory& HyperLevelDBBackendFactory::getInstance () +{ + static HyperLevelDBBackendFactory instance; + + return instance; +} + +String HyperLevelDBBackendFactory::getName () const +{ + return "HyperLevelDB"; +} + +NodeStore::Backend* HyperLevelDBBackendFactory::createInstance (StringPairArray const& keyValues) +{ + return new HyperLevelDBBackendFactory::Backend (keyValues); +} + +//------------------------------------------------------------------------------ + +#endif diff --git a/modules/ripple_app/node/ripple_HyperLevelDBBackendFactory.h b/modules/ripple_app/node/ripple_HyperLevelDBBackendFactory.h new file mode 100644 index 000000000..1b44e4f9d --- /dev/null +++ b/modules/ripple_app/node/ripple_HyperLevelDBBackendFactory.h @@ -0,0 +1,31 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_HYPERLEVELDBBACKENDFACTORY_H_INCLUDED +#define RIPPLE_HYPERLEVELDBBACKENDFACTORY_H_INCLUDED + +#if RIPPLE_HYPERLEVELDB_AVAILABLE + +/** Factory to produce HyperLevelDB backends for the NodeStore. +*/ +class HyperLevelDBBackendFactory : public NodeStore::BackendFactory +{ +private: + class Backend; + + HyperLevelDBBackendFactory (); + ~HyperLevelDBBackendFactory (); + +public: + static HyperLevelDBBackendFactory& getInstance (); + + String getName () const; + NodeStore::Backend* createInstance (StringPairArray const& keyValues); +}; + +#endif + +#endif diff --git a/modules/ripple_app/node/ripple_MdbBackendFactory.h b/modules/ripple_app/node/ripple_MdbBackendFactory.h index 8b2a5b8eb..702ca3a14 100644 --- a/modules/ripple_app/node/ripple_MdbBackendFactory.h +++ b/modules/ripple_app/node/ripple_MdbBackendFactory.h @@ -7,6 +7,8 @@ #ifndef RIPPLE_MDBBACKENDFACTORY_H_INCLUDED #define RIPPLE_MDBBACKENDFACTORY_H_INCLUDED +#if RIPPLE_MDB_AVAILABLE + /** Factory to produce a backend using MDB. @note MDB is not currently available for Win32 @@ -27,3 +29,5 @@ public: }; #endif + +#endif diff --git a/modules/ripple_app/node/ripple_NodeStore.h b/modules/ripple_app/node/ripple_NodeStore.h index 564a85d04..8e6de63fc 100644 --- a/modules/ripple_app/node/ripple_NodeStore.h +++ b/modules/ripple_app/node/ripple_NodeStore.h @@ -9,7 +9,6 @@ /** Persistency layer for NodeObject */ -// VFALCO TODO Move all definitions to the .cpp class NodeStore : LeakChecked { public: @@ -18,12 +17,11 @@ public: class Backend { public: - // VFALCO TODO Remove this, the backend should not be a shared - // object it should be ScopedPointer owned. - // - typedef boost::shared_ptr pointer; - - Backend () : mWriteGeneration(0), mWriteLoad(0), mWritePending(false) + // VFALCO TODO Move the function definition to the .cpp + Backend () + : mWriteGeneration(0) + , mWriteLoad(0) + , mWritePending(false) { mWriteSet.reserve(128); } @@ -45,11 +43,13 @@ public: // This function will only be called during an import operation virtual void visitAll (FUNCTION_TYPE ) = 0; + // VFALCO TODO Put this bulk writing logic into a separate class. virtual void bulkWrite (Job &); virtual void waitWrite (); virtual int getWriteLoad (); protected: + // VFALCO TODO Put this bulk writing logic into a separate class. boost::mutex mWriteMutex; boost::condition_variable mWriteCondition; int mWriteGeneration; diff --git a/modules/ripple_app/ripple_app.cpp b/modules/ripple_app/ripple_app.cpp index 7ee1c95b7..5762a6b59 100644 --- a/modules/ripple_app/ripple_app.cpp +++ b/modules/ripple_app/ripple_app.cpp @@ -75,6 +75,8 @@ #include "../ripple_data/ripple_data.h" #include "../ripple_mdb/ripple_mdb.h" +#include "../ripple_leveldb/ripple_leveldb.h" +#include "../ripple_hyperleveldb/ripple_hyperleveldb.h" #include "../ripple_net/ripple_net.h" #include "../modules/ripple_websocket/ripple_websocket.h" @@ -97,6 +99,7 @@ namespace ripple #include "node/ripple_NodeObject.h" #include "node/ripple_NodeStore.h" #include "node/ripple_LevelDBBackendFactory.h" +#include "node/ripple_HyperLevelDBBackendFactory.h" #include "node/ripple_MdbBackendFactory.h" #include "node/ripple_NullBackendFactory.h" #include "node/ripple_SqliteBackendFactory.h" @@ -239,6 +242,7 @@ static const uint64 tenTo17m1 = tenTo17 - 1; #include "node/ripple_NodeObject.cpp" #include "node/ripple_NodeStore.cpp" #include "node/ripple_LevelDBBackendFactory.cpp" +#include "node/ripple_HyperLevelDBBackendFactory.cpp" #include "node/ripple_MdbBackendFactory.cpp" #include "node/ripple_NullBackendFactory.cpp" #include "node/ripple_SqliteBackendFactory.cpp" diff --git a/modules/ripple_data/ripple_data.h b/modules/ripple_data/ripple_data.h index 122663b2c..82fd48c93 100644 --- a/modules/ripple_data/ripple_data.h +++ b/modules/ripple_data/ripple_data.h @@ -23,9 +23,6 @@ // VFALCO TODO try to reduce these dependencies #include "../ripple_basics/ripple_basics.h" -// VFALCO TODO don't expose leveldb throughout the headers -#include "../ripple_leveldb/ripple_leveldb.h" - // VFALCO TODO figure out a good place for this file, perhaps give it some // additional hierarchy via directories. #include "ripple.pb.h" diff --git a/modules/ripple_hyperleveldb/ripple_hyperleveldb.cpp b/modules/ripple_hyperleveldb/ripple_hyperleveldb.cpp index 3ffe990c0..5b6cf4520 100644 --- a/modules/ripple_hyperleveldb/ripple_hyperleveldb.cpp +++ b/modules/ripple_hyperleveldb/ripple_hyperleveldb.cpp @@ -10,7 +10,7 @@ #include "ripple_hyperleveldb.h" -#include "beast/modules/beast_core/system/beast_TargetPlatform.h" +#if RIPPLE_HYPERLEVELDB_AVAILABLE // Set the appropriate LevelDB platform macro based on our platform. // @@ -93,3 +93,5 @@ #ifdef BEAST_MSVC #pragma warning (pop) #endif + +#endif diff --git a/modules/ripple_hyperleveldb/ripple_hyperleveldb.h b/modules/ripple_hyperleveldb/ripple_hyperleveldb.h index edadc4e14..10e9b2a5d 100644 --- a/modules/ripple_hyperleveldb/ripple_hyperleveldb.h +++ b/modules/ripple_hyperleveldb/ripple_hyperleveldb.h @@ -4,12 +4,24 @@ */ //============================================================================== -#ifndef RIPPLE_LEVELDB_RIPPLEHEADER -#define RIPPLE_LEVELDB_RIPPLEHEADER +#ifndef RIPPLE_HYPERLEVELDB_RIPPLEHEADER +#define RIPPLE_HYPERLEVELDB_RIPPLEHEADER -//#include "hyperleveldb/hyperleveldb/cache.h" -//#include "hyperleveldb/hyperleveldb/filter_policy.h" +#include "beast/modules/beast_core/system/beast_TargetPlatform.h" + +#if ! BEAST_WIN32 + +#define RIPPLE_HYPERLEVELDB_AVAILABLE 1 + +#include "hyperleveldb/hyperleveldb/cache.h" +#include "hyperleveldb/hyperleveldb/filter_policy.h" #include "hyperleveldb/hyperleveldb/db.h" -//#include "hyperleveldb/hyperleveldb/write_batch.h" +#include "hyperleveldb/hyperleveldb/write_batch.h" + +#else + +#define RIPPLE_HYPERLEVELDB_AVAILABLE 0 + +#endif #endif diff --git a/src/cpp/ripple/ripple_Main.cpp b/src/cpp/ripple/ripple_Main.cpp index b8700bb3c..b9783c553 100644 --- a/src/cpp/ripple/ripple_Main.cpp +++ b/src/cpp/ripple/ripple_Main.cpp @@ -184,6 +184,9 @@ int rippleMain (int argc, char** argv) // These must be added before the Application object is created NodeStore::addBackendFactory (SqliteBackendFactory::getInstance ()); NodeStore::addBackendFactory (LevelDBBackendFactory::getInstance ()); +#if RIPPLE_HYPERLEVELDB_AVAILABLE + NodeStore::addBackendFactory (HyperLevelDBBackendFactory::getInstance ()); +#endif #if RIPPLE_MDB_AVAILABLE NodeStore::addBackendFactory (MdbBackendFactory::getInstance ()); #endif