Add HyperLevelDB backend

Builds/QtCreator/rippled.pro

@@ -75,6 +75,7 @@ SOURCES += \
     ../../modules/ripple_core/ripple_core.cpp \
     ../../modules/ripple_client/ripple_client.cpp \
     ../../modules/ripple_data/ripple_data.cpp \
+    ../../modules/ripple_hyperleveldb/ripple_hyperleveldb.cpp \
     ../../modules/ripple_json/ripple_json.cpp \
     ../../modules/ripple_leveldb/ripple_leveldb.cpp \
     ../../modules/ripple_mdb/ripple_mdb.c \

Builds/VisualStudio2012/RippleD.vcxproj

@@ -374,6 +374,7 @@
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
     </ClCompile>
     <ClCompile Include="..\..\modules\ripple_data\ripple_data.cpp" />
+    <ClCompile Include="..\..\modules\ripple_hyperleveldb\ripple_hyperleveldb.cpp" />
     <ClCompile Include="..\..\modules\ripple_json\json\json_reader.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
@@ -1418,6 +1419,7 @@
     <ClInclude Include="..\..\modules\ripple_data\protocol\ripple_TxFormats.h" />
     <ClInclude Include="..\..\modules\ripple_data\ripple_data.h" />
     <ClInclude Include="..\..\modules\ripple_data\utility\ripple_UptimeTimerAdapter.h" />
+    <ClInclude Include="..\..\modules\ripple_hyperleveldb\ripple_hyperleveldb.h" />
     <ClInclude Include="..\..\modules\ripple_json\json\json_autolink.h" />
     <ClInclude Include="..\..\modules\ripple_json\json\json_batchallocator.h" />
     <ClInclude Include="..\..\modules\ripple_json\json\json_config.h" />

Builds/VisualStudio2012/RippleD.vcxproj.filters

@@ -151,6 +151,9 @@
     <Filter Include="[1] Ripple\ripple_mdb">
       <UniqueIdentifier>{6a769530-8edf-4836-afc8-8836fe315603}</UniqueIdentifier>
     </Filter>
+    <Filter Include="[1] Ripple\ripple_hyperleveldb">
+      <UniqueIdentifier>{571acd5b-065c-4202-8de3-8692735171dc}</UniqueIdentifier>
+    </Filter>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\..\Subtrees\sqlite\sqlite3.c">
@@ -870,6 +873,9 @@
     <ClCompile Include="..\..\modules\ripple_mdb\ripple_mdb.c">
       <Filter>[1] Ripple\ripple_mdb</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\modules\ripple_hyperleveldb\ripple_hyperleveldb.cpp">
+      <Filter>[1] Ripple\ripple_hyperleveldb</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\Subtrees\sqlite\sqlite3.h">
@@ -1623,6 +1629,9 @@
     <ClInclude Include="..\..\modules\ripple_mdb\ripple_mdb.h">
       <Filter>[1] Ripple\ripple_mdb</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\modules\ripple_hyperleveldb\ripple_hyperleveldb.h">
+      <Filter>[1] Ripple\ripple_hyperleveldb</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <CustomBuild Include="..\..\src\cpp\ripple\ripple.proto" />

modules/ripple_app/node/ripple_HyperLevelDBBackendFactory.cpp

@@ -0,0 +1,152 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#if RIPPLE_HYPERLEVELDB_AVAILABLE
+
+class HyperLevelDBBackendFactory::Backend : public NodeStore::Backend
+{
+public:
+    Backend (StringPairArray const& keyValues)
+        : mName(keyValues ["path"].toStdString ())
+        , mDB(NULL)
+    {
+        if (mName.empty())
+            throw std::runtime_error ("Missing path in LevelDB backend");
+
+        hyperleveldb::Options options;
+        options.create_if_missing = true;
+
+        if (keyValues["cache_mb"].isEmpty())
+            options.block_cache = hyperleveldb::NewLRUCache (theConfig.getSize (siHashNodeDBCache) * 1024 * 1024);
+        else
+            options.block_cache = hyperleveldb::NewLRUCache (keyValues["cache_mb"].getIntValue() * 1024L * 1024L);
+
+        if (keyValues["filter_bits"].isEmpty())
+        {
+            if (theConfig.NODE_SIZE >= 2)
+                options.filter_policy = hyperleveldb::NewBloomFilterPolicy (10);
+        }
+        else if (keyValues["filter_bits"].getIntValue() != 0)
+            options.filter_policy = hyperleveldb::NewBloomFilterPolicy (keyValues["filter_bits"].getIntValue());
+
+        if (!keyValues["open_files"].isEmpty())
+            options.max_open_files = keyValues["open_files"].getIntValue();
+
+        hyperleveldb::Status status = hyperleveldb::DB::Open (options, mName, &mDB);
+        if (!status.ok () || !mDB)
+            throw (std::runtime_error (std::string("Unable to open/create leveldb: ") + status.ToString()));
+    }
+
+    ~Backend ()
+    {
+        delete mDB;
+    }
+
+    std::string getDataBaseName()
+    {
+        return mName;
+    }
+
+    bool bulkStore (const std::vector< NodeObject::pointer >& objs)
+    {
+        hyperleveldb::WriteBatch batch;
+
+        BOOST_FOREACH (NodeObject::ref obj, objs)
+        {
+            Blob blob (toBlob (obj));
+            batch.Put (
+                hyperleveldb::Slice (reinterpret_cast<char const*>(obj->getHash ().begin ()), 256 / 8),
+                hyperleveldb::Slice (reinterpret_cast<char const*>(&blob.front ()), blob.size ()));
+        }
+        return mDB->Write (hyperleveldb::WriteOptions (), &batch).ok ();
+    }
+
+    NodeObject::pointer retrieve (uint256 const& hash)
+    {
+        std::string sData;
+        if (!mDB->Get (hyperleveldb::ReadOptions (),
+            hyperleveldb::Slice (reinterpret_cast<char const*>(hash.begin ()), 256 / 8), &sData).ok ())
+        {
+            return NodeObject::pointer();
+        }
+        return fromBinary(hash, &sData[0], sData.size ());
+    }
+
+    void visitAll (FUNCTION_TYPE<void (NodeObject::pointer)> func)
+    {
+        hyperleveldb::Iterator* it = mDB->NewIterator (hyperleveldb::ReadOptions ());
+        for (it->SeekToFirst (); it->Valid (); it->Next ())
+        {
+            if (it->key ().size () == 256 / 8)
+            {
+                uint256 hash;
+                memcpy(hash.begin(), it->key ().data(), 256 / 8);
+                func (fromBinary (hash, it->value ().data (), it->value ().size ()));
+            }
+        }
+    }
+
+    Blob toBlob(NodeObject::ref obj)
+    {
+        Blob rawData (9 + obj->getData ().size ());
+        unsigned char* bufPtr = &rawData.front();
+
+        *reinterpret_cast<uint32*> (bufPtr + 0) = ntohl (obj->getIndex ());
+        *reinterpret_cast<uint32*> (bufPtr + 4) = ntohl (obj->getIndex ());
+        * (bufPtr + 8) = static_cast<unsigned char> (obj->getType ());
+        memcpy (bufPtr + 9, &obj->getData ().front (), obj->getData ().size ());
+
+        return rawData;
+    }
+
+    NodeObject::pointer fromBinary(uint256 const& hash,
+        char const* data, int size)
+    {
+        if (size < 9)
+            throw std::runtime_error ("undersized object");
+
+        uint32 index = htonl (*reinterpret_cast<const uint32*> (data));
+        int htype = data[8];
+
+        return boost::make_shared<NodeObject> (static_cast<NodeObjectType> (htype), index,
+            data + 9, size - 9, hash);
+    }
+
+private:
+    std::string mName;
+    hyperleveldb::DB* mDB;
+};
+
+//------------------------------------------------------------------------------
+
+HyperLevelDBBackendFactory::HyperLevelDBBackendFactory ()
+{
+}
+
+HyperLevelDBBackendFactory::~HyperLevelDBBackendFactory ()
+{
+}
+
+HyperLevelDBBackendFactory& HyperLevelDBBackendFactory::getInstance ()
+{
+    static HyperLevelDBBackendFactory instance;
+
+    return instance;
+}
+
+String HyperLevelDBBackendFactory::getName () const
+{
+    return "HyperLevelDB";
+}
+
+NodeStore::Backend* HyperLevelDBBackendFactory::createInstance (StringPairArray const& keyValues)
+{
+    return new HyperLevelDBBackendFactory::Backend (keyValues);
+}
+
+//------------------------------------------------------------------------------
+
+#endif

modules/ripple_app/node/ripple_HyperLevelDBBackendFactory.h

@@ -0,0 +1,31 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_HYPERLEVELDBBACKENDFACTORY_H_INCLUDED
+#define RIPPLE_HYPERLEVELDBBACKENDFACTORY_H_INCLUDED
+
+#if RIPPLE_HYPERLEVELDB_AVAILABLE
+
+/** Factory to produce HyperLevelDB backends for the NodeStore.
+*/
+class HyperLevelDBBackendFactory : public NodeStore::BackendFactory
+{
+private:
+    class Backend;
+
+    HyperLevelDBBackendFactory ();
+    ~HyperLevelDBBackendFactory ();
+
+public:
+    static HyperLevelDBBackendFactory& getInstance ();
+
+    String getName () const;
+    NodeStore::Backend* createInstance (StringPairArray const& keyValues);
+};
+
+#endif
+
+#endif

modules/ripple_app/node/ripple_MdbBackendFactory.h

@@ -7,6 +7,8 @@
 #ifndef RIPPLE_MDBBACKENDFACTORY_H_INCLUDED
 #define RIPPLE_MDBBACKENDFACTORY_H_INCLUDED
 
+#if RIPPLE_MDB_AVAILABLE
+
 /** Factory to produce a backend using MDB.
 
     @note MDB is not currently available for Win32
@@ -27,3 +29,5 @@ public:
 };
 
 #endif
+
+#endif

modules/ripple_app/node/ripple_NodeStore.h

@@ -9,7 +9,6 @@
 
 /** Persistency layer for NodeObject
 */
-// VFALCO TODO Move all definitions to the .cpp
 class NodeStore : LeakChecked <NodeStore>
 {
 public:
@@ -18,12 +17,11 @@ public:
     class Backend
     {
     public:
-        // VFALCO TODO Remove this, the backend should not be a shared
+        // VFALCO TODO Move the function definition to the .cpp
-        //             object it should be ScopedPointer owned.
+        Backend ()
-        //
+            : mWriteGeneration(0)
-        typedef boost::shared_ptr <Backend> pointer;
+            , mWriteLoad(0)
-
+            , mWritePending(false)
-        Backend () : mWriteGeneration(0), mWriteLoad(0), mWritePending(false)
         {
             mWriteSet.reserve(128);
         }
@@ -45,11 +43,13 @@ public:
         // This function will only be called during an import operation
         virtual void visitAll (FUNCTION_TYPE <void (NodeObject::pointer)>) = 0;
 
+        // VFALCO TODO Put this bulk writing logic into a separate class.
         virtual void bulkWrite (Job &);
         virtual void waitWrite ();
         virtual int getWriteLoad ();
 
     protected:
+        // VFALCO TODO Put this bulk writing logic into a separate class.
         boost::mutex                                 mWriteMutex;
         boost::condition_variable                    mWriteCondition;
         int                                          mWriteGeneration;

modules/ripple_app/ripple_app.cpp

@@ -75,6 +75,8 @@
 
 #include "../ripple_data/ripple_data.h"
 #include "../ripple_mdb/ripple_mdb.h"
+#include "../ripple_leveldb/ripple_leveldb.h"
+#include "../ripple_hyperleveldb/ripple_hyperleveldb.h"
 #include "../ripple_net/ripple_net.h"
 
 #include "../modules/ripple_websocket/ripple_websocket.h"
@@ -97,6 +99,7 @@ namespace ripple
 #include "node/ripple_NodeObject.h"
 #include "node/ripple_NodeStore.h"
 #include "node/ripple_LevelDBBackendFactory.h"
+#include "node/ripple_HyperLevelDBBackendFactory.h"
 #include "node/ripple_MdbBackendFactory.h"
 #include "node/ripple_NullBackendFactory.h"
 #include "node/ripple_SqliteBackendFactory.h"
@@ -239,6 +242,7 @@ static const uint64 tenTo17m1 = tenTo17 - 1;
 #include "node/ripple_NodeObject.cpp"
 #include "node/ripple_NodeStore.cpp"
 #include "node/ripple_LevelDBBackendFactory.cpp"
+#include "node/ripple_HyperLevelDBBackendFactory.cpp"
 #include "node/ripple_MdbBackendFactory.cpp"
 #include "node/ripple_NullBackendFactory.cpp"
 #include "node/ripple_SqliteBackendFactory.cpp"

modules/ripple_data/ripple_data.h

@@ -23,9 +23,6 @@
 // VFALCO TODO try to reduce these dependencies
 #include "../ripple_basics/ripple_basics.h"
 
-// VFALCO TODO don't expose leveldb throughout the headers
-#include "../ripple_leveldb/ripple_leveldb.h"
-
 // VFALCO TODO figure out a good place for this file, perhaps give it some
 //         additional hierarchy via directories.
 #include "ripple.pb.h"

modules/ripple_hyperleveldb/ripple_hyperleveldb.cpp

@@ -10,7 +10,7 @@
 
 #include "ripple_hyperleveldb.h"
 
-#include "beast/modules/beast_core/system/beast_TargetPlatform.h"
+#if RIPPLE_HYPERLEVELDB_AVAILABLE
 
 // Set the appropriate LevelDB platform macro based on our platform.
 //
@@ -93,3 +93,5 @@
 #ifdef BEAST_MSVC
 #pragma warning (pop)
 #endif
+
+#endif

modules/ripple_hyperleveldb/ripple_hyperleveldb.h

@@ -4,12 +4,24 @@
 */
 //==============================================================================
 
-#ifndef RIPPLE_LEVELDB_RIPPLEHEADER
+#ifndef RIPPLE_HYPERLEVELDB_RIPPLEHEADER
-#define RIPPLE_LEVELDB_RIPPLEHEADER
+#define RIPPLE_HYPERLEVELDB_RIPPLEHEADER
 
-//#include "hyperleveldb/hyperleveldb/cache.h"
+#include "beast/modules/beast_core/system/beast_TargetPlatform.h"
-//#include "hyperleveldb/hyperleveldb/filter_policy.h"
+
+#if ! BEAST_WIN32
+
+#define RIPPLE_HYPERLEVELDB_AVAILABLE 1
+
+#include "hyperleveldb/hyperleveldb/cache.h"
+#include "hyperleveldb/hyperleveldb/filter_policy.h"
 #include "hyperleveldb/hyperleveldb/db.h"
-//#include "hyperleveldb/hyperleveldb/write_batch.h"
+#include "hyperleveldb/hyperleveldb/write_batch.h"
+
+#else
+
+#define RIPPLE_HYPERLEVELDB_AVAILABLE 0
+
+#endif
 
 #endif

src/cpp/ripple/ripple_Main.cpp

@@ -184,6 +184,9 @@ int rippleMain (int argc, char** argv)
     // These must be added before the Application object is created
     NodeStore::addBackendFactory (SqliteBackendFactory::getInstance ());
     NodeStore::addBackendFactory (LevelDBBackendFactory::getInstance ());
+#if RIPPLE_HYPERLEVELDB_AVAILABLE
+    NodeStore::addBackendFactory (HyperLevelDBBackendFactory::getInstance ());
+#endif
 #if RIPPLE_MDB_AVAILABLE
     NodeStore::addBackendFactory (MdbBackendFactory::getInstance ());
 #endif