Add KeyvaDB, Backend, and unit test

.gitignore

@@ -63,3 +63,7 @@ DerivedData
 
 # Intel Parallel Studio 2013 XE
 My Amplifier XE Results - RippleD
+
+# KeyvaDB files
+*.key
+*.val

Builds/VisualStudio2012/RippleD.vcxproj

@@ -157,6 +157,18 @@
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
     </ClCompile>
+    <ClCompile Include="..\..\modules\ripple_app\node\ripple_KeyvaDB.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
+    </ClCompile>
+    <ClCompile Include="..\..\modules\ripple_app\node\ripple_KeyvaDBBackendFactory.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
+    </ClCompile>
     <ClCompile Include="..\..\modules\ripple_app\node\ripple_NodeObject.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
@@ -1402,6 +1414,8 @@
     <ClInclude Include="..\..\modules\ripple_app\ledger\ripple_LedgerHistory.h" />
     <ClInclude Include="..\..\modules\ripple_app\ledger\SerializedValidation.h" />
     <ClInclude Include="..\..\modules\ripple_app\node\ripple_HyperLevelDBBackendFactory.h" />
+    <ClInclude Include="..\..\modules\ripple_app\node\ripple_KeyvaDB.h" />
+    <ClInclude Include="..\..\modules\ripple_app\node\ripple_KeyvaDBBackendFactory.h" />
     <ClInclude Include="..\..\modules\ripple_app\node\ripple_NodeObject.h" />
     <ClInclude Include="..\..\modules\ripple_app\node\ripple_NodeStore.h" />
     <ClInclude Include="..\..\modules\ripple_app\node\ripple_LevelDBBackendFactory.h" />
@@ -1732,7 +1746,7 @@
       <PrecompiledHeader>
       </PrecompiledHeader>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>_CRTDBG_MAP_ALLOC;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
       <MinimalRebuild>false</MinimalRebuild>
       <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>

Builds/VisualStudio2012/RippleD.vcxproj.filters

@@ -897,6 +897,12 @@
     <ClCompile Include="..\..\modules\ripple_app\node\ripple_HyperLevelDBBackendFactory.cpp">
       <Filter>[1] Ripple\ripple_app\node</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\modules\ripple_app\node\ripple_KeyvaDB.cpp">
+      <Filter>[1] Ripple\ripple_app\node</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\modules\ripple_app\node\ripple_KeyvaDBBackendFactory.cpp">
+      <Filter>[1] Ripple\ripple_app\node</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\Subtrees\sqlite\sqlite3.h">
@@ -1674,6 +1680,12 @@
     <ClInclude Include="..\..\modules\ripple_app\node\ripple_HyperLevelDBBackendFactory.h">
       <Filter>[1] Ripple\ripple_app\node</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\modules\ripple_app\node\ripple_KeyvaDB.h">
+      <Filter>[1] Ripple\ripple_app\node</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\modules\ripple_app\node\ripple_KeyvaDBBackendFactory.h">
+      <Filter>[1] Ripple\ripple_app\node</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <CustomBuild Include="..\..\src\cpp\ripple\ripple.proto" />

TODO.txt

@@ -2,6 +2,17 @@
 RIPPLE TODO
 --------------------------------------------------------------------------------
 
+Vinnie's Short List (Changes day to day)
+- Convert some Ripple boost unit tests to Beast.
+- Eliminate new technical in NodeStore::Backend
+- Improve NodeObject to construct with just a size.
+- Work on KeyvaDB
+- Finish unit tests and code for Validators
+
+--------------------------------------------------------------------------------
+
+- Rewrite boost program_options in Beast
+
 - Examples for different backend key/value config settings
 
 - Unit Test attention
@@ -10,8 +21,6 @@ RIPPLE TODO
 
 - Validations unit test
 
---------------------------------------------------------------------------------
-
 - Replace endian conversion calls with beast calls:
   htobe32, be32toh, ntohl, etc...
   Start by removing the system headers which provide these routines, if possible

modules/ripple_app/node/ripple_KeyvaDB.cpp

@@ -0,0 +1,616 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+class KeyvaDBImp : public KeyvaDB
+{
+private:
+    // These are stored in big endian format in the file.
+
+    // A file offset.
+    typedef int64 FileOffset;
+
+    // Index of a key.
+    typedef int32 KeyIndex;
+
+    // Size of a value.
+    typedef int32 ByteSize;
+
+private:
+    enum
+    {
+        // The size of the fixed area at the beginning of the key file.
+        // This is used to store some housekeeping information like the
+        // key size and version number.
+        //
+        keyFileHeaderBytes = 1024
+    };
+
+    // Accessed by multiple threads
+    struct State
+    {
+        ScopedPointer <FileInputStream> keyIn;
+        ScopedPointer <FileOutputStream> keyOut;
+        KeyIndex newKeyIndex;
+
+        ScopedPointer <FileInputStream> valIn;
+        ScopedPointer <FileOutputStream> valOut;
+        FileOffset valFileSize;
+
+        bool hasKeys () const noexcept
+        {
+            return newKeyIndex > 1;
+        }
+    };
+
+    typedef SharedData <State> SharedState;
+
+    // Key records are indexed starting at one.
+    struct KeyRecord
+    {
+        explicit KeyRecord (void* const keyStorage)
+            : key (keyStorage)
+        {
+        }
+
+        // Absolute byte FileOffset in the value file.
+        FileOffset valFileOffset;
+
+        // Size of the corresponding value, in bytes.
+        ByteSize valSize;
+
+        // Key record index of left node, or 0.
+        KeyIndex leftIndex;
+
+        // Key record index of right node, or 0.
+        KeyIndex rightIndex;
+
+        // Points to keyBytes storage of the key.
+        void* const key;
+    };
+
+public:
+    KeyvaDBImp (int keyBytes,
+                File keyPath,
+                File valPath,
+                bool filesAreTemporary)
+        : m_keyBytes (keyBytes)
+        , m_keyRecordBytes (getKeyRecordBytes ())
+        , m_filesAreTemporary (filesAreTemporary)
+        , m_keyStorage (keyBytes)
+    {
+        SharedState::WriteAccess state (m_state);
+
+        // Output must be opened first, in case it has
+        // to created, or else opening for input will fail.
+        state->keyOut = openForWrite (keyPath);
+        state->keyIn = openForRead (keyPath);
+
+        int64 const fileSize = state->keyIn->getFile ().getSize ();
+
+        if (fileSize == 0)
+        {
+            // initialize the key file
+            state->keyOut->setPosition (keyFileHeaderBytes - 1);
+            state->keyOut->writeByte (0);
+            state->keyOut->flush ();
+        }
+
+        state->newKeyIndex = 1 + (state->keyIn->getFile ().getSize () - keyFileHeaderBytes) / m_keyRecordBytes;
+
+        state->valOut = openForWrite (valPath);
+        state->valIn = openForRead (valPath);
+        state->valFileSize = state->valIn->getFile ().getSize ();
+    }
+
+    ~KeyvaDBImp ()
+    {
+        SharedState::WriteAccess state (m_state);
+
+        flushInternal (state);
+
+        state->keyOut = nullptr;
+        state->valOut = nullptr;
+
+        // Delete the database files if requested.
+        //
+        if (m_filesAreTemporary)
+        {
+            {
+                File const path = state->keyIn->getFile ();
+                state->keyIn = nullptr;
+                path.deleteFile ();
+            }
+
+            {
+                File const path = state->valIn->getFile ();
+                state->valIn = nullptr;
+                path.deleteFile ();
+            }
+        }
+    }
+
+    //--------------------------------------------------------------------------
+
+    // Returns the number of physical bytes in a key record.
+    // This is specific to the format of the data.
+    //
+    int getKeyRecordBytes () const noexcept
+    {
+        int bytes = 0;
+
+        bytes += sizeof (FileOffset);       // valFileOffset
+        bytes += sizeof (ByteSize);         // valSize
+        bytes += sizeof (KeyIndex);         // leftIndex
+        bytes += sizeof (KeyIndex);         // rightIndex
+
+        bytes += m_keyBytes;
+
+        return bytes;
+    }
+
+    FileOffset calcKeyRecordOffset (KeyIndex keyIndex)
+    {
+        bassert (keyIndex > 0);
+
+        FileOffset const byteOffset = keyFileHeaderBytes + (keyIndex - 1) * m_keyRecordBytes;
+
+        return byteOffset;
+    }
+
+    // Read a key record into memory.
+    void readKeyRecord (KeyRecord* const keyRecord,
+                        KeyIndex const keyIndex,
+                        SharedState::WriteAccess& state)
+    {
+        FileOffset const byteOffset = calcKeyRecordOffset (keyIndex);
+
+        bool const success = state->keyIn->setPosition (byteOffset);
+
+        if (success)
+        {
+            // This defines the file format!
+            keyRecord->valFileOffset = state->keyIn->readInt64BigEndian ();
+            keyRecord->valSize = state->keyIn->readIntBigEndian ();
+            keyRecord->leftIndex = state->keyIn->readIntBigEndian ();
+            keyRecord->rightIndex = state->keyIn->readIntBigEndian ();
+
+            // Grab the key
+            state->keyIn->read (keyRecord->key, m_keyBytes);
+        }
+        else
+        {
+            String s;
+            s << "KeyvaDB: Seek failed in " << state->valOut->getFile ().getFileName ();
+            Throw (std::runtime_error (s.toStdString ()));
+        }
+    }
+
+    // Write a key record from memory
+    void writeKeyRecord (KeyRecord const& keyRecord,
+                         KeyIndex const keyIndex,
+                         SharedState::WriteAccess& state,
+                         bool includingKey)
+    {
+        FileOffset const byteOffset = calcKeyRecordOffset (keyIndex);
+
+        bool const success = state->keyOut->setPosition (byteOffset);
+
+        if (success)
+        {
+            // This defines the file format!
+            // VFALCO TODO Make OutputStream return the bool errors here
+            //
+            state->keyOut->writeInt64BigEndian (keyRecord.valFileOffset);
+            state->keyOut->writeIntBigEndian (keyRecord.valSize);
+            state->keyOut->writeIntBigEndian (keyRecord.leftIndex);
+            state->keyOut->writeIntBigEndian (keyRecord.rightIndex);
+
+            // Write the key
+            if (includingKey)
+            {
+                bool const success = state->keyOut->write (keyRecord.key, m_keyBytes);
+
+                if (! success)
+                {
+                    String s;
+                    s << "KeyvaDB: Write failed in " << state->valOut->getFile ().getFileName ();
+                    Throw (std::runtime_error (s.toStdString ()));
+                }
+            }
+
+            state->keyOut->flush ();
+        }
+        else
+        {
+            String s;
+            s << "KeyvaDB: Seek failed in " << state->valOut->getFile ().getFileName ();
+            Throw (std::runtime_error (s.toStdString ()));
+        }
+    }
+
+    // Append a value to the value file.
+    void writeValue (void const* const value, ByteSize valueBytes, SharedState::WriteAccess& state)
+    {
+        bool const success = state->valOut->setPosition (state->valFileSize);
+
+        if (success)
+        {
+            bool const success = state->valOut->write (value, static_cast <size_t> (valueBytes));
+
+            if (! success)
+            {
+                String s;
+                s << "KeyvaDB: Write failed in " << state->valOut->getFile ().getFileName ();
+                Throw (std::runtime_error (s.toStdString ()));
+            }
+
+            state->valFileSize += valueBytes;
+
+            state->valOut->flush ();
+        }
+        else
+        {
+            String s;
+            s << "KeyvaDB: Seek failed in " << state->valOut->getFile ().getFileName ();
+            Throw (std::runtime_error (s.toStdString ()));
+        }
+    }
+
+    //--------------------------------------------------------------------------
+
+    struct FindResult
+    {
+        FindResult (void* const keyStorage)
+            : keyRecord (keyStorage)
+        {
+        }
+
+        int compare;         // result of the last comparison
+        KeyIndex keyIndex;   // index we looked at last
+        KeyRecord keyRecord; // KeyRecord we looked at last
+    };
+
+    // Find a key. If the key doesn't exist, enough information
+    // is left behind in the result to perform an insertion.
+    //
+    // Returns true if the key was found.
+    //
+    bool find (FindResult* findResult, void const* key, SharedState::WriteAccess& state)
+    {
+        // Not okay to call this with an empty key file!
+        bassert (state->hasKeys ());
+
+        // This performs a standard binary search
+
+        findResult->keyIndex = 1;
+
+        do
+        {
+            readKeyRecord (&findResult->keyRecord, findResult->keyIndex, state);
+
+            findResult->compare = memcmp (key, findResult->keyRecord.key, m_keyBytes);
+
+            if (findResult->compare < 0)
+            {
+                if (findResult->keyRecord.leftIndex != 0)
+                {
+                    // Go left
+                    findResult->keyIndex = findResult->keyRecord.leftIndex;
+                }
+                else
+                {
+                    // Insert position is to the left
+                    break;
+                }
+            }
+            else if (findResult->compare > 0)
+            {
+                if (findResult->keyRecord.rightIndex != 0)
+                {
+                    // Go right
+                    findResult->keyIndex = findResult->keyRecord.rightIndex;
+                }
+                else
+                {
+                    // Insert position is to the right
+                    break;
+                }
+            }
+        }
+        while (findResult->compare != 0);
+
+        return findResult->compare == 0;
+    }
+
+    //--------------------------------------------------------------------------
+
+    bool get (void const* key, GetCallback* callback)
+    {
+        FindResult findResult (m_keyStorage.getData ());
+
+        SharedState::WriteAccess state (m_state);
+
+        bool found = false;
+
+        if (state->hasKeys ())
+        {
+            found = find (&findResult, key, state);
+
+            if (found)
+            {
+                void* const destStorage = callback->createStorageForValue (findResult.keyRecord.valSize);
+
+                bool const success = state->valIn->setPosition (findResult.keyRecord.valFileOffset);
+
+                if (! success)
+                {
+                    String s;
+                    s << "KeyvaDB: Seek failed in " << state->valOut->getFile ().getFileName ();
+                    Throw (std::runtime_error (s.toStdString ()));
+                }
+
+                int const bytesRead = state->valIn->read (destStorage, findResult.keyRecord.valSize);
+
+                if (bytesRead != findResult.keyRecord.valSize)
+                {
+                    String s;
+                    s << "KeyvaDB: Couldn't read a value from " << state->valIn->getFile ().getFileName ();
+                    Throw (std::runtime_error (s.toStdString ()));
+                }
+            }
+        }
+
+        return found;
+    }
+
+    //--------------------------------------------------------------------------
+
+    void put (void const* key, void const* value, int valueBytes)
+    {
+        bassert (valueBytes > 0);
+
+        SharedState::WriteAccess state (m_state);
+
+        if (state->hasKeys ())
+        {
+            // Search for the key
+
+            FindResult findResult (m_keyStorage.getData ());
+
+            bool const found = find (&findResult, key, state);
+
+            if (! found )
+            {
+                bassert (findResult.compare != 0);
+
+                // Binary tree insertion.
+                // Link the last key record to the new key
+                {
+                    if (findResult.compare == -1)
+                    {
+                        findResult.keyRecord.leftIndex = state->newKeyIndex;
+                    }
+                    else
+                    {
+                        findResult.keyRecord.rightIndex = state->newKeyIndex;
+                    }
+
+                    writeKeyRecord (findResult.keyRecord, findResult.keyIndex, state, false);
+                }
+
+                // Write the new key
+                {
+                    findResult.keyRecord.valFileOffset = state->valFileSize;
+                    findResult.keyRecord.valSize = valueBytes;
+                    findResult.keyRecord.leftIndex = 0;
+                    findResult.keyRecord.rightIndex = 0;
+
+                    memcpy (findResult.keyRecord.key, key, m_keyBytes);
+
+                    writeKeyRecord (findResult.keyRecord, state->newKeyIndex, state, true);
+                }
+
+                // Key file has grown by one.
+                ++state->newKeyIndex;
+
+                // Write the value
+                writeValue (value, valueBytes, state);
+            }
+            else
+            {
+                String s;
+                s << "KeyvaDB: Attempt to write a duplicate key!";
+                Throw (std::runtime_error (s.toStdString ()));
+            }
+        }
+        else
+        {
+            //
+            // Write first key
+            //
+
+            KeyRecord keyRecord (m_keyStorage.getData ());
+
+            keyRecord.valFileOffset = state->valFileSize;
+            keyRecord.valSize = valueBytes;
+            keyRecord.leftIndex = 0;
+            keyRecord.rightIndex = 0;
+            
+            memcpy (keyRecord.key, key, m_keyBytes);
+            
+            writeKeyRecord (keyRecord, state->newKeyIndex, state, true);
+
+            // Key file has grown by one.
+            ++state->newKeyIndex;
+
+            //
+            // Write value
+            //
+
+            bassert (state->valFileSize == 0);
+
+            writeValue (value, valueBytes, state);
+        }
+    }
+
+    //--------------------------------------------------------------------------
+
+    void flush ()
+    {
+        SharedState::WriteAccess state (m_state);
+
+        flushInternal (state);
+    }
+
+    void flushInternal (SharedState::WriteAccess& state)
+    {
+        state->keyOut->flush ();
+        state->valOut->flush ();
+    }
+
+    //--------------------------------------------------------------------------
+
+private:
+    // Open a file for reading.
+    static FileInputStream* openForRead (File path)
+    {
+        FileInputStream* stream = path.createInputStream ();
+
+        if (stream == nullptr)
+        {
+            String s;
+            s << "KeyvaDB: Couldn't open " << path.getFileName () << " for reading.";
+            Throw (std::runtime_error (s.toStdString ()));
+        }
+
+        return stream;
+    }
+
+    // Open a file for writing.
+    static FileOutputStream* openForWrite (File path)
+    {
+        FileOutputStream* stream = path.createOutputStream ();
+
+        if (stream == nullptr)
+        {
+            String s;
+            s << "KeyvaDB: Couldn't open " << path.getFileName () << " for writing.";
+            Throw (std::runtime_error (s.toStdString ()));
+        }
+
+        return stream;
+    }
+
+private:
+    int const m_keyBytes;
+    int const m_keyRecordBytes;
+    bool const m_filesAreTemporary;
+    SharedState m_state;
+    HeapBlock <char> m_keyStorage;
+};
+
+KeyvaDB* KeyvaDB::New (int keyBytes, File keyPath, File valPath, bool filesAreTemporary)
+{
+    return new KeyvaDBImp (keyBytes, keyPath, valPath, filesAreTemporary);
+}
+
+//------------------------------------------------------------------------------
+
+class KeyvaDBTests : public UnitTest
+{
+public:
+    KeyvaDBTests () : UnitTest ("KevyaDB")
+    {
+    }
+
+    template <class T>
+    void repeatableShuffle (int const numberOfItems, HeapBlock <T>& items)
+    {
+        Random r (69);
+
+        for (int i = numberOfItems - 1; i > 0; --i)
+        {
+            int const choice = r.nextInt (i + 1);
+
+            std::swap (items [i], items [choice]);
+        }
+    }
+
+    template <unsigned int KeyBytes>
+    void testSize (unsigned int const maxItems)
+    {
+        typedef UnsignedInteger <KeyBytes> KeyType;
+
+        String s;
+        s << "keyBytes=" << String (KeyBytes);
+        beginTest (s);
+
+        // Set up the key and value files and open the db.
+        File const keyPath = File::createTempFile ("").withFileExtension (".key");
+        File const valPath = File::createTempFile ("").withFileExtension (".val");
+        ScopedPointer <KeyvaDB> db (KeyvaDB::New (KeyBytes, keyPath, valPath, true));
+
+        {
+            // Create an array of ascending integers.
+            HeapBlock <unsigned int> items (maxItems);
+            for (unsigned int i = 0; i < maxItems; ++i)
+                items [i] = i;
+
+            // Now shuffle it deterministically.
+            repeatableShuffle (maxItems, items);
+
+            // Write all the keys of integers.
+            for (unsigned int i = 0; i < maxItems; ++i)
+            {
+                unsigned int const num = items [i];
+                KeyType const v = KeyType::createFromInteger (num);
+
+                // The value is the same as the key, for ease of comparison.
+                db->put (v.cbegin (), v.cbegin (), KeyBytes);
+            }
+        }
+
+        {
+            // This callback creates storage for the value.
+            struct MyGetCallback : KeyvaDB::GetCallback
+            {
+                KeyType v;
+
+                void* createStorageForValue (int valueBytes)
+                {
+                    bassert (valueBytes == KeyBytes);
+
+                    return v.begin ();
+                }
+            };
+
+            // Go through all of our keys and try to retrieve them.
+            // since this is done in ascending order, we should get
+            // random seeks at this point.
+            //
+            for (unsigned int i = 0; i < maxItems; ++i)
+            {
+                KeyType const v = KeyType::createFromInteger (i);
+
+                MyGetCallback cb;
+
+                bool const found = db->get (v.cbegin (), &cb);
+
+                expect (found, "Should be found");
+
+                expect (v == cb.v, "Should be equal");
+            }
+        }
+    }
+
+    void runTest ()
+    {
+        testSize <4> (512);
+        testSize <32> (4096);
+    }
+};
+
+static KeyvaDBTests keyvaDBTests;

modules/ripple_app/node/ripple_KeyvaDB.h

@@ -0,0 +1,35 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_KEYVADB_H_INCLUDED
+#define RIPPLE_KEYVADB_H_INCLUDED
+
+/** Key/value database optimized for Ripple usage.
+*/
+class KeyvaDB : LeakChecked <KeyvaDB>
+{
+public:
+    class GetCallback
+    {
+    public:
+        virtual void* createStorageForValue (int valueBytes) = 0;
+    };
+
+    static KeyvaDB* New (int keyBytes,
+                         File keyPath,
+                         File valPath,
+                         bool filesAreTemporary);
+
+    virtual ~KeyvaDB () { }
+
+    virtual bool get (void const* key, GetCallback* callback) = 0;
+
+    virtual void put (void const* key, void const* value, int valueBytes) = 0;
+
+    virtual void flush () = 0;
+};
+
+#endif

modules/ripple_app/node/ripple_KeyvaDBBackendFactory.cpp

@@ -0,0 +1,149 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+class KeyvaDBBackendFactory::Backend : public NodeStore::Backend
+{
+public:
+    typedef UnsignedInteger <32> Key;
+
+    enum
+    {
+        keyBytes = Key::sizeInBytes
+    };
+
+    explicit Backend (StringPairArray const& keyValues)
+        : m_path (keyValues ["path"])
+        , m_db (KeyvaDB::New (
+                    keyBytes,
+                    File::getCurrentWorkingDirectory().getChildFile (m_path).withFileExtension ("key"),
+                    File::getCurrentWorkingDirectory().getChildFile (m_path).withFileExtension ("val"),
+                    false))
+    {
+    }
+
+    ~Backend ()
+    {
+    }
+
+    std::string getDataBaseName ()
+    {
+        return m_path.toStdString ();
+    }
+
+    void writeObject (NodeObject const& object)
+    {
+        m_db->put (
+            object.getHash ().begin (),
+            &object.getData () [0],
+            object.getData ().size ());
+    }
+
+    bool bulkStore (std::vector <NodeObject::pointer> const& objs)
+    {
+        for (size_t i = 0; i < objs.size (); ++i)
+        {
+            writeObject (*objs [i]);
+        }
+
+        return true;
+    }
+
+    struct MyGetCallback : KeyvaDB::GetCallback
+    {
+        int valueBytes;
+        HeapBlock <char> data;
+
+        void* createStorageForValue (int valueBytes_)
+        {
+            valueBytes = valueBytes_;
+
+            data.malloc (valueBytes);
+
+            return data.getData ();
+        }
+    };
+
+    NodeObject::pointer retrieve (uint256 const& hash)
+    {
+        NodeObject::pointer result;
+
+        MyGetCallback cb;
+
+        bool const found = m_db->get (hash.begin (), &cb);
+
+        if (found)
+        {
+            result = fromBinary (hash, cb.data.getData (), cb.valueBytes);
+        }
+
+        return result;
+    }
+
+    void visitAll (FUNCTION_TYPE<void (NodeObject::pointer)> func)
+    {
+        bassertfalse;
+    }
+
+    Blob toBlob (NodeObject::ref obj)
+    {
+        Blob rawData (9 + obj->getData ().size ());
+        unsigned char* bufPtr = &rawData.front();
+
+        *reinterpret_cast<uint32*> (bufPtr + 0) = ntohl (obj->getIndex ());
+        *reinterpret_cast<uint32*> (bufPtr + 4) = ntohl (obj->getIndex ());
+        * (bufPtr + 8) = static_cast<unsigned char> (obj->getType ());
+        memcpy (bufPtr + 9, &obj->getData ().front (), obj->getData ().size ());
+
+        return rawData;
+    }
+
+    NodeObject::pointer fromBinary (uint256 const& hash, char const* data, int size)
+    {
+        if (size < 9)
+            throw std::runtime_error ("undersized object");
+
+        uint32 index = htonl (*reinterpret_cast <const uint32*> (data));
+        
+        int htype = data[8];
+
+        return boost::make_shared <NodeObject> (static_cast<NodeObjectType> (htype), index,
+            data + 9, size - 9, hash);
+    }
+
+private:
+    String m_path;
+    ScopedPointer <KeyvaDB> m_db;
+};
+
+//------------------------------------------------------------------------------
+
+KeyvaDBBackendFactory::KeyvaDBBackendFactory ()
+{
+}
+
+KeyvaDBBackendFactory::~KeyvaDBBackendFactory ()
+{
+}
+
+KeyvaDBBackendFactory& KeyvaDBBackendFactory::getInstance ()
+{
+    static KeyvaDBBackendFactory instance;
+
+    return instance;
+}
+
+String KeyvaDBBackendFactory::getName () const
+{
+    return "KeyvaDB";
+}
+
+NodeStore::Backend* KeyvaDBBackendFactory::createInstance (StringPairArray const& keyValues)
+{
+    return new KeyvaDBBackendFactory::Backend (keyValues);
+}
+
+//------------------------------------------------------------------------------
+

modules/ripple_app/node/ripple_KeyvaDBBackendFactory.h

@@ -0,0 +1,27 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_KEYVABACKENDFACTORY_H_INCLUDED
+#define RIPPLE_KEYVABACKENDFACTORY_H_INCLUDED
+
+/** Factory to produce KeyvaDB backends for the NodeStore.
+*/
+class KeyvaDBBackendFactory : public NodeStore::BackendFactory
+{
+private:
+    class Backend;
+
+    KeyvaDBBackendFactory ();
+    ~KeyvaDBBackendFactory ();
+
+public:
+    static KeyvaDBBackendFactory& getInstance ();
+
+    String getName () const;
+    NodeStore::Backend* createInstance (StringPairArray const& keyValues);
+};
+
+#endif

modules/ripple_app/ripple_app.cpp

@@ -104,6 +104,7 @@ namespace ripple
 #include "node/ripple_NodeStore.h"
 #include "node/ripple_LevelDBBackendFactory.h"
 #include "node/ripple_HyperLevelDBBackendFactory.h"
+#include "node/ripple_KeyvaDBBackendFactory.h"
 #include "node/ripple_MdbBackendFactory.h"
 #include "node/ripple_NullBackendFactory.h"
 #include "node/ripple_SqliteBackendFactory.h"
@@ -249,6 +250,9 @@ static const uint64 tenTo17m1 = tenTo17 - 1;
 #include "node/ripple_MdbBackendFactory.cpp"
 #include "node/ripple_NullBackendFactory.cpp"
 #include "node/ripple_SqliteBackendFactory.cpp"
+#include "node/ripple_KeyvaDB.h" // private
+#include "node/ripple_KeyvaDB.cpp"
+#include "node/ripple_KeyvaDBBackendFactory.cpp"
 
 #include "ledger/Ledger.cpp"
 #include "src/cpp/ripple/ripple_SHAMapDelta.cpp"

src/cpp/ripple/ripple_Main.cpp

@@ -155,7 +155,7 @@ static void runBeastUnitTests ()
     {
         UnitTests::TestResult const& r (*tr.getResult (i));
                 
-        for (int j = 0; j < r.messages.size (); ++i)
+        for (int j = 0; j < r.messages.size (); ++j)
             Log::out () << r.messages [j].toStdString ();
     }
 }
@@ -252,6 +252,7 @@ int rippleMain (int argc, char** argv)
     // These must be added before the Application object is created
     NodeStore::addBackendFactory (SqliteBackendFactory::getInstance ());
     NodeStore::addBackendFactory (LevelDBBackendFactory::getInstance ());
+    NodeStore::addBackendFactory (KeyvaDBBackendFactory::getInstance ());
 #if RIPPLE_HYPERLEVELDB_AVAILABLE
     NodeStore::addBackendFactory (HyperLevelDBBackendFactory::getInstance ());
 #endif