Bring in backends from the dead branch

modules/ripple_app/node/ripple_HSBELevelDB.cpp

@@ -0,0 +1,97 @@
+
+HSBELevelDB::HSBELevelDB(std::string const& path) : mName(path), mDB(NULL)
+{
+    leveldb::Options options;
+    options.create_if_missing = true;
+    options.block_cache = leveldb::NewLRUCache (theConfig.getSize (siHashNodeDBCache) * 1024 * 1024);
+
+    if (theConfig.NODE_SIZE >= 2)
+        options.filter_policy = leveldb::NewBloomFilterPolicy (10);
+
+    leveldb::Status status = leveldb::DB::Open (options, path, &mDB);
+    if (!status.ok () || mDB)
+        throw (std::runtime_error (std::string("Unable to open/create leveldb: ") + status.ToString()));
+}
+
+HSBELevelDB::~HSBELevelDB()
+{
+    delete mDB;
+}
+
+std::string HSBELevelDB::getDataBaseName()
+{
+    return mName;
+}
+
+bool HSBELevelDB::store(HashedObject::ref obj)
+{
+    Blob blob (toBlob (obj));
+    return mDB->Put (leveldb::WriteOptions (),
+        leveldb::Slice (reinterpret_cast<char const*>(obj->getHash ().begin ()), 256 / 8),
+        leveldb::Slice (reinterpret_cast<char const*>(&blob.front ()), blob.size ())).ok ();
+}
+
+bool HSBELevelDB::bulkStore(const std::vector< HashedObject::pointer >& objs)
+{
+    leveldb::WriteBatch batch;
+
+    BOOST_FOREACH (HashedObject::ref obj, objs)
+    {
+        Blob blob (toBlob (obj));
+        batch.Put (
+            leveldb::Slice (reinterpret_cast<char const*>(obj->getHash ().begin ()), 256 / 8),
+            leveldb::Slice (reinterpret_cast<char const*>(&blob.front ()), blob.size ()));
+    }
+    return mDB->Write (leveldb::WriteOptions (), &batch).ok ();
+}
+
+HashedObject::pointer HSBELevelDB::retrieve(uint256 const& hash)
+{
+    std::string sData;
+    if (!mDB->Get (leveldb::ReadOptions (),
+        leveldb::Slice (reinterpret_cast<char const*>(hash.begin ()), 256 / 8), &sData).ok ())
+    {
+        return HashedObject::pointer();
+    }
+    return fromBinary(hash, &sData[0], sData.size ());
+}
+
+void HSBELevelDB::visitAll(FUNCTION_TYPE<void (HashedObject::pointer)> func)
+{
+    leveldb::Iterator* it = mDB->NewIterator (leveldb::ReadOptions ());
+    for (it->SeekToFirst (); it->Valid (); it->Next ())
+    {
+        if (it->key ().size () == 256 / 8)
+        {
+            uint256 hash;
+            memcpy(hash.begin(), it->key ().data(), 256 / 8);
+            func (fromBinary (hash, it->value ().data (), it->value ().size ()));
+        }
+    }
+}
+
+Blob HSBELevelDB::toBlob(HashedObject::ref obj)
+{
+    Blob rawData (9 + obj->getData ().size ());
+    unsigned char* bufPtr = &rawData.front();
+
+    *reinterpret_cast<uint32*> (bufPtr + 0) = ntohl (obj->getIndex ());
+    *reinterpret_cast<uint32*> (bufPtr + 4) = ntohl (obj->getIndex ());
+    * (bufPtr + 8) = static_cast<unsigned char> (obj->getType ());
+    memcpy (bufPtr + 9, &obj->getData ().front (), obj->getData ().size ());
+
+    return rawData;
+}
+
+HashedObject::pointer HSBELevelDB::fromBinary(uint256 const& hash,
+    char const* data, int size)
+{
+    if (size < 9)
+        throw std::runtime_error ("undersized object");
+
+    uint32 index = htonl (*reinterpret_cast<const uint32*> (data));
+    int htype = data[8];
+
+    return boost::make_shared<HashedObject> (static_cast<HashedObjectType> (htype), index,
+        data + 9, size - 9, hash);
+}

modules/ripple_app/node/ripple_HSBELevelDB.h

@@ -0,0 +1,32 @@
+#ifndef HSBELEVELDB_H
+#define HSBELEVELDB_H
+
+class HSBELevelDB : public HashStoreBE
+{
+public:
+    HSBELevelDB(std::string const& path);
+    ~HSBELevelDB();
+
+    std::string getBackEndName()
+    {
+        return "LevelDB";
+    }
+
+    std::string getDataBaseName();
+
+    bool store(HashedObject::ref);
+    bool bulkStore(const std::vector< HashedObject::pointer >&);
+
+    HashedObject::pointer retrieve(uint256 const& hash);
+
+    void visitAll(FUNCTION_TYPE<void (HashedObject::pointer)>);
+
+private:
+    std::string      mName;
+    leveldb::DB*     mDB;
+
+    Blob toBlob(HashedObject::ref);
+    HashedObject::pointer fromBinary(uint256 const& hash, char const* data, int size);
+};
+
+#endif

modules/ripple_app/node/ripple_HSBESqlite.cpp

@@ -0,0 +1,125 @@
+
+HSBESQLite::HSBESQLite(std::string const& path) : mName(path)
+{
+    mDb = new DatabaseCon(path, HashNodeDBInit, HashNodeDBCount);
+    mDb->getDB()->executeSQL(boost::str(boost::format("PRAGMA cache_size=-%d;") %
+        (theConfig.getSize(siHashNodeDBCache) * 1024)));
+}
+
+HSBESQLite::~HSBESQLite()
+{
+    delete mDb;
+}
+
+std::string HSBESQLite::getDataBaseName()
+{
+    return mName;
+}
+
+bool HSBESQLite::store(HashedObject::ref object)
+{
+    ScopedLock sl(mDb->getDBLock());
+    static SqliteStatement pSt(mDb->getDB()->getSqliteDB(),
+        "INSERT OR IGNORE INTO CommittedObjects "
+            "(Hash,ObjType,LedgerIndex,Object) VALUES (?, ?, ?, ?);");
+    bind(pSt, object);
+    pSt.step();
+    pSt.reset();
+    return true;
+}
+
+bool HSBESQLite::bulkStore(const std::vector< HashedObject::pointer >& objects)
+{
+    ScopedLock sl(mDb->getDBLock());
+    static SqliteStatement pStB(mDb->getDB()->getSqliteDB(), "BEGIN TRANSACTION;");
+    static SqliteStatement pStE(mDb->getDB()->getSqliteDB(), "END TRANSACTION;");
+    static SqliteStatement pSt(mDb->getDB()->getSqliteDB(),
+        "INSERT OR IGNORE INTO CommittedObjects "
+            "(Hash,ObjType,LedgerIndex,Object) VALUES (?, ?, ?, ?);");
+
+    pStB.step();
+    pStB.reset();
+
+    BOOST_FOREACH(HashedObject::ref object, objects)
+    {
+        bind(pSt, object);
+        pSt.step();
+        pSt.reset();
+    }
+
+    pStE.step();
+    pStE.reset();
+
+    return true;
+
+}
+
+HashedObject::pointer HSBESQLite::retrieve(uint256 const& hash)
+{
+    HashedObject::pointer ret;
+
+    {
+        ScopedLock sl(mDb->getDBLock());
+        static SqliteStatement pSt(mDb->getDB()->getSqliteDB(),
+            "SELECT ObjType,LedgerIndex,Object FROM CommittedObjects WHERE Hash = ?;");
+
+        pSt.bind(1, hash.GetHex());
+
+        if (pSt.isRow(pSt.step()))
+            ret = boost::make_shared<HashedObject>(getType(pSt.peekString(0)), pSt.getUInt32(1), pSt.getBlob(2), hash);
+
+        pSt.reset();
+    }
+
+    return ret;
+}
+
+void HSBESQLite::visitAll(FUNCTION_TYPE<void (HashedObject::pointer)> func)
+{
+    uint256 hash;
+
+    static SqliteStatement pSt(mDb->getDB()->getSqliteDB(),
+        "SELECT ObjType,LedgerIndex,Object,Hash FROM CommittedObjects;");
+
+    while (pSt.isRow(pSt.step()))
+    {
+        hash.SetHexExact(pSt.getString(3));
+        func(boost::make_shared<HashedObject>(getType(pSt.peekString(0)), pSt.getUInt32(1), pSt.getBlob(2), hash));
+    }
+
+    pSt.reset();
+}
+
+void HSBESQLite::bind(SqliteStatement& statement, HashedObject::ref object)
+{
+    char const* type;
+    switch (object->getType())
+    {
+        case hotLEDGER:                type = "L"; break;
+        case hotTRANSACTION:        type = "T"; break;
+        case hotACCOUNT_NODE:        type = "A"; break;
+        case hotTRANSACTION_NODE:    type = "N"; break;
+        default:                    type = "U";
+    }
+
+    statement.bind(1, object->getHash().GetHex());
+    statement.bind(2, type);
+    statement.bind(3, object->getIndex());
+    statement.bindStatic(4, object->getData());
+}
+
+HashedObjectType HSBESQLite::getType(std::string const& type)
+{
+    HashedObjectType htype = hotUNKNOWN;
+    if (!type.empty())
+    {
+        switch (type[0])
+        {
+            case 'L': htype = hotLEDGER; break;
+            case 'T': htype = hotTRANSACTION; break;
+            case 'A': htype = hotACCOUNT_NODE; break;
+            case 'N': htype = hotTRANSACTION_NODE; break;
+        }
+    }
+    return htype;
+}

modules/ripple_app/node/ripple_HSBESqlite.h

@@ -0,0 +1,32 @@
+#ifndef HSBESQLITE_H
+#define HSBESQLITE_H
+
+class HSBESQLite : public HashStoreBE
+{
+public:
+    HSBESQLite(std::string const& path);
+    ~HSBESQLite();
+
+    std::string getBackEndName()
+    {
+        return "SQLite";
+    }
+
+    std::string getDataBaseName();
+
+    bool store(HashedObject::ref);
+    bool bulkStore(const std::vector< HashedObject::pointer >&);
+
+    HashedObject::pointer retrieve(uint256 const& hash);
+
+    void visitAll(FUNCTION_TYPE<void (HashedObject::pointer)>);
+
+private:
+    std::string      mName;
+    DatabaseCon*     mDb;
+
+    void bind(SqliteStatement& statement, HashedObject::ref object);
+    HashedObjectType getType(std::string const&);
+};
+
+#endif

modules/ripple_app/node/ripple_HashStoreBE.h

@@ -0,0 +1,31 @@
+#ifndef RIPPLE_HASHSTOREBE_H
+#define RIPPLE_HASHSTOREBE_H
+
+/** Back end for storing objects indexed by 256-bit hash
+*/
+class HashStoreBE
+{
+public:
+    typedef boost::shared_ptr<HashStoreBE> pointer;
+
+    HashStoreBE()                { ; }
+    virtual ~HashStoreBE()       { ; }
+
+    virtual std::string getBackEndName() = 0;
+    virtual std::string getDataBaseName() = 0;
+
+    // Store/retrieve a single object
+    // These functions must be thread safe
+    virtual bool store(HashedObject::ref) = 0;
+    virtual HashedObject::pointer retrieve(uint256 const &hash) = 0;
+
+    // Store a group of objects
+    // This function will only be called from a single thread
+    virtual bool bulkStore(const std::vector< HashedObject::pointer >&) = 0;
+
+    // Visit every object in the database
+    // This function will only be called during an import operation
+    virtual void visitAll(FUNCTION_TYPE<void (HashedObject::pointer)>) = 0;
+};
+
+#endif

modules/ripple_app/node/ripple_HashedObject.cpp

@@ -0,0 +1,53 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+SETUP_LOG (HashedObject)
+
+HashedObject::HashedObject (
+    HashedObjectType type,
+    LedgerIndex ledgerIndex,
+    Blob const& binaryDataToCopy,
+    uint256 const& hash)
+    : mType (type)
+    , mHash (hash)
+    , mLedgerIndex (ledgerIndex)
+    , mData (binaryDataToCopy)
+{
+}
+
+HashedObject::HashedObject (
+    HashedObjectType type,
+    LedgerIndex ledgerIndex,
+    void const* bufferToCopy,
+    int bytesInBuffer,
+    uint256 const& hash)
+    : mType (type)
+    , mHash (hash)
+    , mLedgerIndex (ledgerIndex)
+    , mData (static_cast <unsigned char const*> (bufferToCopy),
+             static_cast <unsigned char const*> (bufferToCopy) + bytesInBuffer)
+{
+}
+
+HashedObjectType HashedObject::getType () const
+{
+    return mType;
+}
+
+uint256 const& HashedObject::getHash () const
+{
+    return mHash;
+}
+
+LedgerIndex HashedObject::getIndex () const
+{
+    return mLedgerIndex;
+}
+
+Blob const& HashedObject::getData () const
+{
+    return mData;
+}

modules/ripple_app/node/ripple_HashedObject.h

@@ -0,0 +1,88 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_HASHEDOBJECT_H
+#define RIPPLE_HASHEDOBJECT_H
+
+/** The types of hashed objects.
+*/
+enum HashedObjectType
+{
+    hotUNKNOWN = 0,
+    hotLEDGER = 1,
+    hotTRANSACTION = 2,
+    hotACCOUNT_NODE = 3,
+    hotTRANSACTION_NODE = 4
+};
+
+/** A blob of data with associated metadata, referenced by hash.
+
+    The metadata includes the following:
+
+    - Type of the blob
+    - The ledger index in which it appears
+    - The SHA 256 hash
+
+    @note No checking is performed to make sure the hash matches the data.
+    @see SHAMap
+*/
+// VFALCO TODO consider making the instance a private member of SHAMap
+//         since its the primary user.
+//
+class HashedObject
+    : public CountedObject <HashedObject>
+{
+public:
+    static char const* getCountedObjectName () { return "HashedObject"; }
+
+    typedef boost::shared_ptr <HashedObject> pointer;
+    typedef pointer const& ref;
+
+    /** Create from a vector of data.
+
+        @note A copy of the data is created.
+    */
+    HashedObject (HashedObjectType type,
+                  LedgerIndex ledgerIndex,
+                  Blob const & binaryDataToCopy,
+                  uint256 const & hash);
+
+    /** Create from an area of memory.
+
+        @note A copy of the data is created.
+    */
+    HashedObject (HashedObjectType type,
+                  LedgerIndex ledgerIndex,
+                  void const * bufferToCopy,
+                  int bytesInBuffer,
+                  uint256 const & hash);
+
+    /** Retrieve the type of this object.
+    */
+    HashedObjectType getType () const;
+
+    /** Retrieve the hash metadata.
+    */
+    uint256 const& getHash () const;
+
+    /** Retrieve the ledger index in which this object appears.
+    */
+    // VFALCO TODO rename to getLedgerIndex or getLedgerId
+    LedgerIndex getIndex () const;
+
+    /** Retrieve the binary data.
+    */
+    Blob const& getData () const;
+
+private:
+    HashedObjectType const mType;
+    uint256 const mHash;
+    LedgerIndex const mLedgerIndex;
+    Blob const mData;
+};
+
+#endif
+// vim:ts=4

modules/ripple_app/node/ripple_HashedObjectStore.cpp

@@ -0,0 +1,621 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+HashedObjectStore::HashedObjectStore (int cacheSize, int cacheAge) :
+    mCache ("HashedObjectStore", cacheSize, cacheAge), mNegativeCache ("HashedObjectNegativeCache", 0, 120),
+    mWriteGeneration (0), mWriteLoad (0), mWritePending (false), mLevelDB (false), mEphemeralDB (false)
+{
+    mWriteSet.reserve (128);
+
+    if (theConfig.NODE_DB == "leveldb" || theConfig.NODE_DB == "LevelDB")
+        mLevelDB = true;
+    else if (theConfig.NODE_DB == "SQLite" || theConfig.NODE_DB == "sqlite")
+        mLevelDB = false;
+    else
+    {
+        WriteLog (lsFATAL, HashedObject) << "Incorrect database selection";
+        assert (false);
+    }
+
+    if (!theConfig.LDB_EPHEMERAL.empty ())
+        mEphemeralDB = true;
+}
+
+void HashedObjectStore::tune (int size, int age)
+{
+    mCache.setTargetSize (size);
+    mCache.setTargetAge (age);
+}
+
+void HashedObjectStore::waitWrite ()
+{
+    boost::mutex::scoped_lock sl (mWriteMutex);
+    int gen = mWriteGeneration;
+
+    while (mWritePending && (mWriteGeneration == gen))
+        mWriteCondition.wait (sl);
+}
+
+int HashedObjectStore::getWriteLoad ()
+{
+    boost::mutex::scoped_lock sl (mWriteMutex);
+    return std::max (mWriteLoad, static_cast<int> (mWriteSet.size ()));
+}
+
+// low-level retrieve
+HashedObject::pointer HashedObjectStore::LLRetrieve (uint256 const& hash, leveldb::DB* db)
+{
+    std::string sData;
+
+    leveldb::Status st = db->Get (leveldb::ReadOptions (),
+                                  leveldb::Slice (reinterpret_cast<const char*> (hash.begin ()), hash.size ()), &sData);
+
+    if (!st.ok ())
+    {
+        assert (st.IsNotFound ());
+        return HashedObject::pointer ();
+    }
+
+    const unsigned char* bufPtr = reinterpret_cast<const unsigned char*> (&sData[0]);
+    uint32 index = htonl (*reinterpret_cast<const uint32*> (bufPtr));
+    int htype = bufPtr[8];
+
+    return boost::make_shared<HashedObject> (static_cast<HashedObjectType> (htype), index,
+            bufPtr + 9, sData.size () - 9, hash);
+}
+
+// low-level write single
+void HashedObjectStore::LLWrite (boost::shared_ptr<HashedObject> ptr, leveldb::DB* db)
+{
+    HashedObject& obj = *ptr;
+    Blob rawData (9 + obj.getData ().size ());
+    unsigned char* bufPtr = &rawData.front ();
+
+    *reinterpret_cast<uint32*> (bufPtr + 0) = ntohl (obj.getIndex ());
+    *reinterpret_cast<uint32*> (bufPtr + 4) = ntohl (obj.getIndex ());
+    * (bufPtr + 8) = static_cast<unsigned char> (obj.getType ());
+    memcpy (bufPtr + 9, &obj.getData ().front (), obj.getData ().size ());
+
+    leveldb::Status st = db->Put (leveldb::WriteOptions (),
+                                  leveldb::Slice (reinterpret_cast<const char*> (obj.getHash ().begin ()), obj.getHash ().size ()),
+                                  leveldb::Slice (reinterpret_cast<const char*> (bufPtr), rawData.size ()));
+
+    if (!st.ok ())
+    {
+        WriteLog (lsFATAL, HashedObject) << "Failed to store hash node";
+        assert (false);
+    }
+}
+
+// low-level write set
+void HashedObjectStore::LLWrite (const std::vector< boost::shared_ptr<HashedObject> >& set, leveldb::DB* db)
+{
+    leveldb::WriteBatch batch;
+
+    BOOST_FOREACH (const boost::shared_ptr<HashedObject>& it, set)
+    {
+        const HashedObject& obj = *it;
+        Blob rawData (9 + obj.getData ().size ());
+        unsigned char* bufPtr = &rawData.front ();
+
+        *reinterpret_cast<uint32*> (bufPtr + 0) = ntohl (obj.getIndex ());
+        *reinterpret_cast<uint32*> (bufPtr + 4) = ntohl (obj.getIndex ());
+        * (bufPtr + 8) = static_cast<unsigned char> (obj.getType ());
+        memcpy (bufPtr + 9, &obj.getData ().front (), obj.getData ().size ());
+
+        batch.Put (leveldb::Slice (reinterpret_cast<const char*> (obj.getHash ().begin ()), obj.getHash ().size ()),
+                   leveldb::Slice (reinterpret_cast<const char*> (bufPtr), rawData.size ()));
+    }
+
+    leveldb::Status st = db->Write (leveldb::WriteOptions (), &batch);
+
+    if (!st.ok ())
+    {
+        WriteLog (lsFATAL, HashedObject) << "Failed to store hash node";
+        assert (false);
+    }
+}
+
+bool HashedObjectStore::storeLevelDB (HashedObjectType type, uint32 index,
+                                      Blob const& data, uint256 const& hash)
+{
+    // return: false = already in cache, true = added to cache
+    if (!getApp().getHashNodeLDB ())
+        return true;
+
+    if (mCache.touch (hash))
+        return false;
+
+#ifdef PARANOID
+    assert (hash == Serializer::getSHA512Half (data));
+#endif
+
+    HashedObject::pointer object = boost::make_shared<HashedObject> (type, index, data, hash);
+
+    if (!mCache.canonicalize (hash, object))
+    {
+        boost::mutex::scoped_lock sl (mWriteMutex);
+        mWriteSet.push_back (object);
+
+        if (!mWritePending)
+        {
+            mWritePending = true;
+            getApp().getJobQueue ().addJob (jtWRITE, "HashedObject::store",
+                                           BIND_TYPE (&HashedObjectStore::bulkWriteLevelDB, this, P_1));
+        }
+    }
+
+    mNegativeCache.del (hash);
+    return true;
+}
+
+void HashedObjectStore::bulkWriteLevelDB (Job&)
+{
+    assert (mLevelDB);
+    int setSize = 0;
+
+    while (1)
+    {
+        std::vector< boost::shared_ptr<HashedObject> > set;
+        set.reserve (128);
+
+        {
+            boost::mutex::scoped_lock sl (mWriteMutex);
+
+            mWriteSet.swap (set);
+            assert (mWriteSet.empty ());
+            ++mWriteGeneration;
+            mWriteCondition.notify_all ();
+
+            if (set.empty ())
+            {
+                mWritePending = false;
+                mWriteLoad = 0;
+                return;
+            }
+
+            mWriteLoad = std::max (setSize, static_cast<int> (mWriteSet.size ()));
+            setSize = set.size ();
+        }
+
+        LLWrite (set, getApp().getHashNodeLDB ());
+
+        if (mEphemeralDB)
+            LLWrite (set, getApp().getEphemeralLDB ());
+    }
+}
+
+HashedObject::pointer HashedObjectStore::retrieveLevelDB (uint256 const& hash)
+{
+    HashedObject::pointer obj = mCache.fetch (hash);
+
+    if (obj || mNegativeCache.isPresent (hash) || !getApp().getHashNodeLDB ())
+        return obj;
+
+    if (mEphemeralDB)
+    {
+        obj = LLRetrieve (hash, getApp().getEphemeralLDB ());
+
+        if (obj)
+        {
+            mCache.canonicalize (hash, obj);
+            return obj;
+        }
+    }
+
+    {
+        LoadEvent::autoptr event (getApp().getJobQueue ().getLoadEventAP (jtHO_READ, "HOS::retrieve"));
+        obj = LLRetrieve (hash, getApp().getHashNodeLDB ());
+
+        if (!obj)
+        {
+            mNegativeCache.add (hash);
+            return obj;
+        }
+    }
+
+    mCache.canonicalize (hash, obj);
+
+    if (mEphemeralDB)
+        LLWrite (obj, getApp().getEphemeralLDB ());
+
+    WriteLog (lsTRACE, HashedObject) << "HOS: " << hash << " fetch: in db";
+    return obj;
+}
+
+bool HashedObjectStore::storeSQLite (HashedObjectType type, uint32 index,
+                                     Blob const& data, uint256 const& hash)
+{
+    // return: false = already in cache, true = added to cache
+    if (!getApp().getHashNodeDB ())
+    {
+        WriteLog (lsTRACE, HashedObject) << "HOS: no db";
+        return true;
+    }
+
+    if (mCache.touch (hash))
+    {
+        WriteLog (lsTRACE, HashedObject) << "HOS: " << hash << " store: incache";
+        return false;
+    }
+
+    assert (hash == Serializer::getSHA512Half (data));
+
+    HashedObject::pointer object = boost::make_shared<HashedObject> (type, index, data, hash);
+
+    if (!mCache.canonicalize (hash, object))
+    {
+        //      WriteLog (lsTRACE, HashedObject) << "Queuing write for " << hash;
+        boost::mutex::scoped_lock sl (mWriteMutex);
+        mWriteSet.push_back (object);
+
+        if (!mWritePending)
+        {
+            mWritePending = true;
+            getApp().getJobQueue ().addJob (jtWRITE, "HashedObject::store",
+                                           BIND_TYPE (&HashedObjectStore::bulkWriteSQLite, this, P_1));
+        }
+    }
+
+    //  else
+    //      WriteLog (lsTRACE, HashedObject) << "HOS: already had " << hash;
+    mNegativeCache.del (hash);
+
+    return true;
+}
+
+void HashedObjectStore::bulkWriteSQLite (Job&)
+{
+    assert (!mLevelDB);
+    int setSize = 0;
+
+    while (1)
+    {
+        std::vector< boost::shared_ptr<HashedObject> > set;
+        set.reserve (128);
+
+        {
+            boost::mutex::scoped_lock sl (mWriteMutex);
+            mWriteSet.swap (set);
+            assert (mWriteSet.empty ());
+            ++mWriteGeneration;
+            mWriteCondition.notify_all ();
+
+            if (set.empty ())
+            {
+                mWritePending = false;
+                mWriteLoad = 0;
+                return;
+            }
+
+            mWriteLoad = std::max (setSize, static_cast<int> (mWriteSet.size ()));
+            setSize = set.size ();
+        }
+        //      WriteLog (lsTRACE, HashedObject) << "HOS: writing " << set.size();
+
+#ifndef NO_SQLITE3_PREPARE
+
+        if (mEphemeralDB)
+            LLWrite (set, getApp().getEphemeralLDB ());
+
+        {
+            Database* db = getApp().getHashNodeDB ()->getDB ();
+
+
+            // VFALCO TODO Get rid of the last parameter "aux", which is set to !theConfig.RUN_STANDALONE
+            //
+            static SqliteStatement pStB (db->getSqliteDB (), "BEGIN TRANSACTION;", !theConfig.RUN_STANDALONE);
+            static SqliteStatement pStE (db->getSqliteDB (), "END TRANSACTION;", !theConfig.RUN_STANDALONE);
+            static SqliteStatement pSt (db->getSqliteDB (),
+                                        "INSERT OR IGNORE INTO CommittedObjects "
+                                        "(Hash,ObjType,LedgerIndex,Object) VALUES (?, ?, ?, ?);", !theConfig.RUN_STANDALONE);
+
+            pStB.step ();
+            pStB.reset ();
+
+            BOOST_FOREACH (const boost::shared_ptr<HashedObject>& it, set)
+            {
+                const char* type;
+
+                switch (it->getType ())
+                {
+                case hotLEDGER:
+                    type = "L";
+                    break;
+
+                case hotTRANSACTION:
+                    type = "T";
+                    break;
+
+                case hotACCOUNT_NODE:
+                    type = "A";
+                    break;
+
+                case hotTRANSACTION_NODE:
+                    type = "N";
+                    break;
+
+                default:
+                    type = "U";
+                }
+
+                pSt.bind (1, it->getHash ().GetHex ());
+                pSt.bind (2, type);
+                pSt.bind (3, it->getIndex ());
+                pSt.bindStatic (4, it->getData ());
+                int ret = pSt.step ();
+
+                if (!pSt.isDone (ret))
+                {
+                    WriteLog (lsFATAL, HashedObject) << "Error saving hashed object " << ret;
+                    assert (false);
+                }
+
+                pSt.reset ();
+            }
+
+            pStE.step ();
+            pStE.reset ();
+        }
+
+#else
+
+        static boost::format
+        fAdd ("INSERT OR IGNORE INTO CommittedObjects "
+              "(Hash,ObjType,LedgerIndex,Object) VALUES ('%s','%c','%u',%s);");
+
+        Database* db = getApp().getHashNodeDB ()->getDB ();
+        {
+            ScopedLock sl (getApp().getHashNodeDB ()->getDBLock ());
+
+            db->executeSQL ("BEGIN TRANSACTION;");
+
+            BOOST_FOREACH (const boost::shared_ptr<HashedObject>& it, set)
+            {
+                char type;
+
+                switch (it->getType ())
+                {
+                case hotLEDGER:
+                    type = 'L';
+                    break;
+
+                case hotTRANSACTION:
+                    type = 'T';
+                    break;
+
+                case hotACCOUNT_NODE:
+                    type = 'A';
+                    break;
+
+                case hotTRANSACTION_NODE:
+                    type = 'N';
+                    break;
+
+                default:
+                    type = 'U';
+                }
+
+                db->executeSQL (boost::str (boost::format (fAdd)
+                                            % it->getHash ().GetHex () % type % it->getIndex () % sqlEscape (it->getData ())));
+            }
+
+            db->executeSQL ("END TRANSACTION;");
+        }
+#endif
+
+    }
+}
+
+HashedObject::pointer HashedObjectStore::retrieveSQLite (uint256 const& hash)
+{
+    HashedObject::pointer obj = mCache.fetch (hash);
+
+    if (obj)
+        return obj;
+
+    if (mNegativeCache.isPresent (hash))
+        return obj;
+
+    if (mEphemeralDB)
+    {
+        obj = LLRetrieve (hash, getApp().getEphemeralLDB ());
+
+        if (obj)
+        {
+            mCache.canonicalize (hash, obj);
+            return obj;
+        }
+    }
+
+    if (!getApp().getHashNodeDB ())
+        return obj;
+
+    Blob data;
+    std::string type;
+    uint32 index;
+
+#ifndef NO_SQLITE3_PREPARE
+    {
+        ScopedLock sl (getApp().getHashNodeDB ()->getDBLock ());
+        static SqliteStatement pSt (getApp().getHashNodeDB ()->getDB ()->getSqliteDB (),
+                                    "SELECT ObjType,LedgerIndex,Object FROM CommittedObjects WHERE Hash = ?;");
+        LoadEvent::autoptr event (getApp().getJobQueue ().getLoadEventAP (jtDISK, "HOS::retrieve"));
+
+        pSt.bind (1, hash.GetHex ());
+        int ret = pSt.step ();
+
+        if (pSt.isDone (ret))
+        {
+            pSt.reset ();
+            mNegativeCache.add (hash);
+            WriteLog (lsTRACE, HashedObject) << "HOS: " << hash << " fetch: not in db";
+            return obj;
+        }
+
+        type = pSt.peekString (0);
+        index = pSt.getUInt32 (1);
+        pSt.getBlob (2).swap (data);
+        pSt.reset ();
+    }
+
+#else
+
+    std::string sql = "SELECT * FROM CommittedObjects WHERE Hash='";
+    sql.append (hash.GetHex ());
+    sql.append ("';");
+
+
+    {
+        ScopedLock sl (getApp().getHashNodeDB ()->getDBLock ());
+        Database* db = getApp().getHashNodeDB ()->getDB ();
+
+        if (!db->executeSQL (sql) || !db->startIterRows ())
+        {
+            sl.unlock ();
+            mNegativeCache.add (hash);
+            return obj;
+        }
+
+        db->getStr ("ObjType", type);
+        index = db->getBigInt ("LedgerIndex");
+
+        int size = db->getBinary ("Object", NULL, 0);
+        data.resize (size);
+        db->getBinary ("Object", & (data.front ()), size);
+        db->endIterRows ();
+    }
+#endif
+
+#ifdef PARANOID
+    assert (Serializer::getSHA512Half (data) == hash);
+#endif
+
+    HashedObjectType htype = hotUNKNOWN;
+
+    switch (type[0])
+    {
+    case 'L':
+        htype = hotLEDGER;
+        break;
+
+    case 'T':
+        htype = hotTRANSACTION;
+        break;
+
+    case 'A':
+        htype = hotACCOUNT_NODE;
+        break;
+
+    case 'N':
+        htype = hotTRANSACTION_NODE;
+        break;
+
+    default:
+        assert (false);
+        WriteLog (lsERROR, HashedObject) << "Invalid hashed object";
+        mNegativeCache.add (hash);
+        return obj;
+    }
+
+    obj = boost::make_shared<HashedObject> (htype, index, data, hash);
+    mCache.canonicalize (hash, obj);
+
+    if (mEphemeralDB)
+        LLWrite (obj, getApp().getEphemeralLDB ());
+
+    WriteLog (lsTRACE, HashedObject) << "HOS: " << hash << " fetch: in db";
+    return obj;
+}
+
+int HashedObjectStore::import (const std::string& file)
+{
+    WriteLog (lsWARNING, HashedObject) << "Hashed object import from \"" << file << "\".";
+    UPTR_T<Database> importDB (new SqliteDatabase (file.c_str ()));
+    importDB->connect ();
+
+    leveldb::DB* db = getApp().getHashNodeLDB ();
+    leveldb::WriteOptions wo;
+
+    int count = 0;
+
+    SQL_FOREACH (importDB, "SELECT * FROM CommittedObjects;")
+    {
+        uint256 hash;
+        std::string hashStr;
+        importDB->getStr ("Hash", hashStr);
+        hash.SetHexExact (hashStr);
+
+        if (hash.isZero ())
+        {
+            WriteLog (lsWARNING, HashedObject) << "zero hash found in import table";
+        }
+        else
+        {
+            Blob rawData;
+            int size = importDB->getBinary ("Object", NULL, 0);
+            rawData.resize (9 + size);
+            unsigned char* bufPtr = &rawData.front ();
+
+            importDB->getBinary ("Object", bufPtr + 9, size);
+
+            uint32 index = importDB->getBigInt ("LedgerIndex");
+            *reinterpret_cast<uint32*> (bufPtr + 0) = ntohl (index);
+            *reinterpret_cast<uint32*> (bufPtr + 4) = ntohl (index);
+
+            std::string type;
+            importDB->getStr ("ObjType", type);
+            HashedObjectType htype = hotUNKNOWN;
+
+            switch (type[0])
+            {
+            case 'L':
+                htype = hotLEDGER;
+                break;
+
+            case 'T':
+                htype = hotTRANSACTION;
+                break;
+
+            case 'A':
+                htype = hotACCOUNT_NODE;
+                break;
+
+            case 'N':
+                htype = hotTRANSACTION_NODE;
+                break;
+
+            default:
+                assert (false);
+                WriteLog (lsERROR, HashedObject) << "Invalid hashed object";
+            }
+
+            * (bufPtr + 8) = static_cast<unsigned char> (htype);
+
+            leveldb::Status st = db->Put (wo,
+                                          leveldb::Slice (reinterpret_cast<const char*> (hash.begin ()), hash.size ()),
+                                          leveldb::Slice (reinterpret_cast<const char*> (bufPtr), rawData.size ()));
+
+            if (!st.ok ())
+            {
+                WriteLog (lsFATAL, HashedObject) << "Failed to store hash node";
+                assert (false);
+            }
+
+            ++count;
+        }
+
+        if ((count % 10000) == 0)
+        {
+            WriteLog (lsINFO, HashedObject) << "Import in progress: " << count;
+        }
+    }
+
+    WriteLog (lsWARNING, HashedObject) << "Imported " << count << " nodes";
+    return count;
+}
+
+// vim:ts=4

modules/ripple_app/node/ripple_HashedObjectStore.h

@@ -0,0 +1,88 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_HASHEDOBJECTSTORE_H
+#define RIPPLE_HASHEDOBJECTSTORE_H
+
+/** Persistency layer for hashed objects.
+*/
+// VFALCO TODO Move all definitions to the .cpp
+class HashedObjectStore : LeakChecked <HashedObjectStore>
+{
+public:
+    HashedObjectStore (int cacheSize, int cacheAge);
+
+    bool isLevelDB ()
+    {
+        return mLevelDB;
+    }
+
+    float getCacheHitRate ()
+    {
+        return mCache.getHitRate ();
+    }
+
+    bool store (HashedObjectType type, uint32 index, Blob const& data,
+                uint256 const& hash)
+    {
+        if (mLevelDB)
+            return storeLevelDB (type, index, data, hash);
+
+        return storeSQLite (type, index, data, hash);
+    }
+
+    HashedObject::pointer retrieve (uint256 const& hash)
+    {
+        if (mLevelDB)
+            return retrieveLevelDB (hash);
+
+        return retrieveSQLite (hash);
+    }
+
+    bool storeSQLite (HashedObjectType type, uint32 index, Blob const& data,
+                      uint256 const& hash);
+    HashedObject::pointer retrieveSQLite (uint256 const& hash);
+    void bulkWriteSQLite (Job&);
+
+    bool storeLevelDB (HashedObjectType type, uint32 index, Blob const& data,
+                       uint256 const& hash);
+    HashedObject::pointer retrieveLevelDB (uint256 const& hash);
+    void bulkWriteLevelDB (Job&);
+
+
+    void waitWrite ();
+    void tune (int size, int age);
+    void sweep ()
+    {
+        mCache.sweep ();
+        mNegativeCache.sweep ();
+    }
+    int getWriteLoad ();
+
+    int import (const std::string& fileName);
+
+private:
+    static HashedObject::pointer LLRetrieve (uint256 const& hash, leveldb::DB* db);
+    static void LLWrite (boost::shared_ptr<HashedObject> ptr, leveldb::DB* db);
+    static void LLWrite (const std::vector< boost::shared_ptr<HashedObject> >& set, leveldb::DB* db);
+
+private:
+    TaggedCache<uint256, HashedObject, UptimeTimerAdapter>  mCache;
+    KeyCache <uint256, UptimeTimerAdapter> mNegativeCache;
+
+    boost::mutex                mWriteMutex;
+    boost::condition_variable   mWriteCondition;
+    int                         mWriteGeneration;
+    int                         mWriteLoad;
+
+    std::vector< boost::shared_ptr<HashedObject> > mWriteSet;
+    bool mWritePending;
+    bool mLevelDB;
+    bool mEphemeralDB;
+};
+
+#endif
+// vim:ts=4

modules/ripple_app/ripple_app.cpp

@@ -89,7 +89,17 @@ namespace ripple
 //             linearize the include sequence and view it in one place.
 //
 
-#include "src/cpp/ripple/ripple_HashedObject.h"
+#include "src/cpp/ripple/ripple_Database.h"
+#include "src/cpp/ripple/ripple_DatabaseCon.h"
+#include "src/cpp/ripple/ripple_SqliteDatabase.h"
+#include "src/cpp/ripple/ripple_DBInit.h"
+
+#include "node/ripple_HashedObject.h"
+#include "node/ripple_HashedObjectStore.h"
+#include "node/ripple_HashStoreBE.h"
+#include "node/ripple_HSBELevelDB.h"
+#include "node/ripple_HSBESqlite.h"
+
 #include "src/cpp/ripple/ripple_SHAMapItem.h"
 #include "src/cpp/ripple/ripple_SHAMapNode.h"
 #include "src/cpp/ripple/ripple_SHAMapTreeNode.h"
@@ -106,11 +116,8 @@ namespace ripple
 #include "src/cpp/ripple/Ledger.h"
 #include "src/cpp/ripple/SerializedValidation.h"
 #include "src/cpp/ripple/ripple_ILoadManager.h"
-#include "src/cpp/ripple/ripple_DatabaseCon.h"
 #include "src/cpp/ripple/ripple_ProofOfWork.h"
 #include "src/cpp/ripple/ripple_InfoSub.h"
-#include "src/cpp/ripple/ripple_HashedObject.h"
-#include "src/cpp/ripple/ripple_HashedObjectStore.h"
 #include "src/cpp/ripple/ripple_OrderBook.h"
 #include "src/cpp/ripple/ripple_SHAMapSyncFilters.h"
 #include "src/cpp/ripple/ripple_IFeatures.h"
@@ -125,8 +132,6 @@ namespace ripple
 #include "src/cpp/ripple/ripple_PeerSet.h"
 #include "src/cpp/ripple/ripple_InboundLedger.h"
 #include "src/cpp/ripple/ripple_InboundLedgers.h"
-#include "src/cpp/ripple/ripple_Database.h"
-#include "src/cpp/ripple/ripple_SqliteDatabase.h"
 #include "src/cpp/ripple/ScriptData.h"
 #include "src/cpp/ripple/Contract.h"
 #include "src/cpp/ripple/Interpreter.h"
@@ -230,6 +235,10 @@ static const uint64 tenTo17m1 = tenTo17 - 1;
 #if ! defined (RIPPLE_MAIN_PART) || RIPPLE_MAIN_PART == 1
 
 #include "basics/ripple_RPCServerHandler.cpp"
+#include "node/ripple_HashedObject.cpp"
+#include "node/ripple_HashedObjectStore.cpp"
+#include "node/ripple_HSBELevelDB.cpp"
+#include "node/ripple_HSBESqlite.cpp"
 
 #include "src/cpp/ripple/Ledger.cpp"
 #include "src/cpp/ripple/ripple_SHAMapDelta.cpp"
@@ -240,12 +249,6 @@ static const uint64 tenTo17m1 = tenTo17 - 1;
 #include "src/cpp/ripple/ripple_AccountItems.cpp"
 #include "src/cpp/ripple/ripple_AccountState.cpp"
 #include "src/cpp/ripple/ChangeTransactor.cpp"
-#include "src/cpp/ripple/ripple_DBInit.cpp"
-#include "src/cpp/ripple/Interpreter.cpp"
-#include "src/cpp/ripple/LedgerTiming.cpp"
-#include "src/cpp/ripple/ripple_Main.cpp"
-#include "src/cpp/ripple/ripple_Offer.cpp"
-#include "src/cpp/ripple/Operation.cpp"
 
 #endif
 
@@ -337,6 +340,9 @@ static DH* handleTmpDh (SSL* ssl, int is_export, int iKeyLength)
 #include "src/cpp/ripple/ripple_AcceptedLedgerTx.cpp"
 #include "src/cpp/ripple/ripple_DatabaseCon.cpp"
 #include "src/cpp/ripple/ripple_FeeVote.cpp"
+#include "src/cpp/ripple/ripple_DBInit.cpp"
+#include "src/cpp/ripple/Interpreter.cpp"
+#include "src/cpp/ripple/LedgerTiming.cpp"
 
 #endif
 
@@ -349,10 +355,11 @@ static DH* handleTmpDh (SSL* ssl, int is_export, int iKeyLength)
 #include "src/cpp/ripple/ripple_Features.cpp"
 
 #include "src/cpp/ripple/ripple_LocalCredentials.cpp"
-#include "src/cpp/ripple/ripple_HashedObject.cpp"
 #include "src/cpp/ripple/ripple_AcceptedLedger.cpp"
 #include "src/cpp/ripple/ripple_DisputedTx.cpp"
 #include "src/cpp/ripple/ripple_HashRouter.cpp"
+#include "src/cpp/ripple/ripple_Main.cpp"
+#include "src/cpp/ripple/ripple_Offer.cpp"
 
 #endif
 
@@ -362,13 +369,13 @@ static DH* handleTmpDh (SSL* ssl, int is_export, int iKeyLength)
 
 #include "src/cpp/ripple/NetworkOPs.cpp"
 #include "src/cpp/ripple/ripple_Peers.cpp"
-#include "src/cpp/ripple/ripple_HashedObjectStore.cpp"
 
 #include "src/cpp/ripple/ripple_InboundLedgers.cpp"
 #include "src/cpp/ripple/ripple_LedgerHistory.cpp"
 #include "src/cpp/ripple/ripple_PathRequest.cpp"
 #include "src/cpp/ripple/ripple_SerializedLedger.cpp"
 #include "src/cpp/ripple/ripple_TransactionAcquire.cpp"
+#include "src/cpp/ripple/Operation.cpp"
 
 #endif