Fix NodeStore import

modules/ripple_app/node/ripple_NodeStore.cpp

@@ -6,8 +6,9 @@
 
 Array <NodeStore::BackendFactory*> NodeStore::s_factories;
 
-NodeStore::NodeStore (String parameters, int cacheSize, int cacheAge)
+NodeStore::NodeStore (String backendParameters, String fastBackendParameters, int cacheSize, int cacheAge)
-    : mCache ("NodeStore", cacheSize, cacheAge)
+    : m_backend (createBackend (backendParameters))
+    , mCache ("NodeStore", cacheSize, cacheAge)
     , mNegativeCache ("HashedObjectNegativeCache", 0, 120)
     , mWriteGeneration (0)
     , mWriteLoad (0)
@@ -15,36 +16,8 @@ NodeStore::NodeStore (String parameters, int cacheSize, int cacheAge)
     , mLevelDB (false)
     , mEphemeralDB (false)
 {
-    StringPairArray keyValues = parseKeyValueParameters (parameters, '|');
+    if (fastBackendParameters.isNotEmpty ())
-
+        m_fastBackend = createBackend (fastBackendParameters);
-    String const& type = keyValues ["type"];
-
-    if (type.isNotEmpty ())
-    {
-        BackendFactory* factory = nullptr;
-
-        for (int i = 0; i < s_factories.size (); ++i)
-        {
-            if (s_factories [i]->getName () == type)
-            {
-                factory = s_factories [i];
-                break;
-            }
-        }
-
-        if (factory != nullptr)
-        {
-            m_backend = factory->createInstance (keyValues);
-        }
-        else
-        {
-            throw std::runtime_error ("unkown backend type");
-        }
-    }
-    else
-    {
-        throw std::runtime_error ("missing backend type");
-    }
 
     mWriteSet.reserve (128);
 
@@ -177,8 +150,8 @@ void NodeStore::bulkWrite (Job&)
 
         m_backend->bulkStore (set);
 
-        if (m_backendFast)
+        if (m_fastBackend)
-            m_backendFast->bulkStore (set);
+            m_fastBackend->bulkStore (set);
     }
 }
 
@@ -189,9 +162,9 @@ NodeObject::pointer NodeStore::retrieve (uint256 const& hash)
     if (obj || mNegativeCache.isPresent (hash) || !getApp().getHashNodeLDB ())
         return obj;
 
-    if (m_backendFast)
+    if (m_fastBackend)
     {
-        obj = m_backendFast->retrieve (hash);
+        obj = m_fastBackend->retrieve (hash);
 
         if (obj)
         {
@@ -213,35 +186,82 @@ NodeObject::pointer NodeStore::retrieve (uint256 const& hash)
 
     mCache.canonicalize (hash, obj);
 
-    if (m_backendFast)
+    if (m_fastBackend)
-        m_backendFast->store(obj);
+        m_fastBackend->store(obj);
 
     WriteLog (lsTRACE, NodeObject) << "HOS: " << hash << " fetch: in db";
     return obj;
 }
 
-static void importFunc(HashStoreBE::pointer dest, std::vector<HashedObject::pointer>& objects,
+void NodeStore::importVisitor (
-    HashedObject::pointer object)
+    std::vector <NodeObject::pointer>& objects,
+    NodeObject::pointer object)
 {
     if (objects.size() >= 128)
     {
-        dest->bulkStore(objects);
+        m_backend->bulkStore (objects);
-        objects.clear();
+
-        objects.reserve(128);
+        objects.clear ();
+        objects.reserve (128);
     }
-    objects.push_back(object);
+
+    objects.push_back (object);
 }
 
-int HashedObjectStore::import (HashStoreBE::ref src, HashStoreBE::ref dest)
+int NodeStore::import (String sourceBackendParameters)
 {
-    WriteLog (lsWARNING, HashedObject) << "Node import from \""
+    ScopedPointer <Backend> srcBackend = createBackend (sourceBackendParameters);
-        << src->getBackEndName() << ":" << src->getDataBaseName()
-        << "\" to \"" << dest->getBackEndName() << ":" << dest->getDataBaseName() << "\".";
 
-    std::vector<HashedObject::pointer> objects;
+    WriteLog (lsWARNING, NodeObject) <<
-    objects.reserve(128);
+        "Node import from '" << srcBackend->getDataBaseName() << "' to '"
-    src->visitAll(BIND_TYPE(&importFunc, dest, boost::ref(objects), P_1));
+                             << m_backend->getDataBaseName() << "'.";
 
-    if (!objects.empty())
+    std::vector <NodeObject::pointer> objects;
-        dest->bulkStore(objects);
+    
+    objects.reserve (128);
+    
+    srcBackend->visitAll (BIND_TYPE (&NodeStore::importVisitor, this, boost::ref (objects), P_1));
+
+    if (!objects.empty ())
+        m_backend->bulkStore (objects);
+
+    return 0;
+}
+
+NodeStore::Backend* NodeStore::createBackend (String const& parameters)
+{
+    Backend* backend = nullptr;
+
+    StringPairArray keyValues = parseKeyValueParameters (parameters, '|');
+
+    String const& type = keyValues ["type"];
+
+    if (type.isNotEmpty ())
+    {
+        BackendFactory* factory = nullptr;
+
+        for (int i = 0; i < s_factories.size (); ++i)
+        {
+            if (s_factories [i]->getName () == type)
+            {
+                factory = s_factories [i];
+                break;
+            }
+        }
+
+        if (factory != nullptr)
+        {
+            backend = factory->createInstance (keyValues);
+        }
+        else
+        {
+            throw std::runtime_error ("unkown backend type");
+        }
+    }
+    else
+    {
+        throw std::runtime_error ("missing backend type");
+    }
+
+    return backend;
 }

modules/ripple_app/node/ripple_NodeStore.h

@@ -34,11 +34,11 @@ public:
 
         // Store a group of objects
         // This function will only be called from a single thread
-        virtual bool bulkStore(const std::vector< NodeObject::pointer >&) = 0;
+        virtual bool bulkStore (const std::vector< NodeObject::pointer >&) = 0;
 
         // Visit every object in the database
         // This function will only be called during an import operation
-        virtual void visitAll(FUNCTION_TYPE<void (NodeObject::pointer)>) = 0;
+        virtual void visitAll (FUNCTION_TYPE <void (NodeObject::pointer)>) = 0;
     };
 
 public:
@@ -68,7 +68,13 @@ public:
         The key "type" must exist, it defines the backend. For example
             "type=LevelDB|path=/mnt/ephemeral"
     */
-    NodeStore (String parameters, int cacheSize, int cacheAge);
+    // VFALCO NOTE Is cacheSize in bytes? objects? KB?
+    //             Is cacheAge in minutes? seconds?
+    //
+    NodeStore (String backendParameters,
+               String fastBackendParameters,
+               int cacheSize,
+               int cacheAge);
 
     /** Add the specified backend factory to the list of available factories.
 
@@ -92,19 +98,18 @@ public:
     void sweep ();
     int getWriteLoad ();
 
-    int import (const std::string& fileName);
+    int import (String sourceBackendParameters);
 
 private:
-    static NodeObject::pointer LLRetrieve (uint256 const& hash, leveldb::DB* db);
+    void importVisitor (std::vector <NodeObject::pointer>& objects, NodeObject::pointer object);
-    static void LLWrite (boost::shared_ptr<NodeObject> ptr, leveldb::DB* db);
+    
-    static void LLWrite (const std::vector< boost::shared_ptr<NodeObject> >& set, leveldb::DB* db);
+    static Backend* createBackend (String const& parameters);
 
-private:
     static Array <BackendFactory*> s_factories;
 
 private:
     ScopedPointer <Backend> m_backend;
-    ScopedPointer <Backend> m_backendFast;
+    ScopedPointer <Backend> m_fastBackend;
 
     TaggedCache<uint256, NodeObject, UptimeTimerAdapter>  mCache;
     KeyCache <uint256, UptimeTimerAdapter> mNegativeCache;

src/cpp/ripple/ripple_Application.cpp

@@ -45,7 +45,7 @@ public:
         , mNetOps (&mLedgerMaster)
         , m_rpcServerHandler (mNetOps)
         , mTempNodeCache ("NodeCache", 16384, 90)
-        , m_nodeStore ("type=LevelDB|path=/mnt/stuff|compact=1", 16384, 300)
+        , m_nodeStore ("type=LevelDB|path=/mnt/stuff|compact=1", "", 16384, 300)
         , mSLECache ("LedgerEntryCache", 4096, 120)
         , mSNTPClient (mAuxService)
         , mJobQueue (mIOService)