Finish NodeStore import config, add ephemeral db to unit tests

Builds/VisualStudio2012/RippleD.vcxproj

@@ -1456,6 +1456,7 @@
     <ClInclude Include="..\..\modules\ripple_basio\ripple_basio_impl.h" />
     <ClInclude Include="..\..\modules\ripple_client\ripple_client.h" />
     <ClInclude Include="..\..\modules\ripple_core\functional\ripple_Config.h" />
+    <ClInclude Include="..\..\modules\ripple_core\functional\ripple_ConfigSections.h" />
     <ClInclude Include="..\..\modules\ripple_core\functional\ripple_ILoadFeeTrack.h" />
     <ClInclude Include="..\..\modules\ripple_core\functional\ripple_Job.h" />
     <ClInclude Include="..\..\modules\ripple_core\functional\ripple_JobQueue.h" />

Builds/VisualStudio2012/RippleD.vcxproj.filters

@@ -1689,6 +1689,9 @@
     <ClInclude Include="..\..\modules\ripple_basics\utility\ripple_ScopedLock.h">
       <Filter>[1] Ripple\ripple_basics\utility</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\modules\ripple_core\functional\ripple_ConfigSections.h">
+      <Filter>[1] Ripple\ripple_core\functional</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <CustomBuild Include="..\..\src\cpp\ripple\ripple.proto" />

TODO.txt

@@ -2,7 +2,10 @@
 RIPPLE TODO
 --------------------------------------------------------------------------------
 
+Items marked '*' can be handled by third parties.
+
 Vinnie's Short List (Changes day to day)
+- Make theConfig a SharedSingleton to prevent leak warnings
 - Add fast backend to the unit test
 - Refactor Section code into ConfigFile
 - Change NodeStore config file format to multiline key/value pairs
@@ -17,7 +20,10 @@ Vinnie's Short List (Changes day to day)
 
 --------------------------------------------------------------------------------
 
-- Replace all throw with beast::Throw
+* Restyle all the macros in ripple_ConfigSection.h
+
+* Replace all throw with beast::Throw
+  Only in the ripple sources, not in Subtrees/ or protobuf or websocket
 
 - Replace base_uint and uintXXX with UnsignedInteger
   * Need to specialize UnsignedInteger to work efficiently with 4 and 8 byte
@@ -25,12 +31,6 @@ Vinnie's Short List (Changes day to day)
 
 - Rewrite boost program_options in Beast
 
-- Examples for different backend key/value config settings
-
-- Unit Test attention
-
-- NodeStore backend unit test
-
 - Validations unit test
 
 - Replace endian conversion calls with beast calls:

modules/ripple_app/node/ripple_NodeStore.cpp

@@ -402,7 +402,12 @@ public:
 
     //------------------------------------------------------------------------------
 
-    void import (Parameters const& sourceBackendParameters)
+    void visitAll (Backend::VisitCallback& callback)
+    {
+        m_backend->visitAll (callback);
+    }
+
+    void import (NodeStore& sourceDatabase)
     {
         class ImportVisitCallback : public Backend::VisitCallback
         {
@@ -439,16 +444,15 @@ public:
 
         //--------------------------------------------------------------------------
 
-        ScopedPointer <Backend> srcBackend (createBackend (sourceBackendParameters, m_scheduler));
-
         ImportVisitCallback callback (*m_backend);
 
-        srcBackend->visitAll (callback);
+        sourceDatabase.visitAll (callback);
     }
 
     //------------------------------------------------------------------------------
 
-    static NodeStore::Backend* createBackend (Parameters const& parameters, Scheduler& scheduler)
+    static NodeStore::Backend* createBackend (
+        Parameters const& parameters, Scheduler& scheduler = getSynchronousScheduler ())
     {
         Backend* backend = nullptr;
 
@@ -484,11 +488,6 @@ public:
         return backend;
     }
 
-    static NodeStore::Backend* createBackend (String const& parameterString, Scheduler& scheduler)
-    {
-        return createBackend (parseDelimitedKeyValueString (parameterString), scheduler);
-    }
-
     static void addBackendFactory (BackendFactory& factory)
     {
         s_factories.add (&factory);
@@ -516,52 +515,29 @@ Array <NodeStore::BackendFactory*> NodeStoreImp::s_factories;
 
 //------------------------------------------------------------------------------
 
-NodeStore::Parameters NodeStore::parseDelimitedKeyValueString (String parameters, beast_wchar delimiter)
-{
-    StringPairArray keyValues;
-
-    while (parameters.isNotEmpty ())
-    {
-        String pair;
-
-        {
-            int const delimiterPos = parameters.indexOfChar (delimiter);
-
-            if (delimiterPos != -1)
-            {
-                pair = parameters.substring (0, delimiterPos);
-
-                parameters = parameters.substring (delimiterPos + 1);
-            }
-            else
-            {
-                pair = parameters;
-
-                parameters = String::empty;
-            }
-        }
-
-        int const equalPos = pair.indexOfChar ('=');
-
-        if (equalPos != -1)
-        {
-            String const key = pair.substring (0, equalPos);
-            String const value = pair.substring (equalPos + 1, pair.length ());
-
-            keyValues.set (key, value);
-        }
-    }
-
-    return keyValues;
-}
-
 void NodeStore::addBackendFactory (BackendFactory& factory)
 {
     NodeStoreImp::addBackendFactory (factory);
 }
 
+NodeStore::Scheduler& NodeStore::getSynchronousScheduler ()
+{
+    // Simple scheduler that performs the task immediately
+    struct SynchronousScheduler : Scheduler
+    {
+        void scheduleTask (Task* task)
+        {
+            task->performScheduledTask ();
+        }
+    };
+
+    static SynchronousScheduler scheduler;
+
+    return scheduler;
+}
+
 NodeStore* NodeStore::New (Parameters const& backendParameters,
-                           Parameters const& fastBackendParameters,
+                           Parameters fastBackendParameters,
                            Scheduler& scheduler)
 {
     return new NodeStoreImp (backendParameters,
@@ -569,15 +545,6 @@ NodeStore* NodeStore::New (Parameters const& backendParameters,
                              scheduler);
 }
 
-NodeStore* NodeStore::New (String const& backendParameters,
-                           String const& fastBackendParameters,
-                           Scheduler& scheduler)
-{
-    return new NodeStoreImp (parseDelimitedKeyValueString (backendParameters),
-                             parseDelimitedKeyValueString (fastBackendParameters),
-                             scheduler);
-}
-
 //==============================================================================
 
 // Some common code for the unit tests
@@ -598,15 +565,6 @@ public:
     typedef NodeStore::Backend Backend;
     typedef NodeStore::Batch Batch;
 
-    // Immediately performs the task
-    struct TestScheduler : NodeStore::Scheduler
-    {
-        void scheduleTask (Task* task)
-        {
-            task->performScheduledTask ();
-        }
-    };
-
     // Creates predictable objects
     class PredictableObjectFactory
     {
@@ -846,19 +804,18 @@ public:
     {
         beginTest (String ("NodeStore::Backend type=") + type);
 
-        String params;
+        StringPairArray params;
-        params << "type=" << type
+        File const path (File::createTempFile ("node_db"));
-               << "|path=" << File::createTempFile ("unittest").getFullPathName ();
+        params.set ("type", type);
+        params.set ("path", path.getFullPathName ());
 
         // Create a batch
         NodeStore::Batch batch;
         createPredictableBatch (batch, 0, numObjectsToTest, seedValue);
-        //createPredictableBatch (batch, 0, 10, seedValue);
 
         {
             // Open the backend
-            ScopedPointer <Backend> backend (
+            ScopedPointer <Backend> backend (NodeStoreImp::createBackend (params));
-                NodeStoreImp::createBackend (params, m_scheduler));
 
             // Write the batch
             storeBatch (*backend, batch);
@@ -881,8 +838,7 @@ public:
 
         {
             // Re-open the backend
-            ScopedPointer <Backend> backend (
+            ScopedPointer <Backend> backend (NodeStoreImp::createBackend (params));
-                NodeStoreImp::createBackend (params, m_scheduler));
 
             // Read it back in
             NodeStore::Batch copy;
@@ -894,6 +850,8 @@ public:
         }
     }
 
+    //--------------------------------------------------------------------------
+
     void runTest ()
     {
         int const seedValue = 50;
@@ -912,9 +870,6 @@ public:
         testBackend ("mdb", seedValue);
         #endif
     }
-
-private:
-    TestScheduler m_scheduler;
 };
 
 static NodeStoreBackendTests nodeStoreBackendTests;
@@ -957,15 +912,18 @@ public:
         int64 m_startTime;
     };
 
+    //--------------------------------------------------------------------------
+
     void testBackend (String type, int64 const seedValue)
     {
         String s;
         s << "Testing backend '" << type << "' performance";
         beginTest (s);
 
-        String params;
+        StringPairArray params;
-        params << "type=" << type
+        File const path (File::createTempFile ("node_db"));
-               << "|path=" << File::createTempFile ("unittest").getFullPathName ();
+        params.set ("type", type);
+        params.set ("path", path.getFullPathName ());
 
         // Create batches
         NodeStore::Batch batch1;
@@ -974,9 +932,7 @@ public:
         createPredictableBatch (batch2, 0, numObjectsToTest, seedValue);
 
         // Open the backend
-        ScopedPointer <Backend> backend (
+        ScopedPointer <Backend> backend (NodeStoreImp::createBackend (params));
-            NodeStoreImp::createBackend (
-            NodeStore::parseDelimitedKeyValueString (params), m_scheduler));
 
         Stopwatch t;
 
@@ -1004,6 +960,8 @@ public:
         logMessage (s);
     }
 
+    //--------------------------------------------------------------------------
+
     void runTest ()
     {
         int const seedValue = 50;
@@ -1022,11 +980,10 @@ public:
 
         testBackend ("sqlite", seedValue);
     }
-
-private:
-    TestScheduler m_scheduler;
 };
 
+static NodeStoreTimingTests nodeStoreTimingTests;
+
 //------------------------------------------------------------------------------
 
 class NodeStoreTests : public NodeStoreUnitTest
@@ -1038,9 +995,10 @@ public:
 
     void testImport (String destBackendType, String srcBackendType, int64 seedValue)
     {
-        String srcParams;
+        File const node_db (File::createTempFile ("node_db"));
-        srcParams << "type=" << srcBackendType
+        StringPairArray srcParams;
-                  << "|path=" << File::createTempFile ("unittest").getFullPathName ();
+        srcParams.set ("type", srcBackendType);
+        srcParams.set ("path", node_db.getFullPathName ());
 
         // Create a batch
         NodeStore::Batch batch;
@@ -1048,26 +1006,33 @@ public:
 
         // Write to source db
         {
-            ScopedPointer <NodeStore> src (NodeStore::New (srcParams, "", m_scheduler));
+            ScopedPointer <NodeStore> src (NodeStore::New (srcParams));
 
             storeBatch (*src, batch);
         }
 
-        String destParams;
-        destParams << "type=" << destBackendType
-                   << "|path=" << File::createTempFile ("unittest").getFullPathName ();
-
-        ScopedPointer <NodeStore> dest (NodeStore::New (
-            destParams, "", m_scheduler));
-
-        beginTest (String ("import into '") + destBackendType + "' from '" + srcBackendType + "'");
-
-        // Do the import
-        dest->import (NodeStore::parseDelimitedKeyValueString (srcParams));
-
-        // Get the results of the import
         NodeStore::Batch copy;
-        fetchCopyOfBatch (*dest, &copy, batch);
+
+        {
+            // Re-open the db
+            ScopedPointer <NodeStore> src (NodeStore::New (srcParams));
+
+            // Set up the destination database
+            File const dest_db (File::createTempFile ("dest_db"));
+            StringPairArray destParams;
+            destParams.set ("type", destBackendType);
+            destParams.set ("path", dest_db.getFullPathName ());
+
+            ScopedPointer <NodeStore> dest (NodeStore::New (destParams));
+
+            beginTest (String ("import into '") + destBackendType + "' from '" + srcBackendType + "'");
+
+            // Do the import
+            dest->import (*src);
+
+            // Get the results of the import
+            fetchCopyOfBatch (*dest, &copy, batch);
+        }
 
         // Canonicalize the source and destination batches
         std::sort (batch.begin (), batch.end (), NodeObject::LessThan ());
@@ -1076,16 +1041,29 @@ public:
 
     }
 
-    void testBackend (String type, int64 const seedValue)
+    //--------------------------------------------------------------------------
+
+    void testNodeStore (String type, bool const useEphemeralDatabase, int64 const seedValue)
     {
         String s;
-        s << String ("NodeStore backend type=") + type;
+        s << String ("NodeStore backend '") + type + "'";
+        if (useEphemeralDatabase)
+            s << " (with ephemeral database)";
 
         beginTest (s);
 
-        String params;
+        File const node_db (File::createTempFile ("node_db"));
-        params << "type=" << type
+        StringPairArray nodeParams;
-               << "|path=" << File::createTempFile ("unittest").getFullPathName ();
+        nodeParams.set ("type", type);
+        nodeParams.set ("path", node_db.getFullPathName ());
+
+        File const temp_db  (File::createTempFile ("temp_db"));
+        StringPairArray tempParams;
+        if (useEphemeralDatabase)
+        {
+            tempParams.set ("type", type);
+            tempParams.set ("path", temp_db.getFullPathName ());
+        }
 
         // Create a batch
         NodeStore::Batch batch;
@@ -1093,7 +1071,7 @@ public:
 
         {
             // Open the database
-            ScopedPointer <NodeStore> db (NodeStore::New (params, "", m_scheduler));
+            ScopedPointer <NodeStore> db (NodeStore::New (nodeParams, tempParams));
 
             // Write the batch
             storeBatch (*db, batch);
@@ -1115,12 +1093,28 @@ public:
         }
 
         {
-            // Re-open the database
+            // Re-open the database without the ephemeral DB
-            ScopedPointer <NodeStore> db (NodeStore::New (params, "", m_scheduler));
+            ScopedPointer <NodeStore> db (NodeStore::New (nodeParams));
 
             // Read it back in
             NodeStore::Batch copy;
             fetchCopyOfBatch (*db, &copy, batch);
+
+            // Canonicalize the source and destination batches
+            std::sort (batch.begin (), batch.end (), NodeObject::LessThan ());
+            std::sort (copy.begin (), copy.end (), NodeObject::LessThan ());
+            expect (areBatchesEqual (batch, copy), "Should be equal");
+        }
+
+        if (useEphemeralDatabase)
+        {
+            // Verify the ephemeral db
+            ScopedPointer <NodeStore> db (NodeStore::New (tempParams, StringPairArray ()));
+
+            // Read it back in
+            NodeStore::Batch copy;
+            fetchCopyOfBatch (*db, &copy, batch);
+
             // Canonicalize the source and destination batches
             std::sort (batch.begin (), batch.end (), NodeObject::LessThan ());
             std::sort (copy.begin (), copy.end (), NodeObject::LessThan ());
@@ -1128,33 +1122,29 @@ public:
         }
     }
 
-public:
+    //--------------------------------------------------------------------------
-    void runTest ()
+
+    void runBackendTests (bool useEphemeralDatabase, int64 const seedValue)
     {
-        int64 const seedValue = 50;
+        testNodeStore ("keyvadb", useEphemeralDatabase, seedValue);
 
-        //
+        testNodeStore ("leveldb", useEphemeralDatabase, seedValue);
-        // Backend tests
-        //
 
-        testBackend ("keyvadb", seedValue);
+        testNodeStore ("sqlite", useEphemeralDatabase, seedValue);
-
-        testBackend ("leveldb", seedValue);
-
-        testBackend ("sqlite", seedValue);
 
     #if RIPPLE_HYPERLEVELDB_AVAILABLE
-        testBackend ("hyperleveldb", seedValue);
+        testNodeStore ("hyperleveldb", useEphemeralDatabase, seedValue);
     #endif
 
     #if RIPPLE_MDB_AVAILABLE
-        testBackend ("mdb", seedValue);
+        testNodeStore ("mdb", useEphemeralDatabase, seedValue);
     #endif
+    }
 
-        //
+    //--------------------------------------------------------------------------
-        // Import tests
-        //
 
+    void runImportTests (int64 const seedValue)
+    {
         //testImport ("keyvadb", "keyvadb", seedValue);
 
         testImport ("leveldb", "leveldb", seedValue);
@@ -1170,10 +1160,18 @@ public:
         testImport ("sqlite", "sqlite", seedValue);
     }
 
-private:
+    //--------------------------------------------------------------------------
-    TestScheduler m_scheduler;
+
+    void runTest ()
+    {
+        int64 const seedValue = 50;
+
+        runBackendTests (false, seedValue);
+
+        runBackendTests (true, seedValue);
+
+        runImportTests (seedValue);
+    }
 };
 
 static NodeStoreTests nodeStoreTests;
-
-static NodeStoreTimingTests nodeStoreTimingTests;

modules/ripple_app/node/ripple_NodeStore.h

@@ -102,6 +102,8 @@ public:
         For improved performance, a backend has the option of performing writes
         in batches. These writes can be scheduled using the provided scheduler
         object.
+
+        @see BatchWriter
     */
     class Scheduler
     {
@@ -272,7 +274,7 @@ public:
             @note This routine will not be called concurrently with itself
                   or other methods.
 
-            @see import
+            @see import, VisitCallback
         */
         virtual void visitAll (VisitCallback& callback) = 0;
 
@@ -307,49 +309,38 @@ public:
 
     //--------------------------------------------------------------------------
 
-    /** Create a Parameters from a String.
-
-        Parameter strings have the format:
-
-        <key>=<value>['|'<key>=<value>]
-
-        The key "type" must exist, it defines the choice of backend.
-        For example
-            `type=LevelDB|path=/mnt/ephemeral`
-
-        This is a convenience function for unit tests.
-    */
-    static Parameters parseDelimitedKeyValueString (String s, beast_wchar delimiter='|');
-
     /** Construct a node store.
 
-        Parameter strings have the format:
+        The parameters are key value pairs passed to the backend. The
+        'type' key must exist, it defines the choice of backend. Most
+        backends also require a 'path' field.
         
-        <key>=<value>['|'<key>=<value>]
+        Some choices for 'type' are:
+            HyperLevelDB, LevelDB, SQLite, KeyvaDB, MDB
 
-        The key "type" must exist, it defines the choice of backend.
+        If the fastBackendParameter is omitted or empty, no ephemeral database
-        For example
+        is used. If the scheduler parameter is omited or unspecified, a
-            `type=LevelDB|path=/mnt/ephemeral`
+        synchronous scheduler is used which performs all tasks immediately on
+        the caller's thread.
+
+        @note If the database cannot be opened or created, an exception is thrown.
 
         @param backendParameters The parameter string for the persistent backend.
-        @param fastBackendParameters The parameter string for the ephemeral backend.
+        @param fastBackendParameters [optional] The parameter string for the ephemeral backend.                        
-        @param cacheSize ?
+        @param scheduler [optional The scheduler to use for performing asynchronous tasks.
-        @param cacheAge ?
-        @param scheduler The scheduler to use for performing asynchronous tasks.
 
-        @return A pointer to the created object.
+        @return The opened database.
     */
     static NodeStore* New (Parameters const& backendParameters,
-                           Parameters const& fastBackendParameters,
+                           Parameters fastBackendParameters = Parameters (),
-                           Scheduler& scheduler);
+                           Scheduler& scheduler = getSynchronousScheduler ());
 
-    /** Construct a node store from a pipe delimited parameter string.
+    /** Get the synchronous scheduler.
 
-        This is used for unit tests.
+        The synchronous scheduler performs all tasks immediately, before
+        returning to the caller, using the caller's thread.
     */
-    static NodeStore* New (String const& backendParameters,
+    static Scheduler& getSynchronousScheduler ();
-                           String const& fastBackendParameters,
-                           Scheduler& scheduler);
 
     /** Destroy the node store.
 
@@ -405,12 +396,20 @@ public:
                         Blob& data,
                         uint256 const& hash) = 0;
 
-    /** Import objects from another database.
+    /** Visit every object in the database
             
-        The other NodeStore database is constructed using the specified
+        This is usually called during import.
-        backend parameters.
+
+        @note This routine will not be called concurrently with itself
+                or other methods.
+
+        @see import
     */
-    virtual void import (Parameters const& sourceBackendParameters) = 0;
+    virtual void visitAll (Backend::VisitCallback& callback) = 0;
+
+    /** Import objects from another database. */
+    virtual void import (NodeStore& sourceDatabase) = 0;
+
 
     /** Retrieve the estimated number of pending write operations.
 

modules/ripple_basics/utility/ripple_IniFile.cpp

@@ -128,15 +128,18 @@ bool SectionSingleB (Section& secSource, const std::string& strSection, std::str
     return bSingle;
 }
 
-StringPairArray parseKeyValueSection (Section& secSource, std::string const& strSection)
+StringPairArray parseKeyValueSection (Section& secSource, String const& strSection)
 {
     StringPairArray result;
 
-    int const count = SectionCount (secSource, strSection);
+    // yuck.
+    std::string const stdStrSection (strSection.toStdString ());
+
+    int const count = SectionCount (secSource, stdStrSection);
 
     typedef Section::mapped_type Entries;
 
-    Entries* const entries = SectionEntries (secSource, strSection);
+    Entries* const entries = SectionEntries (secSource, stdStrSection);
 
     if (entries != nullptr)
     {

modules/ripple_basics/utility/ripple_IniFile.h

@@ -25,6 +25,6 @@ Section::mapped_type* SectionEntries (Section& secSource, const std::string& str
     Each line is in the form <key>=<value>.
     Spaces are considered part of the key and value.
 */
-StringPairArray parseKeyValueSection (Section& secSource, std::string const& strSection);
+StringPairArray parseKeyValueSection (Section& secSource, String const& strSection);
 
 #endif

modules/ripple_basics/utility/ripple_StringUtilities.cpp

@@ -271,4 +271,41 @@ std::string addressToString (void const* address)
     return strHex (static_cast <char const*> (address) - static_cast <char const*> (0));
 }
 
+StringPairArray parseDelimitedKeyValueString (String parameters, beast_wchar delimiter)
+{
+    StringPairArray keyValues;
 
+    while (parameters.isNotEmpty ())
+    {
+        String pair;
+
+        {
+            int const delimiterPos = parameters.indexOfChar (delimiter);
+
+            if (delimiterPos != -1)
+            {
+                pair = parameters.substring (0, delimiterPos);
+
+                parameters = parameters.substring (delimiterPos + 1);
+            }
+            else
+            {
+                pair = parameters;
+
+                parameters = String::empty;
+            }
+        }
+
+        int const equalPos = pair.indexOfChar ('=');
+
+        if (equalPos != -1)
+        {
+            String const key = pair.substring (0, equalPos);
+            String const value = pair.substring (equalPos + 1, pair.length ());
+
+            keyValues.set (key, value);
+        }
+    }
+
+    return keyValues;
+}

modules/ripple_basics/utility/ripple_StringUtilities.h

@@ -214,4 +214,12 @@ bool parseUrl (const std::string& strUrl, std::string& strScheme, std::string& s
 */
 extern std::string addressToString (void const* address);
 
+/** Create a Parameters from a String.
+
+    Parameter strings have the format:
+
+    <key>=<value>['|'<key>=<value>]
+*/
+extern StringPairArray parseDelimitedKeyValueString (String s, beast_wchar delimiter='|');
+
 #endif

modules/ripple_core/functional/ripple_Config.cpp

@@ -8,71 +8,6 @@
 // TODO: Check permissions on config file before using it.
 //
 
-// VFALCO TODO Rename and replace these macros with variables.
-#define SECTION_ACCOUNT_PROBE_MAX       "account_probe_max"
-#define SECTION_CLUSTER_NODES           "cluster_nodes"
-#define SECTION_DATABASE_PATH           "database_path"
-#define SECTION_DEBUG_LOGFILE           "debug_logfile"
-#define SECTION_ELB_SUPPORT             "elb_support"
-#define SECTION_FEE_DEFAULT             "fee_default"
-#define SECTION_FEE_NICKNAME_CREATE     "fee_nickname_create"
-#define SECTION_FEE_OFFER               "fee_offer"
-#define SECTION_FEE_OPERATION           "fee_operation"
-#define SECTION_FEE_ACCOUNT_RESERVE     "fee_account_reserve"
-#define SECTION_FEE_OWNER_RESERVE       "fee_owner_reserve"
-#define SECTION_NODE_DB                 "node_db"
-#define SECTION_FASTNODE_DB             "temp_db"
-#define SECTION_LEDGER_HISTORY          "ledger_history"
-#define SECTION_IPS                     "ips"
-#define SECTION_NETWORK_QUORUM          "network_quorum"
-#define SECTION_NODE_SEED               "node_seed"
-#define SECTION_NODE_SIZE               "node_size"
-#define SECTION_PATH_SEARCH_SIZE        "path_search_size"
-#define SECTION_PEER_CONNECT_LOW_WATER  "peer_connect_low_water"
-#define SECTION_PEER_IP                 "peer_ip"
-#define SECTION_PEER_PORT               "peer_port"
-#define SECTION_PEER_PRIVATE            "peer_private"
-#define SECTION_PEER_SCAN_INTERVAL_MIN  "peer_scan_interval_min"
-#define SECTION_PEER_SSL_CIPHER_LIST    "peer_ssl_cipher_list"
-#define SECTION_PEER_START_MAX          "peer_start_max"
-#define SECTION_RPC_ALLOW_REMOTE        "rpc_allow_remote"
-#define SECTION_RPC_ADMIN_ALLOW         "rpc_admin_allow"
-#define SECTION_RPC_ADMIN_USER          "rpc_admin_user"
-#define SECTION_RPC_ADMIN_PASSWORD      "rpc_admin_password"
-#define SECTION_RPC_IP                  "rpc_ip"
-#define SECTION_RPC_PORT                "rpc_port"
-#define SECTION_RPC_USER                "rpc_user"
-#define SECTION_RPC_PASSWORD            "rpc_password"
-#define SECTION_RPC_STARTUP             "rpc_startup"
-#define SECTION_RPC_SECURE              "rpc_secure"
-#define SECTION_RPC_SSL_CERT            "rpc_ssl_cert"
-#define SECTION_RPC_SSL_CHAIN           "rpc_ssl_chain"
-#define SECTION_RPC_SSL_KEY             "rpc_ssl_key"
-#define SECTION_SMS_FROM                "sms_from"
-#define SECTION_SMS_KEY                 "sms_key"
-#define SECTION_SMS_SECRET              "sms_secret"
-#define SECTION_SMS_TO                  "sms_to"
-#define SECTION_SMS_URL                 "sms_url"
-#define SECTION_SNTP                    "sntp_servers"
-#define SECTION_SSL_VERIFY              "ssl_verify"
-#define SECTION_SSL_VERIFY_FILE         "ssl_verify_file"
-#define SECTION_SSL_VERIFY_DIR          "ssl_verify_dir"
-#define SECTION_VALIDATORS_FILE         "validators_file"
-#define SECTION_VALIDATION_QUORUM       "validation_quorum"
-#define SECTION_VALIDATION_SEED         "validation_seed"
-#define SECTION_WEBSOCKET_PUBLIC_IP     "websocket_public_ip"
-#define SECTION_WEBSOCKET_PUBLIC_PORT   "websocket_public_port"
-#define SECTION_WEBSOCKET_PUBLIC_SECURE "websocket_public_secure"
-#define SECTION_WEBSOCKET_PING_FREQ     "websocket_ping_frequency"
-#define SECTION_WEBSOCKET_IP            "websocket_ip"
-#define SECTION_WEBSOCKET_PORT          "websocket_port"
-#define SECTION_WEBSOCKET_SECURE        "websocket_secure"
-#define SECTION_WEBSOCKET_SSL_CERT      "websocket_ssl_cert"
-#define SECTION_WEBSOCKET_SSL_CHAIN     "websocket_ssl_chain"
-#define SECTION_WEBSOCKET_SSL_KEY       "websocket_ssl_key"
-#define SECTION_VALIDATORS              "validators"
-#define SECTION_VALIDATORS_SITE         "validators_site"
-
 // Fees are in XRP.
 #define DEFAULT_FEE_DEFAULT             10
 #define DEFAULT_FEE_ACCOUNT_RESERVE     200*SYSTEM_CURRENCY_PARTS
@@ -81,6 +16,8 @@
 #define DEFAULT_FEE_OFFER               DEFAULT_FEE_DEFAULT
 #define DEFAULT_FEE_OPERATION           1
 
+// VFALCO TODO Convert this to a SharedSingleton to prevent exit leaks
+//
 Config theConfig;
 
 void Config::setup (const std::string& strConf, bool bTestNet, bool bQuiet)
@@ -373,8 +310,23 @@ void Config::load ()
             (void) SectionSingleB (secConfig, SECTION_RPC_IP, m_rpcIP);
             (void) SectionSingleB (secConfig, SECTION_RPC_PASSWORD, RPC_PASSWORD);
             (void) SectionSingleB (secConfig, SECTION_RPC_USER, RPC_USER);
-            theConfig.nodeDatabase = parseKeyValueSection (secConfig, SECTION_NODE_DB);
+
-            theConfig.ephemeralNodeDatabase = parseKeyValueSection (secConfig, SECTION_FASTNODE_DB);
+            //---------------------------------------
+            //
+            // VFALCO BEGIN CLEAN
+            //
+            theConfig.nodeDatabase = parseKeyValueSection (
+                secConfig, ConfigSection::nodeDatabase ());
+
+            theConfig.ephemeralNodeDatabase = parseKeyValueSection (
+                secConfig, ConfigSection::tempNodeDatabase ());
+
+            theConfig.importNodeDatabase = parseKeyValueSection (
+                secConfig, ConfigSection::importNodeDatabase ());
+            //
+            // VFALCO END CLEAN
+            //
+            //---------------------------------------
 
             if (SectionSingleB (secConfig, SECTION_RPC_PORT, strTemp))
                 m_rpcPort = boost::lexical_cast<int> (strTemp);

modules/ripple_core/functional/ripple_Config.h

@@ -85,12 +85,40 @@ public:
     boost::filesystem::path     DEBUG_LOGFILE;
     boost::filesystem::path     VALIDATORS_FILE;        // As specifed in rippled.cfg.
 
-    StringPairArray nodeDatabase;
+    /** Parameters for the main NodeStore database.
-    StringPairArray ephemeralNodeDatabase;
+
-    //std::string                 NODE_DB;                // Database to use for nodes
+        This is 1 or more strings of the form <key>=<value>
-    //std::string                 FASTNODE_DB;            // Database for temporary storage
+        The 'type' and 'path' keys are required, see rippled-example.cfg
+
+        @see NodeStore
+    */
+    StringPairArray nodeDatabase;
+
+    /** Parameters for the ephemeral NodeStore database.
+
+        This is an auxiliary database for the NodeStore, usually placed
+        on a separate faster volume. However, the volume data may not persist
+        between launches. Use of the ephemeral database is optional.
+
+        The format is the same as that for @ref nodeDatabase
+
+        @see NodeStore
+    */
+    StringPairArray ephemeralNodeDatabase;
+
+    /** Parameters for importing an old database in to the current node database.
+
+        If this is not empty, then it specifies the key/value parameters for
+        another node database from which to import all data into the current
+        node database specified by @ref nodeDatabase.
+
+        The format of this string is in the form:
+            <key>'='<value>['|'<key>'='value]
+
+        @see parseDelimitedKeyValueString
+    */
+    StringPairArray importNodeDatabase;
 
-    std::string                 DB_IMPORT;              // Import from old DB
     bool                        ELB_SUPPORT;            // Support Amazon ELB
 
     std::string                 VALIDATORS_SITE;        // Where to find validators.txt on the Internet.

modules/ripple_core/functional/ripple_ConfigSections.h

@@ -0,0 +1,86 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_CONFIGSECTIONS_H_INCLUDED
+#define RIPPLE_CONFIGSECTIONS_H_INCLUDED
+
+// VFALCO NOTE
+//
+//      Please use this style for all new sections
+//      And if you're feeling generous, convert all the
+//      existing macros to this format as well.
+//
+struct ConfigSection
+{
+    static String nodeDatabase ()                 { return "node_db"; }
+    static String tempNodeDatabase ()             { return "temp_db"; }
+    static String importNodeDatabase ()           { return "import_db"; }
+};
+
+// VFALCO TODO Rename and replace these macros with variables.
+#define SECTION_ACCOUNT_PROBE_MAX       "account_probe_max"
+#define SECTION_CLUSTER_NODES           "cluster_nodes"
+#define SECTION_DATABASE_PATH           "database_path"
+#define SECTION_DEBUG_LOGFILE           "debug_logfile"
+#define SECTION_ELB_SUPPORT             "elb_support"
+#define SECTION_FEE_DEFAULT             "fee_default"
+#define SECTION_FEE_NICKNAME_CREATE     "fee_nickname_create"
+#define SECTION_FEE_OFFER               "fee_offer"
+#define SECTION_FEE_OPERATION           "fee_operation"
+#define SECTION_FEE_ACCOUNT_RESERVE     "fee_account_reserve"
+#define SECTION_FEE_OWNER_RESERVE       "fee_owner_reserve"
+#define SECTION_LEDGER_HISTORY          "ledger_history"
+#define SECTION_IPS                     "ips"
+#define SECTION_NETWORK_QUORUM          "network_quorum"
+#define SECTION_NODE_SEED               "node_seed"
+#define SECTION_NODE_SIZE               "node_size"
+#define SECTION_PATH_SEARCH_SIZE        "path_search_size"
+#define SECTION_PEER_CONNECT_LOW_WATER  "peer_connect_low_water"
+#define SECTION_PEER_IP                 "peer_ip"
+#define SECTION_PEER_PORT               "peer_port"
+#define SECTION_PEER_PRIVATE            "peer_private"
+#define SECTION_PEER_SCAN_INTERVAL_MIN  "peer_scan_interval_min"
+#define SECTION_PEER_SSL_CIPHER_LIST    "peer_ssl_cipher_list"
+#define SECTION_PEER_START_MAX          "peer_start_max"
+#define SECTION_RPC_ALLOW_REMOTE        "rpc_allow_remote"
+#define SECTION_RPC_ADMIN_ALLOW         "rpc_admin_allow"
+#define SECTION_RPC_ADMIN_USER          "rpc_admin_user"
+#define SECTION_RPC_ADMIN_PASSWORD      "rpc_admin_password"
+#define SECTION_RPC_IP                  "rpc_ip"
+#define SECTION_RPC_PORT                "rpc_port"
+#define SECTION_RPC_USER                "rpc_user"
+#define SECTION_RPC_PASSWORD            "rpc_password"
+#define SECTION_RPC_STARTUP             "rpc_startup"
+#define SECTION_RPC_SECURE              "rpc_secure"
+#define SECTION_RPC_SSL_CERT            "rpc_ssl_cert"
+#define SECTION_RPC_SSL_CHAIN           "rpc_ssl_chain"
+#define SECTION_RPC_SSL_KEY             "rpc_ssl_key"
+#define SECTION_SMS_FROM                "sms_from"
+#define SECTION_SMS_KEY                 "sms_key"
+#define SECTION_SMS_SECRET              "sms_secret"
+#define SECTION_SMS_TO                  "sms_to"
+#define SECTION_SMS_URL                 "sms_url"
+#define SECTION_SNTP                    "sntp_servers"
+#define SECTION_SSL_VERIFY              "ssl_verify"
+#define SECTION_SSL_VERIFY_FILE         "ssl_verify_file"
+#define SECTION_SSL_VERIFY_DIR          "ssl_verify_dir"
+#define SECTION_VALIDATORS_FILE         "validators_file"
+#define SECTION_VALIDATION_QUORUM       "validation_quorum"
+#define SECTION_VALIDATION_SEED         "validation_seed"
+#define SECTION_WEBSOCKET_PUBLIC_IP     "websocket_public_ip"
+#define SECTION_WEBSOCKET_PUBLIC_PORT   "websocket_public_port"
+#define SECTION_WEBSOCKET_PUBLIC_SECURE "websocket_public_secure"
+#define SECTION_WEBSOCKET_PING_FREQ     "websocket_ping_frequency"
+#define SECTION_WEBSOCKET_IP            "websocket_ip"
+#define SECTION_WEBSOCKET_PORT          "websocket_port"
+#define SECTION_WEBSOCKET_SECURE        "websocket_secure"
+#define SECTION_WEBSOCKET_SSL_CERT      "websocket_ssl_cert"
+#define SECTION_WEBSOCKET_SSL_CHAIN     "websocket_ssl_chain"
+#define SECTION_WEBSOCKET_SSL_KEY       "websocket_ssl_key"
+#define SECTION_VALIDATORS              "validators"
+#define SECTION_VALIDATORS_SITE         "validators_site"
+
+#endif

modules/ripple_core/ripple_core.h

@@ -30,6 +30,7 @@ namespace ripple
 
 // VFALCO NOTE Indentation shows dependency hierarchy
 //
+/***/#include "functional/ripple_ConfigSections.h"
 /**/#include "functional/ripple_Config.h"
 /**/#include "functional/ripple_ILoadFeeTrack.h"
 /*..*/#include "functional/ripple_LoadEvent.h"

rippled-example.cfg

@@ -223,18 +223,22 @@
 #              shfArahZT9Q9ckTf3s1psJ7C7qzVN
 #
 #
+#-------------------------------------------------------------------------------
 #
 # [node_db]
 # [temp_db]
+# [import_db]
 #
-#   Set the choice of databases for storing Node objects.
+#   Set database options for storing node objects in the primary database,
+#   caching node objects in the temporary database, or importing node objects
+#   from a previous database.
 #
 #   Format (without spaces):
 #       One or more lines of key / value pairs:
 #       <key> '=' <value>
 #       ...
 #
-#   Example:
+#   Examples:
 #       type=HyperLevelDB
 #       path=db/hashnode
 #
@@ -252,11 +256,19 @@
 #   Optional keys:
 #       (none yet)
 #
-#   Notes
+#   Notes:
+#
 #       The 'node_db' entry configures the primary, persistent storage.
+#
 #       The 'temp_db' configures a look-aside cache for high volume storage
 #           which doesn't necessarily persist between server launches.
 #
+#       The 'import_db' is used with the '--import' command line option to
+#           migrate the specified database into the current database given
+#           in the [node_db] section.
+#
+#-------------------------------------------------------------------------------
+#
 # [node_size]
 #   Tunes the servers based on the expected load and available memory. Legal
 #   sizes are "tiny", "small", "medium", "large", and "huge". We recommend

src/cpp/ripple/ripple_Application.cpp

@@ -974,13 +974,15 @@ void ApplicationImp::updateTables ()
         exit (1);
     }
 
-    if (!theConfig.DB_IMPORT.empty())
+    if (theConfig.importNodeDatabase.size () > 0)
     {
+        ScopedPointer <NodeStore> source (NodeStore::New (theConfig.importNodeDatabase));
+
         WriteLog (lsWARNING, NodeObject) <<
-            "Node import from '" << theConfig.DB_IMPORT << "' to '"
+            "Node import from '" << source->getName () << "' to '"
                                  << getApp().getNodeStore().getName () << "'.";
 
-        getApp().getNodeStore().import(NodeStore::parseDelimitedKeyValueString (theConfig.DB_IMPORT));
+        getApp().getNodeStore().import (*source);
     }
 }
 

src/cpp/ripple/ripple_Main.cpp

@@ -216,6 +216,15 @@ int rippleMain (int argc, char** argv)
     int iResult = 0;
     po::variables_map   vm;                                     // Map of options.
 
+    String importDescription;
+    {
+        importDescription <<
+            "Import an existing node database (specified in the " <<
+            "[" << ConfigSection::importNodeDatabase () << "] configuration file section) "
+            "into the current node database (specified in the " <<
+            "[" << ConfigSection::nodeDatabase () << "] configuration file section). ";
+    }
+
     // VFALCO TODO Replace boost program options with something from Beast.
     //
     // Set up option parsing.
@@ -240,7 +249,7 @@ int rippleMain (int argc, char** argv)
     ("start", "Start from a fresh Ledger.")
     ("net", "Get the initial ledger from the network.")
     ("fg", "Run in the foreground.")
-    ("import", po::value<std::string> (), "Import old DB into new DB.")
+    ("import", importDescription.toStdString ().c_str ())
     ;
 
     // Interpret positional arguments as --parameters.
@@ -354,8 +363,14 @@ int rippleMain (int argc, char** argv)
 
     if (vm.count ("start")) theConfig.START_UP = Config::FRESH;
 
+    // Handle a one-time import option
+    //
     if (vm.count ("import"))
-        theConfig.DB_IMPORT = vm["import"].as<std::string> ();
+    {
+        String const optionString (vm ["import"].as <std::string> ());
+
+        theConfig.importNodeDatabase = parseDelimitedKeyValueString (optionString);
+    }
 
     if (vm.count ("ledger"))
     {