diff --git a/Builds/VisualStudio2012/RippleD.vcxproj b/Builds/VisualStudio2012/RippleD.vcxproj index f4e259fdc..10595bf0b 100644 --- a/Builds/VisualStudio2012/RippleD.vcxproj +++ b/Builds/VisualStudio2012/RippleD.vcxproj @@ -1456,6 +1456,7 @@ + diff --git a/Builds/VisualStudio2012/RippleD.vcxproj.filters b/Builds/VisualStudio2012/RippleD.vcxproj.filters index a2d81919f..63a4e3ae6 100644 --- a/Builds/VisualStudio2012/RippleD.vcxproj.filters +++ b/Builds/VisualStudio2012/RippleD.vcxproj.filters @@ -1689,6 +1689,9 @@ [1] Ripple\ripple_basics\utility + + [1] Ripple\ripple_core\functional + diff --git a/TODO.txt b/TODO.txt index 517589b43..d2e68f765 100644 --- a/TODO.txt +++ b/TODO.txt @@ -2,7 +2,10 @@ RIPPLE TODO -------------------------------------------------------------------------------- +Items marked '*' can be handled by third parties. + Vinnie's Short List (Changes day to day) +- Make theConfig a SharedSingleton to prevent leak warnings - Add fast backend to the unit test - Refactor Section code into ConfigFile - Change NodeStore config file format to multiline key/value pairs @@ -17,7 +20,10 @@ Vinnie's Short List (Changes day to day) -------------------------------------------------------------------------------- -- Replace all throw with beast::Throw +* Restyle all the macros in ripple_ConfigSection.h + +* Replace all throw with beast::Throw + Only in the ripple sources, not in Subtrees/ or protobuf or websocket - Replace base_uint and uintXXX with UnsignedInteger * Need to specialize UnsignedInteger to work efficiently with 4 and 8 byte @@ -25,12 +31,6 @@ Vinnie's Short List (Changes day to day) - Rewrite boost program_options in Beast -- Examples for different backend key/value config settings - -- Unit Test attention - -- NodeStore backend unit test - - Validations unit test - Replace endian conversion calls with beast calls: diff --git a/modules/ripple_app/node/ripple_NodeStore.cpp b/modules/ripple_app/node/ripple_NodeStore.cpp index 1ffa2a09f..b0ddd751d 100644 --- a/modules/ripple_app/node/ripple_NodeStore.cpp +++ b/modules/ripple_app/node/ripple_NodeStore.cpp @@ -402,7 +402,12 @@ public: //------------------------------------------------------------------------------ - void import (Parameters const& sourceBackendParameters) + void visitAll (Backend::VisitCallback& callback) + { + m_backend->visitAll (callback); + } + + void import (NodeStore& sourceDatabase) { class ImportVisitCallback : public Backend::VisitCallback { @@ -439,16 +444,15 @@ public: //-------------------------------------------------------------------------- - ScopedPointer srcBackend (createBackend (sourceBackendParameters, m_scheduler)); - ImportVisitCallback callback (*m_backend); - srcBackend->visitAll (callback); + sourceDatabase.visitAll (callback); } //------------------------------------------------------------------------------ - static NodeStore::Backend* createBackend (Parameters const& parameters, Scheduler& scheduler) + static NodeStore::Backend* createBackend ( + Parameters const& parameters, Scheduler& scheduler = getSynchronousScheduler ()) { Backend* backend = nullptr; @@ -484,11 +488,6 @@ public: return backend; } - static NodeStore::Backend* createBackend (String const& parameterString, Scheduler& scheduler) - { - return createBackend (parseDelimitedKeyValueString (parameterString), scheduler); - } - static void addBackendFactory (BackendFactory& factory) { s_factories.add (&factory); @@ -516,52 +515,29 @@ Array NodeStoreImp::s_factories; //------------------------------------------------------------------------------ -NodeStore::Parameters NodeStore::parseDelimitedKeyValueString (String parameters, beast_wchar delimiter) -{ - StringPairArray keyValues; - - while (parameters.isNotEmpty ()) - { - String pair; - - { - int const delimiterPos = parameters.indexOfChar (delimiter); - - if (delimiterPos != -1) - { - pair = parameters.substring (0, delimiterPos); - - parameters = parameters.substring (delimiterPos + 1); - } - else - { - pair = parameters; - - parameters = String::empty; - } - } - - int const equalPos = pair.indexOfChar ('='); - - if (equalPos != -1) - { - String const key = pair.substring (0, equalPos); - String const value = pair.substring (equalPos + 1, pair.length ()); - - keyValues.set (key, value); - } - } - - return keyValues; -} - void NodeStore::addBackendFactory (BackendFactory& factory) { NodeStoreImp::addBackendFactory (factory); } +NodeStore::Scheduler& NodeStore::getSynchronousScheduler () +{ + // Simple scheduler that performs the task immediately + struct SynchronousScheduler : Scheduler + { + void scheduleTask (Task* task) + { + task->performScheduledTask (); + } + }; + + static SynchronousScheduler scheduler; + + return scheduler; +} + NodeStore* NodeStore::New (Parameters const& backendParameters, - Parameters const& fastBackendParameters, + Parameters fastBackendParameters, Scheduler& scheduler) { return new NodeStoreImp (backendParameters, @@ -569,15 +545,6 @@ NodeStore* NodeStore::New (Parameters const& backendParameters, scheduler); } -NodeStore* NodeStore::New (String const& backendParameters, - String const& fastBackendParameters, - Scheduler& scheduler) -{ - return new NodeStoreImp (parseDelimitedKeyValueString (backendParameters), - parseDelimitedKeyValueString (fastBackendParameters), - scheduler); -} - //============================================================================== // Some common code for the unit tests @@ -598,15 +565,6 @@ public: typedef NodeStore::Backend Backend; typedef NodeStore::Batch Batch; - // Immediately performs the task - struct TestScheduler : NodeStore::Scheduler - { - void scheduleTask (Task* task) - { - task->performScheduledTask (); - } - }; - // Creates predictable objects class PredictableObjectFactory { @@ -846,19 +804,18 @@ public: { beginTest (String ("NodeStore::Backend type=") + type); - String params; - params << "type=" << type - << "|path=" << File::createTempFile ("unittest").getFullPathName (); + StringPairArray params; + File const path (File::createTempFile ("node_db")); + params.set ("type", type); + params.set ("path", path.getFullPathName ()); // Create a batch NodeStore::Batch batch; createPredictableBatch (batch, 0, numObjectsToTest, seedValue); - //createPredictableBatch (batch, 0, 10, seedValue); { // Open the backend - ScopedPointer backend ( - NodeStoreImp::createBackend (params, m_scheduler)); + ScopedPointer backend (NodeStoreImp::createBackend (params)); // Write the batch storeBatch (*backend, batch); @@ -881,8 +838,7 @@ public: { // Re-open the backend - ScopedPointer backend ( - NodeStoreImp::createBackend (params, m_scheduler)); + ScopedPointer backend (NodeStoreImp::createBackend (params)); // Read it back in NodeStore::Batch copy; @@ -894,6 +850,8 @@ public: } } + //-------------------------------------------------------------------------- + void runTest () { int const seedValue = 50; @@ -912,9 +870,6 @@ public: testBackend ("mdb", seedValue); #endif } - -private: - TestScheduler m_scheduler; }; static NodeStoreBackendTests nodeStoreBackendTests; @@ -957,15 +912,18 @@ public: int64 m_startTime; }; + //-------------------------------------------------------------------------- + void testBackend (String type, int64 const seedValue) { String s; s << "Testing backend '" << type << "' performance"; beginTest (s); - String params; - params << "type=" << type - << "|path=" << File::createTempFile ("unittest").getFullPathName (); + StringPairArray params; + File const path (File::createTempFile ("node_db")); + params.set ("type", type); + params.set ("path", path.getFullPathName ()); // Create batches NodeStore::Batch batch1; @@ -974,9 +932,7 @@ public: createPredictableBatch (batch2, 0, numObjectsToTest, seedValue); // Open the backend - ScopedPointer backend ( - NodeStoreImp::createBackend ( - NodeStore::parseDelimitedKeyValueString (params), m_scheduler)); + ScopedPointer backend (NodeStoreImp::createBackend (params)); Stopwatch t; @@ -1004,6 +960,8 @@ public: logMessage (s); } + //-------------------------------------------------------------------------- + void runTest () { int const seedValue = 50; @@ -1022,11 +980,10 @@ public: testBackend ("sqlite", seedValue); } - -private: - TestScheduler m_scheduler; }; +static NodeStoreTimingTests nodeStoreTimingTests; + //------------------------------------------------------------------------------ class NodeStoreTests : public NodeStoreUnitTest @@ -1038,9 +995,10 @@ public: void testImport (String destBackendType, String srcBackendType, int64 seedValue) { - String srcParams; - srcParams << "type=" << srcBackendType - << "|path=" << File::createTempFile ("unittest").getFullPathName (); + File const node_db (File::createTempFile ("node_db")); + StringPairArray srcParams; + srcParams.set ("type", srcBackendType); + srcParams.set ("path", node_db.getFullPathName ()); // Create a batch NodeStore::Batch batch; @@ -1048,26 +1006,33 @@ public: // Write to source db { - ScopedPointer src (NodeStore::New (srcParams, "", m_scheduler)); + ScopedPointer src (NodeStore::New (srcParams)); storeBatch (*src, batch); } - String destParams; - destParams << "type=" << destBackendType - << "|path=" << File::createTempFile ("unittest").getFullPathName (); - - ScopedPointer dest (NodeStore::New ( - destParams, "", m_scheduler)); - - beginTest (String ("import into '") + destBackendType + "' from '" + srcBackendType + "'"); - - // Do the import - dest->import (NodeStore::parseDelimitedKeyValueString (srcParams)); - - // Get the results of the import NodeStore::Batch copy; - fetchCopyOfBatch (*dest, ©, batch); + + { + // Re-open the db + ScopedPointer src (NodeStore::New (srcParams)); + + // Set up the destination database + File const dest_db (File::createTempFile ("dest_db")); + StringPairArray destParams; + destParams.set ("type", destBackendType); + destParams.set ("path", dest_db.getFullPathName ()); + + ScopedPointer dest (NodeStore::New (destParams)); + + beginTest (String ("import into '") + destBackendType + "' from '" + srcBackendType + "'"); + + // Do the import + dest->import (*src); + + // Get the results of the import + fetchCopyOfBatch (*dest, ©, batch); + } // Canonicalize the source and destination batches std::sort (batch.begin (), batch.end (), NodeObject::LessThan ()); @@ -1076,16 +1041,29 @@ public: } - void testBackend (String type, int64 const seedValue) + //-------------------------------------------------------------------------- + + void testNodeStore (String type, bool const useEphemeralDatabase, int64 const seedValue) { String s; - s << String ("NodeStore backend type=") + type; + s << String ("NodeStore backend '") + type + "'"; + if (useEphemeralDatabase) + s << " (with ephemeral database)"; beginTest (s); - String params; - params << "type=" << type - << "|path=" << File::createTempFile ("unittest").getFullPathName (); + File const node_db (File::createTempFile ("node_db")); + StringPairArray nodeParams; + nodeParams.set ("type", type); + nodeParams.set ("path", node_db.getFullPathName ()); + + File const temp_db (File::createTempFile ("temp_db")); + StringPairArray tempParams; + if (useEphemeralDatabase) + { + tempParams.set ("type", type); + tempParams.set ("path", temp_db.getFullPathName ()); + } // Create a batch NodeStore::Batch batch; @@ -1093,7 +1071,7 @@ public: { // Open the database - ScopedPointer db (NodeStore::New (params, "", m_scheduler)); + ScopedPointer db (NodeStore::New (nodeParams, tempParams)); // Write the batch storeBatch (*db, batch); @@ -1115,12 +1093,28 @@ public: } { - // Re-open the database - ScopedPointer db (NodeStore::New (params, "", m_scheduler)); + // Re-open the database without the ephemeral DB + ScopedPointer db (NodeStore::New (nodeParams)); // Read it back in NodeStore::Batch copy; fetchCopyOfBatch (*db, ©, batch); + + // Canonicalize the source and destination batches + std::sort (batch.begin (), batch.end (), NodeObject::LessThan ()); + std::sort (copy.begin (), copy.end (), NodeObject::LessThan ()); + expect (areBatchesEqual (batch, copy), "Should be equal"); + } + + if (useEphemeralDatabase) + { + // Verify the ephemeral db + ScopedPointer db (NodeStore::New (tempParams, StringPairArray ())); + + // Read it back in + NodeStore::Batch copy; + fetchCopyOfBatch (*db, ©, batch); + // Canonicalize the source and destination batches std::sort (batch.begin (), batch.end (), NodeObject::LessThan ()); std::sort (copy.begin (), copy.end (), NodeObject::LessThan ()); @@ -1128,33 +1122,29 @@ public: } } -public: - void runTest () + //-------------------------------------------------------------------------- + + void runBackendTests (bool useEphemeralDatabase, int64 const seedValue) { - int64 const seedValue = 50; + testNodeStore ("keyvadb", useEphemeralDatabase, seedValue); - // - // Backend tests - // + testNodeStore ("leveldb", useEphemeralDatabase, seedValue); - testBackend ("keyvadb", seedValue); - - testBackend ("leveldb", seedValue); - - testBackend ("sqlite", seedValue); + testNodeStore ("sqlite", useEphemeralDatabase, seedValue); #if RIPPLE_HYPERLEVELDB_AVAILABLE - testBackend ("hyperleveldb", seedValue); + testNodeStore ("hyperleveldb", useEphemeralDatabase, seedValue); #endif #if RIPPLE_MDB_AVAILABLE - testBackend ("mdb", seedValue); + testNodeStore ("mdb", useEphemeralDatabase, seedValue); #endif + } - // - // Import tests - // + //-------------------------------------------------------------------------- + void runImportTests (int64 const seedValue) + { //testImport ("keyvadb", "keyvadb", seedValue); testImport ("leveldb", "leveldb", seedValue); @@ -1170,10 +1160,18 @@ public: testImport ("sqlite", "sqlite", seedValue); } -private: - TestScheduler m_scheduler; + //-------------------------------------------------------------------------- + + void runTest () + { + int64 const seedValue = 50; + + runBackendTests (false, seedValue); + + runBackendTests (true, seedValue); + + runImportTests (seedValue); + } }; static NodeStoreTests nodeStoreTests; - -static NodeStoreTimingTests nodeStoreTimingTests; diff --git a/modules/ripple_app/node/ripple_NodeStore.h b/modules/ripple_app/node/ripple_NodeStore.h index 4bd44cd1e..a2c26f72d 100644 --- a/modules/ripple_app/node/ripple_NodeStore.h +++ b/modules/ripple_app/node/ripple_NodeStore.h @@ -102,6 +102,8 @@ public: For improved performance, a backend has the option of performing writes in batches. These writes can be scheduled using the provided scheduler object. + + @see BatchWriter */ class Scheduler { @@ -272,7 +274,7 @@ public: @note This routine will not be called concurrently with itself or other methods. - @see import + @see import, VisitCallback */ virtual void visitAll (VisitCallback& callback) = 0; @@ -307,49 +309,38 @@ public: //-------------------------------------------------------------------------- - /** Create a Parameters from a String. - - Parameter strings have the format: - - =['|'=] - - The key "type" must exist, it defines the choice of backend. - For example - `type=LevelDB|path=/mnt/ephemeral` - - This is a convenience function for unit tests. - */ - static Parameters parseDelimitedKeyValueString (String s, beast_wchar delimiter='|'); - /** Construct a node store. - Parameter strings have the format: + The parameters are key value pairs passed to the backend. The + 'type' key must exist, it defines the choice of backend. Most + backends also require a 'path' field. + + Some choices for 'type' are: + HyperLevelDB, LevelDB, SQLite, KeyvaDB, MDB - =['|'=] + If the fastBackendParameter is omitted or empty, no ephemeral database + is used. If the scheduler parameter is omited or unspecified, a + synchronous scheduler is used which performs all tasks immediately on + the caller's thread. - The key "type" must exist, it defines the choice of backend. - For example - `type=LevelDB|path=/mnt/ephemeral` + @note If the database cannot be opened or created, an exception is thrown. @param backendParameters The parameter string for the persistent backend. - @param fastBackendParameters The parameter string for the ephemeral backend. - @param cacheSize ? - @param cacheAge ? - @param scheduler The scheduler to use for performing asynchronous tasks. + @param fastBackendParameters [optional] The parameter string for the ephemeral backend. + @param scheduler [optional The scheduler to use for performing asynchronous tasks. - @return A pointer to the created object. + @return The opened database. */ static NodeStore* New (Parameters const& backendParameters, - Parameters const& fastBackendParameters, - Scheduler& scheduler); + Parameters fastBackendParameters = Parameters (), + Scheduler& scheduler = getSynchronousScheduler ()); - /** Construct a node store from a pipe delimited parameter string. + /** Get the synchronous scheduler. - This is used for unit tests. + The synchronous scheduler performs all tasks immediately, before + returning to the caller, using the caller's thread. */ - static NodeStore* New (String const& backendParameters, - String const& fastBackendParameters, - Scheduler& scheduler); + static Scheduler& getSynchronousScheduler (); /** Destroy the node store. @@ -405,12 +396,20 @@ public: Blob& data, uint256 const& hash) = 0; - /** Import objects from another database. + /** Visit every object in the database + + This is usually called during import. - The other NodeStore database is constructed using the specified - backend parameters. + @note This routine will not be called concurrently with itself + or other methods. + + @see import */ - virtual void import (Parameters const& sourceBackendParameters) = 0; + virtual void visitAll (Backend::VisitCallback& callback) = 0; + + /** Import objects from another database. */ + virtual void import (NodeStore& sourceDatabase) = 0; + /** Retrieve the estimated number of pending write operations. diff --git a/modules/ripple_basics/utility/ripple_IniFile.cpp b/modules/ripple_basics/utility/ripple_IniFile.cpp index 795b010f6..7c230cb03 100644 --- a/modules/ripple_basics/utility/ripple_IniFile.cpp +++ b/modules/ripple_basics/utility/ripple_IniFile.cpp @@ -128,15 +128,18 @@ bool SectionSingleB (Section& secSource, const std::string& strSection, std::str return bSingle; } -StringPairArray parseKeyValueSection (Section& secSource, std::string const& strSection) +StringPairArray parseKeyValueSection (Section& secSource, String const& strSection) { StringPairArray result; - int const count = SectionCount (secSource, strSection); + // yuck. + std::string const stdStrSection (strSection.toStdString ()); + + int const count = SectionCount (secSource, stdStrSection); typedef Section::mapped_type Entries; - Entries* const entries = SectionEntries (secSource, strSection); + Entries* const entries = SectionEntries (secSource, stdStrSection); if (entries != nullptr) { diff --git a/modules/ripple_basics/utility/ripple_IniFile.h b/modules/ripple_basics/utility/ripple_IniFile.h index 79e7f546b..d8e27abb9 100644 --- a/modules/ripple_basics/utility/ripple_IniFile.h +++ b/modules/ripple_basics/utility/ripple_IniFile.h @@ -25,6 +25,6 @@ Section::mapped_type* SectionEntries (Section& secSource, const std::string& str Each line is in the form =. Spaces are considered part of the key and value. */ -StringPairArray parseKeyValueSection (Section& secSource, std::string const& strSection); +StringPairArray parseKeyValueSection (Section& secSource, String const& strSection); #endif diff --git a/modules/ripple_basics/utility/ripple_StringUtilities.cpp b/modules/ripple_basics/utility/ripple_StringUtilities.cpp index bfa42c589..550fab921 100644 --- a/modules/ripple_basics/utility/ripple_StringUtilities.cpp +++ b/modules/ripple_basics/utility/ripple_StringUtilities.cpp @@ -271,4 +271,41 @@ std::string addressToString (void const* address) return strHex (static_cast (address) - static_cast (0)); } +StringPairArray parseDelimitedKeyValueString (String parameters, beast_wchar delimiter) +{ + StringPairArray keyValues; + while (parameters.isNotEmpty ()) + { + String pair; + + { + int const delimiterPos = parameters.indexOfChar (delimiter); + + if (delimiterPos != -1) + { + pair = parameters.substring (0, delimiterPos); + + parameters = parameters.substring (delimiterPos + 1); + } + else + { + pair = parameters; + + parameters = String::empty; + } + } + + int const equalPos = pair.indexOfChar ('='); + + if (equalPos != -1) + { + String const key = pair.substring (0, equalPos); + String const value = pair.substring (equalPos + 1, pair.length ()); + + keyValues.set (key, value); + } + } + + return keyValues; +} diff --git a/modules/ripple_basics/utility/ripple_StringUtilities.h b/modules/ripple_basics/utility/ripple_StringUtilities.h index 3ddcf75ae..08dc14c54 100644 --- a/modules/ripple_basics/utility/ripple_StringUtilities.h +++ b/modules/ripple_basics/utility/ripple_StringUtilities.h @@ -214,4 +214,12 @@ bool parseUrl (const std::string& strUrl, std::string& strScheme, std::string& s */ extern std::string addressToString (void const* address); +/** Create a Parameters from a String. + + Parameter strings have the format: + + =['|'=] +*/ +extern StringPairArray parseDelimitedKeyValueString (String s, beast_wchar delimiter='|'); + #endif diff --git a/modules/ripple_core/functional/ripple_Config.cpp b/modules/ripple_core/functional/ripple_Config.cpp index 54d9157a9..2706f1292 100644 --- a/modules/ripple_core/functional/ripple_Config.cpp +++ b/modules/ripple_core/functional/ripple_Config.cpp @@ -8,71 +8,6 @@ // TODO: Check permissions on config file before using it. // -// VFALCO TODO Rename and replace these macros with variables. -#define SECTION_ACCOUNT_PROBE_MAX "account_probe_max" -#define SECTION_CLUSTER_NODES "cluster_nodes" -#define SECTION_DATABASE_PATH "database_path" -#define SECTION_DEBUG_LOGFILE "debug_logfile" -#define SECTION_ELB_SUPPORT "elb_support" -#define SECTION_FEE_DEFAULT "fee_default" -#define SECTION_FEE_NICKNAME_CREATE "fee_nickname_create" -#define SECTION_FEE_OFFER "fee_offer" -#define SECTION_FEE_OPERATION "fee_operation" -#define SECTION_FEE_ACCOUNT_RESERVE "fee_account_reserve" -#define SECTION_FEE_OWNER_RESERVE "fee_owner_reserve" -#define SECTION_NODE_DB "node_db" -#define SECTION_FASTNODE_DB "temp_db" -#define SECTION_LEDGER_HISTORY "ledger_history" -#define SECTION_IPS "ips" -#define SECTION_NETWORK_QUORUM "network_quorum" -#define SECTION_NODE_SEED "node_seed" -#define SECTION_NODE_SIZE "node_size" -#define SECTION_PATH_SEARCH_SIZE "path_search_size" -#define SECTION_PEER_CONNECT_LOW_WATER "peer_connect_low_water" -#define SECTION_PEER_IP "peer_ip" -#define SECTION_PEER_PORT "peer_port" -#define SECTION_PEER_PRIVATE "peer_private" -#define SECTION_PEER_SCAN_INTERVAL_MIN "peer_scan_interval_min" -#define SECTION_PEER_SSL_CIPHER_LIST "peer_ssl_cipher_list" -#define SECTION_PEER_START_MAX "peer_start_max" -#define SECTION_RPC_ALLOW_REMOTE "rpc_allow_remote" -#define SECTION_RPC_ADMIN_ALLOW "rpc_admin_allow" -#define SECTION_RPC_ADMIN_USER "rpc_admin_user" -#define SECTION_RPC_ADMIN_PASSWORD "rpc_admin_password" -#define SECTION_RPC_IP "rpc_ip" -#define SECTION_RPC_PORT "rpc_port" -#define SECTION_RPC_USER "rpc_user" -#define SECTION_RPC_PASSWORD "rpc_password" -#define SECTION_RPC_STARTUP "rpc_startup" -#define SECTION_RPC_SECURE "rpc_secure" -#define SECTION_RPC_SSL_CERT "rpc_ssl_cert" -#define SECTION_RPC_SSL_CHAIN "rpc_ssl_chain" -#define SECTION_RPC_SSL_KEY "rpc_ssl_key" -#define SECTION_SMS_FROM "sms_from" -#define SECTION_SMS_KEY "sms_key" -#define SECTION_SMS_SECRET "sms_secret" -#define SECTION_SMS_TO "sms_to" -#define SECTION_SMS_URL "sms_url" -#define SECTION_SNTP "sntp_servers" -#define SECTION_SSL_VERIFY "ssl_verify" -#define SECTION_SSL_VERIFY_FILE "ssl_verify_file" -#define SECTION_SSL_VERIFY_DIR "ssl_verify_dir" -#define SECTION_VALIDATORS_FILE "validators_file" -#define SECTION_VALIDATION_QUORUM "validation_quorum" -#define SECTION_VALIDATION_SEED "validation_seed" -#define SECTION_WEBSOCKET_PUBLIC_IP "websocket_public_ip" -#define SECTION_WEBSOCKET_PUBLIC_PORT "websocket_public_port" -#define SECTION_WEBSOCKET_PUBLIC_SECURE "websocket_public_secure" -#define SECTION_WEBSOCKET_PING_FREQ "websocket_ping_frequency" -#define SECTION_WEBSOCKET_IP "websocket_ip" -#define SECTION_WEBSOCKET_PORT "websocket_port" -#define SECTION_WEBSOCKET_SECURE "websocket_secure" -#define SECTION_WEBSOCKET_SSL_CERT "websocket_ssl_cert" -#define SECTION_WEBSOCKET_SSL_CHAIN "websocket_ssl_chain" -#define SECTION_WEBSOCKET_SSL_KEY "websocket_ssl_key" -#define SECTION_VALIDATORS "validators" -#define SECTION_VALIDATORS_SITE "validators_site" - // Fees are in XRP. #define DEFAULT_FEE_DEFAULT 10 #define DEFAULT_FEE_ACCOUNT_RESERVE 200*SYSTEM_CURRENCY_PARTS @@ -81,6 +16,8 @@ #define DEFAULT_FEE_OFFER DEFAULT_FEE_DEFAULT #define DEFAULT_FEE_OPERATION 1 +// VFALCO TODO Convert this to a SharedSingleton to prevent exit leaks +// Config theConfig; void Config::setup (const std::string& strConf, bool bTestNet, bool bQuiet) @@ -373,8 +310,23 @@ void Config::load () (void) SectionSingleB (secConfig, SECTION_RPC_IP, m_rpcIP); (void) SectionSingleB (secConfig, SECTION_RPC_PASSWORD, RPC_PASSWORD); (void) SectionSingleB (secConfig, SECTION_RPC_USER, RPC_USER); - theConfig.nodeDatabase = parseKeyValueSection (secConfig, SECTION_NODE_DB); - theConfig.ephemeralNodeDatabase = parseKeyValueSection (secConfig, SECTION_FASTNODE_DB); + + //--------------------------------------- + // + // VFALCO BEGIN CLEAN + // + theConfig.nodeDatabase = parseKeyValueSection ( + secConfig, ConfigSection::nodeDatabase ()); + + theConfig.ephemeralNodeDatabase = parseKeyValueSection ( + secConfig, ConfigSection::tempNodeDatabase ()); + + theConfig.importNodeDatabase = parseKeyValueSection ( + secConfig, ConfigSection::importNodeDatabase ()); + // + // VFALCO END CLEAN + // + //--------------------------------------- if (SectionSingleB (secConfig, SECTION_RPC_PORT, strTemp)) m_rpcPort = boost::lexical_cast (strTemp); diff --git a/modules/ripple_core/functional/ripple_Config.h b/modules/ripple_core/functional/ripple_Config.h index 05b6a377d..6f1cfc9bc 100644 --- a/modules/ripple_core/functional/ripple_Config.h +++ b/modules/ripple_core/functional/ripple_Config.h @@ -85,12 +85,40 @@ public: boost::filesystem::path DEBUG_LOGFILE; boost::filesystem::path VALIDATORS_FILE; // As specifed in rippled.cfg. - StringPairArray nodeDatabase; - StringPairArray ephemeralNodeDatabase; - //std::string NODE_DB; // Database to use for nodes - //std::string FASTNODE_DB; // Database for temporary storage + /** Parameters for the main NodeStore database. + + This is 1 or more strings of the form = + The 'type' and 'path' keys are required, see rippled-example.cfg + + @see NodeStore + */ + StringPairArray nodeDatabase; + + /** Parameters for the ephemeral NodeStore database. + + This is an auxiliary database for the NodeStore, usually placed + on a separate faster volume. However, the volume data may not persist + between launches. Use of the ephemeral database is optional. + + The format is the same as that for @ref nodeDatabase + + @see NodeStore + */ + StringPairArray ephemeralNodeDatabase; + + /** Parameters for importing an old database in to the current node database. + + If this is not empty, then it specifies the key/value parameters for + another node database from which to import all data into the current + node database specified by @ref nodeDatabase. + + The format of this string is in the form: + '='['|''='value] + + @see parseDelimitedKeyValueString + */ + StringPairArray importNodeDatabase; - std::string DB_IMPORT; // Import from old DB bool ELB_SUPPORT; // Support Amazon ELB std::string VALIDATORS_SITE; // Where to find validators.txt on the Internet. diff --git a/modules/ripple_core/functional/ripple_ConfigSections.h b/modules/ripple_core/functional/ripple_ConfigSections.h new file mode 100644 index 000000000..445137ae4 --- /dev/null +++ b/modules/ripple_core/functional/ripple_ConfigSections.h @@ -0,0 +1,86 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_CONFIGSECTIONS_H_INCLUDED +#define RIPPLE_CONFIGSECTIONS_H_INCLUDED + +// VFALCO NOTE +// +// Please use this style for all new sections +// And if you're feeling generous, convert all the +// existing macros to this format as well. +// +struct ConfigSection +{ + static String nodeDatabase () { return "node_db"; } + static String tempNodeDatabase () { return "temp_db"; } + static String importNodeDatabase () { return "import_db"; } +}; + +// VFALCO TODO Rename and replace these macros with variables. +#define SECTION_ACCOUNT_PROBE_MAX "account_probe_max" +#define SECTION_CLUSTER_NODES "cluster_nodes" +#define SECTION_DATABASE_PATH "database_path" +#define SECTION_DEBUG_LOGFILE "debug_logfile" +#define SECTION_ELB_SUPPORT "elb_support" +#define SECTION_FEE_DEFAULT "fee_default" +#define SECTION_FEE_NICKNAME_CREATE "fee_nickname_create" +#define SECTION_FEE_OFFER "fee_offer" +#define SECTION_FEE_OPERATION "fee_operation" +#define SECTION_FEE_ACCOUNT_RESERVE "fee_account_reserve" +#define SECTION_FEE_OWNER_RESERVE "fee_owner_reserve" +#define SECTION_LEDGER_HISTORY "ledger_history" +#define SECTION_IPS "ips" +#define SECTION_NETWORK_QUORUM "network_quorum" +#define SECTION_NODE_SEED "node_seed" +#define SECTION_NODE_SIZE "node_size" +#define SECTION_PATH_SEARCH_SIZE "path_search_size" +#define SECTION_PEER_CONNECT_LOW_WATER "peer_connect_low_water" +#define SECTION_PEER_IP "peer_ip" +#define SECTION_PEER_PORT "peer_port" +#define SECTION_PEER_PRIVATE "peer_private" +#define SECTION_PEER_SCAN_INTERVAL_MIN "peer_scan_interval_min" +#define SECTION_PEER_SSL_CIPHER_LIST "peer_ssl_cipher_list" +#define SECTION_PEER_START_MAX "peer_start_max" +#define SECTION_RPC_ALLOW_REMOTE "rpc_allow_remote" +#define SECTION_RPC_ADMIN_ALLOW "rpc_admin_allow" +#define SECTION_RPC_ADMIN_USER "rpc_admin_user" +#define SECTION_RPC_ADMIN_PASSWORD "rpc_admin_password" +#define SECTION_RPC_IP "rpc_ip" +#define SECTION_RPC_PORT "rpc_port" +#define SECTION_RPC_USER "rpc_user" +#define SECTION_RPC_PASSWORD "rpc_password" +#define SECTION_RPC_STARTUP "rpc_startup" +#define SECTION_RPC_SECURE "rpc_secure" +#define SECTION_RPC_SSL_CERT "rpc_ssl_cert" +#define SECTION_RPC_SSL_CHAIN "rpc_ssl_chain" +#define SECTION_RPC_SSL_KEY "rpc_ssl_key" +#define SECTION_SMS_FROM "sms_from" +#define SECTION_SMS_KEY "sms_key" +#define SECTION_SMS_SECRET "sms_secret" +#define SECTION_SMS_TO "sms_to" +#define SECTION_SMS_URL "sms_url" +#define SECTION_SNTP "sntp_servers" +#define SECTION_SSL_VERIFY "ssl_verify" +#define SECTION_SSL_VERIFY_FILE "ssl_verify_file" +#define SECTION_SSL_VERIFY_DIR "ssl_verify_dir" +#define SECTION_VALIDATORS_FILE "validators_file" +#define SECTION_VALIDATION_QUORUM "validation_quorum" +#define SECTION_VALIDATION_SEED "validation_seed" +#define SECTION_WEBSOCKET_PUBLIC_IP "websocket_public_ip" +#define SECTION_WEBSOCKET_PUBLIC_PORT "websocket_public_port" +#define SECTION_WEBSOCKET_PUBLIC_SECURE "websocket_public_secure" +#define SECTION_WEBSOCKET_PING_FREQ "websocket_ping_frequency" +#define SECTION_WEBSOCKET_IP "websocket_ip" +#define SECTION_WEBSOCKET_PORT "websocket_port" +#define SECTION_WEBSOCKET_SECURE "websocket_secure" +#define SECTION_WEBSOCKET_SSL_CERT "websocket_ssl_cert" +#define SECTION_WEBSOCKET_SSL_CHAIN "websocket_ssl_chain" +#define SECTION_WEBSOCKET_SSL_KEY "websocket_ssl_key" +#define SECTION_VALIDATORS "validators" +#define SECTION_VALIDATORS_SITE "validators_site" + +#endif diff --git a/modules/ripple_core/ripple_core.h b/modules/ripple_core/ripple_core.h index 4b8b13a48..e3849298c 100644 --- a/modules/ripple_core/ripple_core.h +++ b/modules/ripple_core/ripple_core.h @@ -30,6 +30,7 @@ namespace ripple // VFALCO NOTE Indentation shows dependency hierarchy // +/***/#include "functional/ripple_ConfigSections.h" /**/#include "functional/ripple_Config.h" /**/#include "functional/ripple_ILoadFeeTrack.h" /*..*/#include "functional/ripple_LoadEvent.h" diff --git a/rippled-example.cfg b/rippled-example.cfg index 699202072..7d59091af 100644 --- a/rippled-example.cfg +++ b/rippled-example.cfg @@ -223,18 +223,22 @@ # shfArahZT9Q9ckTf3s1psJ7C7qzVN # # +#------------------------------------------------------------------------------- # # [node_db] # [temp_db] +# [import_db] # -# Set the choice of databases for storing Node objects. +# Set database options for storing node objects in the primary database, +# caching node objects in the temporary database, or importing node objects +# from a previous database. # # Format (without spaces): # One or more lines of key / value pairs: # '=' # ... # -# Example: +# Examples: # type=HyperLevelDB # path=db/hashnode # @@ -252,11 +256,19 @@ # Optional keys: # (none yet) # -# Notes +# Notes: +# # The 'node_db' entry configures the primary, persistent storage. +# # The 'temp_db' configures a look-aside cache for high volume storage # which doesn't necessarily persist between server launches. # +# The 'import_db' is used with the '--import' command line option to +# migrate the specified database into the current database given +# in the [node_db] section. +# +#------------------------------------------------------------------------------- +# # [node_size] # Tunes the servers based on the expected load and available memory. Legal # sizes are "tiny", "small", "medium", "large", and "huge". We recommend diff --git a/src/cpp/ripple/ripple_Application.cpp b/src/cpp/ripple/ripple_Application.cpp index ff89b7d47..a4e37d3e0 100644 --- a/src/cpp/ripple/ripple_Application.cpp +++ b/src/cpp/ripple/ripple_Application.cpp @@ -974,13 +974,15 @@ void ApplicationImp::updateTables () exit (1); } - if (!theConfig.DB_IMPORT.empty()) + if (theConfig.importNodeDatabase.size () > 0) { + ScopedPointer source (NodeStore::New (theConfig.importNodeDatabase)); + WriteLog (lsWARNING, NodeObject) << - "Node import from '" << theConfig.DB_IMPORT << "' to '" + "Node import from '" << source->getName () << "' to '" << getApp().getNodeStore().getName () << "'."; - getApp().getNodeStore().import(NodeStore::parseDelimitedKeyValueString (theConfig.DB_IMPORT)); + getApp().getNodeStore().import (*source); } } diff --git a/src/cpp/ripple/ripple_Main.cpp b/src/cpp/ripple/ripple_Main.cpp index eaaa83ab6..598962ebf 100644 --- a/src/cpp/ripple/ripple_Main.cpp +++ b/src/cpp/ripple/ripple_Main.cpp @@ -216,6 +216,15 @@ int rippleMain (int argc, char** argv) int iResult = 0; po::variables_map vm; // Map of options. + String importDescription; + { + importDescription << + "Import an existing node database (specified in the " << + "[" << ConfigSection::importNodeDatabase () << "] configuration file section) " + "into the current node database (specified in the " << + "[" << ConfigSection::nodeDatabase () << "] configuration file section). "; + } + // VFALCO TODO Replace boost program options with something from Beast. // // Set up option parsing. @@ -240,7 +249,7 @@ int rippleMain (int argc, char** argv) ("start", "Start from a fresh Ledger.") ("net", "Get the initial ledger from the network.") ("fg", "Run in the foreground.") - ("import", po::value (), "Import old DB into new DB.") + ("import", importDescription.toStdString ().c_str ()) ; // Interpret positional arguments as --parameters. @@ -354,8 +363,14 @@ int rippleMain (int argc, char** argv) if (vm.count ("start")) theConfig.START_UP = Config::FRESH; + // Handle a one-time import option + // if (vm.count ("import")) - theConfig.DB_IMPORT = vm["import"].as (); + { + String const optionString (vm ["import"].as ()); + + theConfig.importNodeDatabase = parseDelimitedKeyValueString (optionString); + } if (vm.count ("ledger")) {