From 02e55f9794092fc034942d69fd387d93e6382af0 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Mon, 22 Jul 2013 16:01:57 -0700 Subject: [PATCH] Change format of config file for NodeStore --- TODO.txt | 3 +- modules/ripple_app/node/ripple_NodeStore.cpp | 83 +++++++++++++++---- modules/ripple_app/node/ripple_NodeStore.h | 32 ++++++- .../ripple_basics/utility/ripple_IniFile.cpp | 34 +++++++- .../ripple_basics/utility/ripple_IniFile.h | 7 ++ .../utility/ripple_StringUtilities.cpp | 38 --------- .../utility/ripple_StringUtilities.h | 4 - .../ripple_core/functional/ripple_Config.cpp | 4 +- .../ripple_core/functional/ripple_Config.h | 8 +- rippled-example.cfg | 14 ++-- src/cpp/ripple/ripple_Application.cpp | 16 ++-- 11 files changed, 160 insertions(+), 83 deletions(-) diff --git a/TODO.txt b/TODO.txt index 4cbeeda45e..517589b43c 100644 --- a/TODO.txt +++ b/TODO.txt @@ -3,7 +3,8 @@ RIPPLE TODO -------------------------------------------------------------------------------- Vinnie's Short List (Changes day to day) -- Refactor Section code into COnfigFile +- Add fast backend to the unit test +- Refactor Section code into ConfigFile - Change NodeStore config file format to multiline key/value pairs - Improved Mutex to track deadlocks - Memory NodeStore::Backend for unit tests [*] diff --git a/modules/ripple_app/node/ripple_NodeStore.cpp b/modules/ripple_app/node/ripple_NodeStore.cpp index ea088a07ad..1ffa2a09f9 100644 --- a/modules/ripple_app/node/ripple_NodeStore.cpp +++ b/modules/ripple_app/node/ripple_NodeStore.cpp @@ -206,12 +206,12 @@ class NodeStoreImp , LeakChecked { public: - NodeStoreImp (String backendParameters, - String fastBackendParameters, + NodeStoreImp (Parameters const& backendParameters, + Parameters const& fastBackendParameters, Scheduler& scheduler) : m_scheduler (scheduler) , m_backend (createBackend (backendParameters, scheduler)) - , m_fastBackend (fastBackendParameters.isNotEmpty () + , m_fastBackend ((fastBackendParameters.size () > 0) ? createBackend (fastBackendParameters, scheduler) : nullptr) , m_cache ("NodeStore", 16384, 300) , m_negativeCache ("NoteStoreNegativeCache", 0, 120) @@ -402,7 +402,7 @@ public: //------------------------------------------------------------------------------ - void import (String sourceBackendParameters) + void import (Parameters const& sourceBackendParameters) { class ImportVisitCallback : public Backend::VisitCallback { @@ -448,13 +448,11 @@ public: //------------------------------------------------------------------------------ - static NodeStore::Backend* createBackend (String const& parameters, Scheduler& scheduler) + static NodeStore::Backend* createBackend (Parameters const& parameters, Scheduler& scheduler) { Backend* backend = nullptr; - StringPairArray keyValues = parseKeyValueParameters (parameters, '|'); - - String const& type = keyValues ["type"]; + String const& type = parameters ["type"]; if (type.isNotEmpty ()) { @@ -471,7 +469,7 @@ public: if (factory != nullptr) { - backend = factory->createInstance (NodeObject::keyBytes, keyValues, scheduler); + backend = factory->createInstance (NodeObject::keyBytes, parameters, scheduler); } else { @@ -486,6 +484,11 @@ public: return backend; } + static NodeStore::Backend* createBackend (String const& parameterString, Scheduler& scheduler) + { + return createBackend (parseDelimitedKeyValueString (parameterString), scheduler); + } + static void addBackendFactory (BackendFactory& factory) { s_factories.add (&factory); @@ -513,13 +516,52 @@ Array NodeStoreImp::s_factories; //------------------------------------------------------------------------------ +NodeStore::Parameters NodeStore::parseDelimitedKeyValueString (String parameters, beast_wchar delimiter) +{ + StringPairArray keyValues; + + while (parameters.isNotEmpty ()) + { + String pair; + + { + int const delimiterPos = parameters.indexOfChar (delimiter); + + if (delimiterPos != -1) + { + pair = parameters.substring (0, delimiterPos); + + parameters = parameters.substring (delimiterPos + 1); + } + else + { + pair = parameters; + + parameters = String::empty; + } + } + + int const equalPos = pair.indexOfChar ('='); + + if (equalPos != -1) + { + String const key = pair.substring (0, equalPos); + String const value = pair.substring (equalPos + 1, pair.length ()); + + keyValues.set (key, value); + } + } + + return keyValues; +} + void NodeStore::addBackendFactory (BackendFactory& factory) { NodeStoreImp::addBackendFactory (factory); } -NodeStore* NodeStore::New (String backendParameters, - String fastBackendParameters, +NodeStore* NodeStore::New (Parameters const& backendParameters, + Parameters const& fastBackendParameters, Scheduler& scheduler) { return new NodeStoreImp (backendParameters, @@ -527,6 +569,15 @@ NodeStore* NodeStore::New (String backendParameters, scheduler); } +NodeStore* NodeStore::New (String const& backendParameters, + String const& fastBackendParameters, + Scheduler& scheduler) +{ + return new NodeStoreImp (parseDelimitedKeyValueString (backendParameters), + parseDelimitedKeyValueString (fastBackendParameters), + scheduler); +} + //============================================================================== // Some common code for the unit tests @@ -924,7 +975,8 @@ public: // Open the backend ScopedPointer backend ( - NodeStoreImp::createBackend (params, m_scheduler)); + NodeStoreImp::createBackend ( + NodeStore::parseDelimitedKeyValueString (params), m_scheduler)); Stopwatch t; @@ -1011,7 +1063,7 @@ public: beginTest (String ("import into '") + destBackendType + "' from '" + srcBackendType + "'"); // Do the import - dest->import (srcParams); + dest->import (NodeStore::parseDelimitedKeyValueString (srcParams)); // Get the results of the import NodeStore::Batch copy; @@ -1026,7 +1078,10 @@ public: void testBackend (String type, int64 const seedValue) { - beginTest (String ("NodeStore backend type=") + type); + String s; + s << String ("NodeStore backend type=") + type; + + beginTest (s); String params; params << "type=" << type diff --git a/modules/ripple_app/node/ripple_NodeStore.h b/modules/ripple_app/node/ripple_NodeStore.h index 78784df66b..4bd44cd1ec 100644 --- a/modules/ripple_app/node/ripple_NodeStore.h +++ b/modules/ripple_app/node/ripple_NodeStore.h @@ -33,6 +33,8 @@ public: typedef std::vector Batch; + typedef StringPairArray Parameters; + //-------------------------------------------------------------------------- /** Parsed key/value blob into NodeObject components. @@ -299,12 +301,26 @@ public: @return A pointer to the Backend object. */ virtual Backend* createInstance (size_t keyBytes, - StringPairArray const& keyValues, + Parameters const& parameters, Scheduler& scheduler) = 0; }; //-------------------------------------------------------------------------- + /** Create a Parameters from a String. + + Parameter strings have the format: + + =['|'=] + + The key "type" must exist, it defines the choice of backend. + For example + `type=LevelDB|path=/mnt/ephemeral` + + This is a convenience function for unit tests. + */ + static Parameters parseDelimitedKeyValueString (String s, beast_wchar delimiter='|'); + /** Construct a node store. Parameter strings have the format: @@ -323,8 +339,16 @@ public: @return A pointer to the created object. */ - static NodeStore* New (String backendParameters, - String fastBackendParameters, + static NodeStore* New (Parameters const& backendParameters, + Parameters const& fastBackendParameters, + Scheduler& scheduler); + + /** Construct a node store from a pipe delimited parameter string. + + This is used for unit tests. + */ + static NodeStore* New (String const& backendParameters, + String const& fastBackendParameters, Scheduler& scheduler); /** Destroy the node store. @@ -386,7 +410,7 @@ public: The other NodeStore database is constructed using the specified backend parameters. */ - virtual void import (String sourceBackendParameters) = 0; + virtual void import (Parameters const& sourceBackendParameters) = 0; /** Retrieve the estimated number of pending write operations. diff --git a/modules/ripple_basics/utility/ripple_IniFile.cpp b/modules/ripple_basics/utility/ripple_IniFile.cpp index 8f60104d83..795b010f6a 100644 --- a/modules/ripple_basics/utility/ripple_IniFile.cpp +++ b/modules/ripple_basics/utility/ripple_IniFile.cpp @@ -106,7 +106,7 @@ int SectionCount (Section& secSource, const std::string& strSection) { Section::mapped_type* pmtEntries = SectionEntries (secSource, strSection); - return pmtEntries ? -1 : pmtEntries->size (); + return pmtEntries ? pmtEntries->size () : 0; } bool SectionSingleB (Section& secSource, const std::string& strSection, std::string& strValue) @@ -128,4 +128,34 @@ bool SectionSingleB (Section& secSource, const std::string& strSection, std::str return bSingle; } -// vim:ts=4 +StringPairArray parseKeyValueSection (Section& secSource, std::string const& strSection) +{ + StringPairArray result; + + int const count = SectionCount (secSource, strSection); + + typedef Section::mapped_type Entries; + + Entries* const entries = SectionEntries (secSource, strSection); + + if (entries != nullptr) + { + for (Entries::const_iterator iter = entries->begin (); iter != entries->end (); ++iter) + { + String const line (iter->c_str ()); + + int const equalPos = line.indexOfChar ('='); + + if (equalPos != -1) + { + String const key = line.substring (0, equalPos); + String const value = line.substring (equalPos + 1, line.length ()); + + result.set (key, value); + } + } + } + + return result; +} + diff --git a/modules/ripple_basics/utility/ripple_IniFile.h b/modules/ripple_basics/utility/ripple_IniFile.h index fe5327ec89..79e7f546bc 100644 --- a/modules/ripple_basics/utility/ripple_IniFile.h +++ b/modules/ripple_basics/utility/ripple_IniFile.h @@ -20,4 +20,11 @@ bool SectionSingleB (Section& secSource, const std::string& strSection, std::str int SectionCount (Section& secSource, const std::string& strSection); Section::mapped_type* SectionEntries (Section& secSource, const std::string& strSection); +/** Parse a section of lines as a key/value array. + + Each line is in the form =. + Spaces are considered part of the key and value. +*/ +StringPairArray parseKeyValueSection (Section& secSource, std::string const& strSection); + #endif diff --git a/modules/ripple_basics/utility/ripple_StringUtilities.cpp b/modules/ripple_basics/utility/ripple_StringUtilities.cpp index cdc438a6ef..bfa42c589e 100644 --- a/modules/ripple_basics/utility/ripple_StringUtilities.cpp +++ b/modules/ripple_basics/utility/ripple_StringUtilities.cpp @@ -271,42 +271,4 @@ std::string addressToString (void const* address) return strHex (static_cast (address) - static_cast (0)); } -StringPairArray parseKeyValueParameters (String parameters, beast_wchar delimiter) -{ - StringPairArray keyValues; - - while (parameters.isNotEmpty ()) - { - String pair; - - { - int const delimiterPos = parameters.indexOfChar (delimiter); - - if (delimiterPos != -1) - { - pair = parameters.substring (0, delimiterPos); - - parameters = parameters.substring (delimiterPos + 1); - } - else - { - pair = parameters; - - parameters = String::empty; - } - } - - int const equalPos = pair.indexOfChar ('='); - - if (equalPos != -1) - { - String const key = pair.substring (0, equalPos); - String const value = pair.substring (equalPos + 1, pair.length ()); - - keyValues.set (key, value); - } - } - - return keyValues; -} diff --git a/modules/ripple_basics/utility/ripple_StringUtilities.h b/modules/ripple_basics/utility/ripple_StringUtilities.h index dd002a2a23..3ddcf75ae9 100644 --- a/modules/ripple_basics/utility/ripple_StringUtilities.h +++ b/modules/ripple_basics/utility/ripple_StringUtilities.h @@ -214,8 +214,4 @@ bool parseUrl (const std::string& strUrl, std::string& strScheme, std::string& s */ extern std::string addressToString (void const* address); -/** Parse a pipe delimited key/value parameter string. -*/ -StringPairArray parseKeyValueParameters (String parameters, beast_wchar delimiter); - #endif diff --git a/modules/ripple_core/functional/ripple_Config.cpp b/modules/ripple_core/functional/ripple_Config.cpp index 896d298280..54d9157a90 100644 --- a/modules/ripple_core/functional/ripple_Config.cpp +++ b/modules/ripple_core/functional/ripple_Config.cpp @@ -373,8 +373,8 @@ void Config::load () (void) SectionSingleB (secConfig, SECTION_RPC_IP, m_rpcIP); (void) SectionSingleB (secConfig, SECTION_RPC_PASSWORD, RPC_PASSWORD); (void) SectionSingleB (secConfig, SECTION_RPC_USER, RPC_USER); - (void) SectionSingleB (secConfig, SECTION_NODE_DB, NODE_DB); - (void) SectionSingleB (secConfig, SECTION_FASTNODE_DB, FASTNODE_DB); + theConfig.nodeDatabase = parseKeyValueSection (secConfig, SECTION_NODE_DB); + theConfig.ephemeralNodeDatabase = parseKeyValueSection (secConfig, SECTION_FASTNODE_DB); if (SectionSingleB (secConfig, SECTION_RPC_PORT, strTemp)) m_rpcPort = boost::lexical_cast (strTemp); diff --git a/modules/ripple_core/functional/ripple_Config.h b/modules/ripple_core/functional/ripple_Config.h index 47ecf9a5b9..05b6a377d4 100644 --- a/modules/ripple_core/functional/ripple_Config.h +++ b/modules/ripple_core/functional/ripple_Config.h @@ -84,8 +84,12 @@ public: boost::filesystem::path DATA_DIR; boost::filesystem::path DEBUG_LOGFILE; boost::filesystem::path VALIDATORS_FILE; // As specifed in rippled.cfg. - std::string NODE_DB; // Database to use for nodes - std::string FASTNODE_DB; // Database for temporary storage + + StringPairArray nodeDatabase; + StringPairArray ephemeralNodeDatabase; + //std::string NODE_DB; // Database to use for nodes + //std::string FASTNODE_DB; // Database for temporary storage + std::string DB_IMPORT; // Import from old DB bool ELB_SUPPORT; // Support Amazon ELB diff --git a/rippled-example.cfg b/rippled-example.cfg index 2c89e205ae..6992020723 100644 --- a/rippled-example.cfg +++ b/rippled-example.cfg @@ -230,10 +230,13 @@ # Set the choice of databases for storing Node objects. # # Format (without spaces): -# '=' [ '|' '=' ]... +# One or more lines of key / value pairs: +# '=' +# ... # -# Examples: -# type=HyperLevelDB|path=db/hashnode +# Example: +# type=HyperLevelDB +# path=db/hashnode # # Choices for 'type' (not case-sensitive) # HyperLevelDB Use an improved version of LevelDB (preferred) @@ -327,8 +330,9 @@ [node_size] medium -[node_db] -type=mdb|path=db +#[node_db] +#type=HyperLevelDB +#path=hyperldb [debug_logfile] log/debug.log diff --git a/src/cpp/ripple/ripple_Application.cpp b/src/cpp/ripple/ripple_Application.cpp index bbef763057..ff89b7d47e 100644 --- a/src/cpp/ripple/ripple_Application.cpp +++ b/src/cpp/ripple/ripple_Application.cpp @@ -52,8 +52,8 @@ public: , mJobQueue (mIOService) // VFALCO New stuff , m_nodeStore (NodeStore::New ( - theConfig.NODE_DB, - theConfig.FASTNODE_DB, + theConfig.nodeDatabase, + theConfig.ephemeralNodeDatabase, *this)) , m_validators (Validators::New (this)) , mFeatures (IFeatures::New (2 * 7 * 24 * 60 * 60, 200)) // two weeks, 200/256 @@ -955,15 +955,9 @@ static void addTxnSeqField () void ApplicationImp::updateTables () { - if (theConfig.NODE_DB.empty ()) + if (theConfig.nodeDatabase.size () <= 0) { - Log (lsFATAL) << "The NODE_DB configuration setting MUST be set"; - StopSustain (); - exit (1); - } - else if (theConfig.NODE_DB == "LevelDB" || theConfig.NODE_DB == "SQLite") - { - Log (lsFATAL) << "The NODE_DB setting has been updated, your value is out of date"; + Log (lsFATAL) << "The [node_db] configuration setting has been updated and must be set"; StopSustain (); exit (1); } @@ -986,7 +980,7 @@ void ApplicationImp::updateTables () "Node import from '" << theConfig.DB_IMPORT << "' to '" << getApp().getNodeStore().getName () << "'."; - getApp().getNodeStore().import(theConfig.DB_IMPORT); + getApp().getNodeStore().import(NodeStore::parseDelimitedKeyValueString (theConfig.DB_IMPORT)); } }