Replaces StringPairArray with Section in Config.

This commit is contained in:
seelabs
2015-03-06 15:32:20 -08:00
committed by Tom Ritchford
parent 8f88d915ba
commit 6b9e842ddd
28 changed files with 165 additions and 281 deletions

View File

@@ -20,6 +20,7 @@
#include <BeastConfig.h> #include <BeastConfig.h>
#include <ripple/basics/StringUtilities.h> #include <ripple/basics/StringUtilities.h>
#include <ripple/app/data/Database.h> #include <ripple/app/data/Database.h>
#include <cstring>
namespace ripple { namespace ripple {

View File

@@ -20,6 +20,7 @@
#include <BeastConfig.h> #include <BeastConfig.h>
#include <ripple/app/data/DatabaseCon.h> #include <ripple/app/data/DatabaseCon.h>
#include <ripple/app/data/SqliteDatabase.h> #include <ripple/app/data/SqliteDatabase.h>
#include <ripple/core/ConfigSections.h>
namespace ripple { namespace ripple {
@@ -54,8 +55,8 @@ setup_DatabaseCon (Config const& c)
{ {
DatabaseCon::Setup setup; DatabaseCon::Setup setup;
if (c.nodeDatabase["online_delete"].isNotEmpty()) auto const& sec = c.section (ConfigSection::nodeDatabase ());
setup.onlineDelete = c.nodeDatabase["online_delete"].getIntValue(); get_if_exists (sec, "online_delete", setup.onlineDelete);
setup.startUp = c.START_UP; setup.startUp = c.START_UP;
setup.standAlone = c.RUN_STANDALONE; setup.standAlone = c.RUN_STANDALONE;
setup.dataDir = c.legacy ("database_path"); setup.dataDir = c.legacy ("database_path");

View File

@@ -337,7 +337,7 @@ public:
m_logs.journal("TaggedCache")) m_logs.journal("TaggedCache"))
, m_collectorManager (CollectorManager::New ( , m_collectorManager (CollectorManager::New (
getConfig().insightSettings, m_logs.journal("Collector"))) getConfig().section (SECTION_INSIGHT), m_logs.journal("Collector")))
, family_ (*m_nodeStore, *m_collectorManager) , family_ (*m_nodeStore, *m_collectorManager)
@@ -1457,7 +1457,7 @@ static void addTxnSeqField ()
void ApplicationImp::updateTables () void ApplicationImp::updateTables ()
{ {
if (getConfig ().nodeDatabase.size () <= 0) if (getConfig ().section (ConfigSection::nodeDatabase ()).empty ())
{ {
WriteLog (lsFATAL, Application) << "The [node_db] configuration setting has been updated and must be set"; WriteLog (lsFATAL, Application) << "The [node_db] configuration setting has been updated and must be set";
exitWithCode(1); exitWithCode(1);
@@ -1480,7 +1480,7 @@ void ApplicationImp::updateTables ()
std::unique_ptr <NodeStore::Database> source = std::unique_ptr <NodeStore::Database> source =
NodeStore::Manager::instance().make_Database ("NodeStore.import", scheduler, NodeStore::Manager::instance().make_Database ("NodeStore.import", scheduler,
deprecatedLogs().journal("NodeObject"), 0, deprecatedLogs().journal("NodeObject"), 0,
getConfig ().importNodeDatabase); getConfig ()[ConfigSection::importNodeDatabase ()]);
WriteLog (lsWARNING, NodeObject) << WriteLog (lsWARNING, NodeObject) <<
"Node import from '" << source->getName () << "' to '" "Node import from '" << source->getName () << "' to '"

View File

@@ -30,17 +30,17 @@ public:
beast::insight::Collector::ptr m_collector; beast::insight::Collector::ptr m_collector;
std::unique_ptr <beast::insight::Groups> m_groups; std::unique_ptr <beast::insight::Groups> m_groups;
CollectorManagerImp (beast::StringPairArray const& params, CollectorManagerImp (Section const& params,
beast::Journal journal) beast::Journal journal)
: m_journal (journal) : m_journal (journal)
{ {
std::string const& server (params ["server"].toStdString()); std::string const& server = get<std::string> (params, "server");
if (server == "statsd") if (server == "statsd")
{ {
beast::IP::Endpoint const address (beast::IP::Endpoint::from_string ( beast::IP::Endpoint const address (beast::IP::Endpoint::from_string (
params ["address"].toStdString ())); get<std::string> (params, "address")));
std::string const& prefix (params ["prefix"].toStdString ()); std::string const& prefix (get<std::string> (params, "prefix"));
m_collector = beast::insight::StatsDCollector::New (address, prefix, journal); m_collector = beast::insight::StatsDCollector::New (address, prefix, journal);
} }
@@ -73,7 +73,7 @@ CollectorManager::~CollectorManager ()
{ {
} }
CollectorManager* CollectorManager::New (beast::StringPairArray const& params, CollectorManager* CollectorManager::New (Section const& params,
beast::Journal journal) beast::Journal journal)
{ {
return new CollectorManagerImp (params, journal); return new CollectorManagerImp (params, journal);

View File

@@ -20,7 +20,7 @@
#ifndef RIPPLE_APP_MAIN_COLLECTORMANAGER_H_INCLUDED #ifndef RIPPLE_APP_MAIN_COLLECTORMANAGER_H_INCLUDED
#define RIPPLE_APP_MAIN_COLLECTORMANAGER_H_INCLUDED #define RIPPLE_APP_MAIN_COLLECTORMANAGER_H_INCLUDED
#include <beast/module/core/text/StringPairArray.h> #include <ripple/basics/BasicConfig.h>
#include <beast/Insight.h> #include <beast/Insight.h>
namespace ripple { namespace ripple {
@@ -29,7 +29,7 @@ namespace ripple {
class CollectorManager class CollectorManager
{ {
public: public:
static CollectorManager* New (beast::StringPairArray const& params, static CollectorManager* New (Section const& params,
beast::Journal journal); beast::Journal journal);
virtual ~CollectorManager () = 0; virtual ~CollectorManager () = 0;
virtual beast::insight::Collector::ptr const& collector () = 0; virtual beast::insight::Collector::ptr const& collector () = 0;

View File

@@ -163,9 +163,11 @@ static
void void
setupConfigForUnitTests (Config* config) setupConfigForUnitTests (Config* config)
{ {
config->nodeDatabase = parseDelimitedKeyValueString ("type=memory|path=main"); config->overwrite (ConfigSection::nodeDatabase (), "type", "memory");
config->ephemeralNodeDatabase = beast::StringPairArray (); config->overwrite (ConfigSection::nodeDatabase (), "path", "main");
config->importNodeDatabase = beast::StringPairArray ();
config->deprecatedClearSection (ConfigSection::tempNodeDatabase ());
config->deprecatedClearSection (ConfigSection::importNodeDatabase ());
} }
static int runShutdownTests () static int runShutdownTests ()

View File

@@ -43,8 +43,8 @@ public:
std::uint32_t deleteInterval = 0; std::uint32_t deleteInterval = 0;
bool advisoryDelete = false; bool advisoryDelete = false;
std::uint32_t ledgerHistory = 0; std::uint32_t ledgerHistory = 0;
beast::StringPairArray nodeDatabase; Section nodeDatabase;
beast::StringPairArray ephemeralNodeDatabase; Section ephemeralNodeDatabase;
std::string databasePath; std::string databasePath;
std::uint32_t deleteBatch = 100; std::uint32_t deleteBatch = 100;
std::uint32_t backOff = 100; std::uint32_t backOff = 100;

View File

@@ -21,6 +21,7 @@
#include <ripple/app/misc/SHAMapStoreImp.h> #include <ripple/app/misc/SHAMapStoreImp.h>
#include <ripple/app/ledger/LedgerMaster.h> #include <ripple/app/ledger/LedgerMaster.h>
#include <ripple/app/main/Application.h> #include <ripple/app/main/Application.h>
#include <ripple/core/ConfigSections.h>
#include <boost/format.hpp> #include <boost/format.hpp>
#include <beast/cxx14/memory.h> // <memory> #include <beast/cxx14/memory.h> // <memory>
@@ -437,7 +438,7 @@ void
SHAMapStoreImp::dbPaths() SHAMapStoreImp::dbPaths()
{ {
boost::filesystem::path dbPath = boost::filesystem::path dbPath =
setup_.nodeDatabase["path"].toStdString(); get<std::string>(setup_.nodeDatabase, "path");
if (boost::filesystem::exists (dbPath)) if (boost::filesystem::exists (dbPath))
{ {
@@ -488,7 +489,7 @@ SHAMapStoreImp::dbPaths()
<< "remove the files matching " << "remove the files matching "
<< stateDbPathName.string() << stateDbPathName.string()
<< " and contents of the directory " << " and contents of the directory "
<< setup_.nodeDatabase["path"].toStdString() << get<std::string>(setup_.nodeDatabase, "path")
<< std::endl; << std::endl;
throw std::runtime_error ("state db error"); throw std::runtime_error ("state db error");
@@ -499,7 +500,7 @@ std::shared_ptr <NodeStore::Backend>
SHAMapStoreImp::makeBackendRotating (std::string path) SHAMapStoreImp::makeBackendRotating (std::string path)
{ {
boost::filesystem::path newPath; boost::filesystem::path newPath;
NodeStore::Parameters parameters = setup_.nodeDatabase; Section parameters = setup_.nodeDatabase;
if (path.size()) if (path.size())
{ {
@@ -507,7 +508,7 @@ SHAMapStoreImp::makeBackendRotating (std::string path)
} }
else else
{ {
boost::filesystem::path p = parameters["path"].toStdString(); boost::filesystem::path p = get<std::string>(parameters, "path");
p /= dbPrefix_; p /= dbPrefix_;
p += ".%%%%"; p += ".%%%%";
newPath = boost::filesystem::unique_path (p); newPath = boost::filesystem::unique_path (p);
@@ -693,20 +694,20 @@ setup_SHAMapStore (Config const& c)
{ {
SHAMapStore::Setup setup; SHAMapStore::Setup setup;
if (c.nodeDatabase["online_delete"].isNotEmpty()) auto const& sec = c.section (ConfigSection::nodeDatabase ());
setup.deleteInterval = c.nodeDatabase["online_delete"].getIntValue(); get_if_exists (sec, "online_delete", setup.deleteInterval);
if (c.nodeDatabase["advisory_delete"].isNotEmpty() && setup.deleteInterval)
setup.advisoryDelete = c.nodeDatabase["advisory_delete"].getIntValue(); if (setup.deleteInterval)
get_if_exists (sec, "advisory_delete", setup.advisoryDelete);
setup.ledgerHistory = c.LEDGER_HISTORY; setup.ledgerHistory = c.LEDGER_HISTORY;
setup.nodeDatabase = c.nodeDatabase; setup.nodeDatabase = c[ConfigSection::nodeDatabase ()];
setup.ephemeralNodeDatabase = c.ephemeralNodeDatabase; setup.ephemeralNodeDatabase = c[ConfigSection::tempNodeDatabase ()];
setup.databasePath = c.legacy("database_path"); setup.databasePath = c.legacy("database_path");
if (c.nodeDatabase["delete_batch"].isNotEmpty())
setup.deleteBatch = c.nodeDatabase["delete_batch"].getIntValue(); get_if_exists (sec, "delete_batch", setup.deleteBatch);
if (c.nodeDatabase["backOff"].isNotEmpty()) get_if_exists (sec, "backOff", setup.backOff);
setup.backOff = c.nodeDatabase["backOff"].getIntValue(); get_if_exists (sec, "age_threshold", setup.ageThreshold);
if (c.nodeDatabase["age_threshold"].isNotEmpty())
setup.ageThreshold = c.nodeDatabase["age_threshold"].getIntValue();
return setup; return setup;
} }

View File

@@ -249,11 +249,11 @@ public:
} }
std::unique_ptr <NodeStore::Backend> createInstance ( std::unique_ptr <NodeStore::Backend> createInstance (
size_t, NodeStore::Parameters const& keyValues, size_t, Section const& keyValues,
NodeStore::Scheduler&, beast::Journal) NodeStore::Scheduler&, beast::Journal)
{ {
return std::make_unique <SqliteBackend> ( return std::make_unique <SqliteBackend> (
keyValues ["path"].toStdString (), get<std::string>(keyValues, "path"),
getConfig ().getSize(siHashNodeDBCache) * 1024); getConfig ().getSize(siHashNodeDBCache) * 1024);
} }
}; };

View File

@@ -182,6 +182,11 @@ public:
overwrite (std::string const& section, std::string const& key, overwrite (std::string const& section, std::string const& key,
std::string const& value); std::string const& value);
/** Remove all the key/value pairs from the section.
*/
void
deprecatedClearSection (std::string const& section);
/** /**
* Set a value that is not a key/value pair. * Set a value that is not a key/value pair.
* *
@@ -232,7 +237,7 @@ set (T& target, std::string const& name, Section const& section)
target = boost::lexical_cast <T> (result.first); target = boost::lexical_cast <T> (result.first);
return true; return true;
} }
catch(...) catch (boost::bad_lexical_cast&)
{ {
} }
return false; return false;
@@ -256,7 +261,7 @@ set (T& target, T const& defaultValue,
target = boost::lexical_cast <T> (result.first); target = boost::lexical_cast <T> (result.first);
return true; return true;
} }
catch(...) catch (boost::bad_lexical_cast&)
{ {
target = defaultValue; target = defaultValue;
} }
@@ -280,12 +285,46 @@ get (Section const& section,
{ {
return boost::lexical_cast <T> (result.first); return boost::lexical_cast <T> (result.first);
} }
catch(...) catch (boost::bad_lexical_cast&)
{ {
} }
return defaultValue; return defaultValue;
} }
template <class T>
bool
get_if_exists (Section const& section,
std::string const& name, T& v)
{
auto const result = section.find (name);
if (! result.second)
return false;
try
{
v = boost::lexical_cast <T> (result.first);
return true;
}
catch (boost::bad_lexical_cast&)
{
}
return false;
}
template <>
inline
bool
get_if_exists<bool> (Section const& section,
std::string const& name, bool& v)
{
int intVal = 0;
if (get_if_exists (section, name, intVal))
{
v = bool (intVal);
return true;
}
return false;
}
} // ripple } // ripple
#endif #endif

View File

@@ -24,7 +24,6 @@
#include <ripple/basics/Blob.h> #include <ripple/basics/Blob.h>
#include <ripple/basics/strHex.h> #include <ripple/basics/strHex.h>
#include <boost/format.hpp> #include <boost/format.hpp>
#include <beast/module/core/text/StringPairArray.h>
#include <sstream> #include <sstream>
#include <string> #include <string>
@@ -96,16 +95,6 @@ bool parseIpPort (std::string const& strSource, std::string& strIP, int& iPort);
bool parseUrl (std::string const& strUrl, std::string& strScheme, bool parseUrl (std::string const& strUrl, std::string& strScheme,
std::string& strDomain, int& iPort, std::string& strPath); std::string& strDomain, int& iPort, std::string& strPath);
/** Create a Parameters from a String.
Parameter strings have the format:
<key>=<value>['|'<key>=<value>]
*/
extern beast::StringPairArray
parseDelimitedKeyValueString (
beast::String s, beast::beast_wchar delimiter='|');
} // ripple } // ripple
#endif #endif

View File

@@ -115,6 +115,14 @@ BasicConfig::overwrite (std::string const& section, std::string const& key,
result.first->second.set (key, value); result.first->second.set (key, value);
} }
void
BasicConfig::deprecatedClearSection (std::string const& section)
{
auto i = map_.find(section);
if (i != map_.end())
i->second = Section(section);
}
void void
BasicConfig::legacy(std::string const& section, std::string value) BasicConfig::legacy(std::string const& section, std::string value)
{ {

View File

@@ -221,45 +221,4 @@ bool parseUrl (std::string const& strUrl, std::string& strScheme, std::string& s
return bMatch; return bMatch;
} }
beast::StringPairArray parseDelimitedKeyValueString (beast::String parameters,
beast::beast_wchar delimiter)
{
beast::StringPairArray keyValues;
while (parameters.isNotEmpty ())
{
beast::String pair;
{
int const delimiterPos = parameters.indexOfChar (delimiter);
if (delimiterPos != -1)
{
pair = parameters.substring (0, delimiterPos);
parameters = parameters.substring (delimiterPos + 1);
}
else
{
pair = parameters;
parameters = beast::String::empty;
}
}
int const equalPos = pair.indexOfChar ('=');
if (equalPos != -1)
{
beast::String const key = pair.substring (0, equalPos);
beast::String const value = pair.substring (equalPos + 1, pair.length ());
keyValues.set (key, value);
}
}
return keyValues;
}
} // ripple } // ripple

View File

@@ -27,7 +27,6 @@
#include <beast/http/URL.h> #include <beast/http/URL.h>
#include <beast/net/IPEndpoint.h> #include <beast/net/IPEndpoint.h>
#include <beast/module/core/files/File.h> #include <beast/module/core/files/File.h>
#include <beast/module/core/text/StringPairArray.h>
#include <beast/utility/ci_char_traits.h> #include <beast/utility/ci_char_traits.h>
#include <boost/filesystem.hpp> #include <boost/filesystem.hpp>
#include <boost/lexical_cast.hpp> #include <boost/lexical_cast.hpp>
@@ -52,14 +51,6 @@ countSectionEntries (IniFileSections& secSource, std::string const& strSection);
IniFileSections::mapped_type* IniFileSections::mapped_type*
getIniFileSection (IniFileSections& secSource, std::string const& strSection); getIniFileSection (IniFileSections& secSource, std::string const& strSection);
/** Parse a section of lines as a key/value array.
Each line is in the form <key>=<value>.
Spaces are considered part of the key and value.
*/
// DEPRECATED
beast::StringPairArray
parseKeyValueSection (IniFileSections& secSource, std::string const& strSection);
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
enum SizedItemName enum SizedItemName
@@ -164,40 +155,7 @@ public:
//-------------------------------------------------------------------------- //--------------------------------------------------------------------------
/** Parameters for the insight collection module */
beast::StringPairArray insightSettings;
/** Parameters for the main NodeStore database.
This is 1 or more strings of the form <key>=<value>
The 'type' and 'path' keys are required, see rippled-example.cfg
@see Database
*/
beast::StringPairArray nodeDatabase;
/** Parameters for the ephemeral NodeStore database.
This is an auxiliary database for the NodeStore, usually placed
on a separate faster volume. However, the volume data may not persist
between launches. Use of the ephemeral database is optional.
The format is the same as that for @ref nodeDatabase
@see Database
*/
beast::StringPairArray ephemeralNodeDatabase;
/** Parameters for importing an old database in to the current node database.
If this is not empty, then it specifies the key/value parameters for
another node database from which to import all data into the current
node database specified by @ref nodeDatabase.
The format of this string is in the form:
<key>'='<value>['|'<key>'='value]
@see parseDelimitedKeyValueString
*/
bool doImport; bool doImport;
beast::StringPairArray importNodeDatabase;
// //
// //

View File

@@ -147,36 +147,6 @@ bool getSingleSection (IniFileSections& secSource,
return bSingle; return bSingle;
} }
beast::StringPairArray
parseKeyValueSection (IniFileSections& secSource, std::string const& strSection)
{
beast::StringPairArray result;
typedef IniFileSections::mapped_type Entries;
Entries* const entries = getIniFileSection (secSource, strSection);
if (entries != nullptr)
{
for (Entries::const_iterator iter = entries->begin ();
iter != entries->end (); ++iter)
{
std::string const line (iter->c_str ());
int const equalPos = line.find ('=');
if (equalPos != std::string::npos)
{
result.set (
line.substr (0, equalPos),
line.substr (equalPos + 1));
}
}
}
return result;
}
/** Parses a set of strings into IP::Endpoint /** Parses a set of strings into IP::Endpoint
Strings which fail to parse are not included in the output. If a stream is Strings which fail to parse are not included in the output. If a stream is
provided, human readable diagnostic error messages are written for each provided, human readable diagnostic error messages are written for each
@@ -469,17 +439,6 @@ void Config::loadFromString (std::string const& fileContents)
parsedAddresses.cbegin (), parsedAddresses.cend ()); parsedAddresses.cbegin (), parsedAddresses.cend ());
} }
insightSettings = parseKeyValueSection (secConfig, SECTION_INSIGHT);
nodeDatabase = parseKeyValueSection (
secConfig, ConfigSection::nodeDatabase ());
ephemeralNodeDatabase = parseKeyValueSection (
secConfig, ConfigSection::tempNodeDatabase ());
importNodeDatabase = parseKeyValueSection (
secConfig, ConfigSection::importNodeDatabase ());
if (getSingleSection (secConfig, SECTION_NODE_SIZE, strTemp)) if (getSingleSection (secConfig, SECTION_NODE_SIZE, strTemp))
{ {
if (strTemp == "tiny") if (strTemp == "tiny")

View File

@@ -48,7 +48,7 @@ public:
*/ */
virtual virtual
std::unique_ptr <Backend> std::unique_ptr <Backend>
createInstance (size_t keyBytes, Parameters const& parameters, createInstance (size_t keyBytes, Section const& parameters,
Scheduler& scheduler, beast::Journal journal) = 0; Scheduler& scheduler, beast::Journal journal) = 0;
}; };

View File

@@ -56,7 +56,7 @@ public:
/** Create a backend. */ /** Create a backend. */
virtual virtual
std::unique_ptr <Backend> std::unique_ptr <Backend>
make_Backend (Parameters const& parameters, make_Backend (Section const& parameters,
Scheduler& scheduler, beast::Journal journal) = 0; Scheduler& scheduler, beast::Journal journal) = 0;
/** Construct a NodeStore database. /** Construct a NodeStore database.
@@ -87,8 +87,8 @@ public:
std::unique_ptr <Database> std::unique_ptr <Database>
make_Database (std::string const& name, Scheduler& scheduler, make_Database (std::string const& name, Scheduler& scheduler,
beast::Journal journal, int readThreads, beast::Journal journal, int readThreads,
Parameters const& backendParameters, Section const& backendParameters,
Parameters fastBackendParameters = Parameters()) = 0; Section fastBackendParameters = Section()) = 0;
virtual virtual
std::unique_ptr <DatabaseRotating> std::unique_ptr <DatabaseRotating>

View File

@@ -21,7 +21,7 @@
#define RIPPLE_NODESTORE_TYPES_H_INCLUDED #define RIPPLE_NODESTORE_TYPES_H_INCLUDED
#include <ripple/nodestore/NodeObject.h> #include <ripple/nodestore/NodeObject.h>
#include <beast/module/core/text/StringPairArray.h> #include <ripple/basics/BasicConfig.h>
#include <vector> #include <vector>
namespace ripple { namespace ripple {
@@ -48,11 +48,6 @@ enum Status
/** A batch of NodeObjects to write at once. */ /** A batch of NodeObjects to write at once. */
typedef std::vector <NodeObject::Ptr> Batch; typedef std::vector <NodeObject::Ptr> Batch;
/** A list of key/value parameter pairs passed to the backend. */
// VFALCO TODO Use std::string, pair, vector
typedef beast::StringPairArray Parameters;
} }
} }

View File

@@ -51,7 +51,7 @@ public:
std::unique_ptr <Backend> std::unique_ptr <Backend>
createInstance ( createInstance (
size_t keyBytes, size_t keyBytes,
Parameters const& keyValues, Section const& keyValues,
Scheduler& scheduler, Scheduler& scheduler,
beast::Journal journal); beast::Journal journal);
@@ -82,9 +82,9 @@ private:
MemoryDB* db_; MemoryDB* db_;
public: public:
MemoryBackend (size_t keyBytes, Parameters const& keyValues, MemoryBackend (size_t keyBytes, Section const& keyValues,
Scheduler& scheduler, beast::Journal journal) Scheduler& scheduler, beast::Journal journal)
: name_ (keyValues ["path"].toStdString ()) : name_ (get<std::string>(keyValues, "path"))
, journal_ (journal) , journal_ (journal)
{ {
if (name_.empty()) if (name_.empty())
@@ -187,7 +187,7 @@ MemoryFactory::getName() const
std::unique_ptr <Backend> std::unique_ptr <Backend>
MemoryFactory::createInstance ( MemoryFactory::createInstance (
size_t keyBytes, size_t keyBytes,
Parameters const& keyValues, Section const& keyValues,
Scheduler& scheduler, Scheduler& scheduler,
beast::Journal journal) beast::Journal journal)
{ {

View File

@@ -62,11 +62,11 @@ public:
std::atomic <bool> deletePath_; std::atomic <bool> deletePath_;
Scheduler& scheduler_; Scheduler& scheduler_;
NuDBBackend (int keyBytes, Parameters const& keyValues, NuDBBackend (int keyBytes, Section const& keyValues,
Scheduler& scheduler, beast::Journal journal) Scheduler& scheduler, beast::Journal journal)
: journal_ (journal) : journal_ (journal)
, keyBytes_ (keyBytes) , keyBytes_ (keyBytes)
, name_ (keyValues ["path"].toStdString ()) , name_ (get<std::string>(keyValues, "path"))
, deletePath_(false) , deletePath_(false)
, scheduler_ (scheduler) , scheduler_ (scheduler)
{ {
@@ -259,7 +259,7 @@ public:
std::unique_ptr <Backend> std::unique_ptr <Backend>
createInstance ( createInstance (
size_t keyBytes, size_t keyBytes,
Parameters const& keyValues, Section const& keyValues,
Scheduler& scheduler, Scheduler& scheduler,
beast::Journal journal) beast::Journal journal)
{ {

View File

@@ -111,7 +111,7 @@ public:
std::unique_ptr <Backend> std::unique_ptr <Backend>
createInstance ( createInstance (
size_t, size_t,
Parameters const&, Section const&,
Scheduler&, beast::Journal) Scheduler&, beast::Journal)
{ {
return std::make_unique <NullBackend> (); return std::make_unique <NullBackend> ();

View File

@@ -99,16 +99,15 @@ public:
std::string m_name; std::string m_name;
std::unique_ptr <rocksdb::DB> m_db; std::unique_ptr <rocksdb::DB> m_db;
RocksDBBackend (int keyBytes, Parameters const& keyValues, RocksDBBackend (int keyBytes, Section const& keyValues,
Scheduler& scheduler, beast::Journal journal, RocksDBEnv* env) Scheduler& scheduler, beast::Journal journal, RocksDBEnv* env)
: m_deletePath (false) : m_deletePath (false)
, m_journal (journal) , m_journal (journal)
, m_keyBytes (keyBytes) , m_keyBytes (keyBytes)
, m_scheduler (scheduler) , m_scheduler (scheduler)
, m_batch (*this, scheduler) , m_batch (*this, scheduler)
, m_name (keyValues ["path"].toStdString ())
{ {
if (m_name.empty()) if (!get_if_exists(keyValues, "path", m_name))
throw std::runtime_error ("Missing path in RocksDBFactory backend"); throw std::runtime_error ("Missing path in RocksDBFactory backend");
rocksdb::Options options; rocksdb::Options options;
@@ -116,51 +115,45 @@ public:
options.create_if_missing = true; options.create_if_missing = true;
options.env = env; options.env = env;
if (keyValues["cache_mb"].isEmpty()) if (!keyValues.exists ("cache_mb"))
{ {
table_options.block_cache = rocksdb::NewLRUCache (getConfig ().getSize (siHashNodeDBCache) * 1024 * 1024); table_options.block_cache = rocksdb::NewLRUCache (getConfig ().getSize (siHashNodeDBCache) * 1024 * 1024);
} }
else else
{ {
table_options.block_cache = rocksdb::NewLRUCache (keyValues["cache_mb"].getIntValue() * 1024L * 1024L); table_options.block_cache = rocksdb::NewLRUCache (get<int>(keyValues, "cache_mb") * 1024L * 1024L);
} }
if (keyValues["filter_bits"].isEmpty()) if (!keyValues.exists ("filter_bits"))
{ {
if (getConfig ().NODE_SIZE >= 2) if (getConfig ().NODE_SIZE >= 2)
table_options.filter_policy.reset (rocksdb::NewBloomFilterPolicy (10)); table_options.filter_policy.reset (rocksdb::NewBloomFilterPolicy (10));
} }
else if (keyValues["filter_bits"].getIntValue() != 0) else if (auto const v = get<int>(keyValues, "filter_bits"))
{ {
table_options.filter_policy.reset (rocksdb::NewBloomFilterPolicy (keyValues["filter_bits"].getIntValue())); table_options.filter_policy.reset (rocksdb::NewBloomFilterPolicy (v));
} }
if (! keyValues["open_files"].isEmpty()) get_if_exists (keyValues, "open_files", options.max_open_files);
{
options.max_open_files = keyValues["open_files"].getIntValue();
}
if (! keyValues["file_size_mb"].isEmpty()) if (keyValues.exists ("file_size_mb"))
{ {
options.target_file_size_base = 1024 * 1024 * keyValues["file_size_mb"].getIntValue(); options.target_file_size_base = 1024 * 1024 * get<int>(keyValues,"file_size_mb");
options.max_bytes_for_level_base = 5 * options.target_file_size_base; options.max_bytes_for_level_base = 5 * options.target_file_size_base;
options.write_buffer_size = 2 * options.target_file_size_base; options.write_buffer_size = 2 * options.target_file_size_base;
} }
if (! keyValues["file_size_mult"].isEmpty()) get_if_exists (keyValues, "file_size_mult", options.target_file_size_multiplier);
{
options.target_file_size_multiplier = keyValues["file_size_mult"].getIntValue();
}
if (! keyValues["bg_threads"].isEmpty()) if (keyValues.exists ("bg_threads"))
{ {
options.env->SetBackgroundThreads options.env->SetBackgroundThreads
(keyValues["bg_threads"].getIntValue(), rocksdb::Env::LOW); (get<int>(keyValues, "bg_threads"), rocksdb::Env::LOW);
} }
if (! keyValues["high_threads"].isEmpty()) if (keyValues.exists ("high_threads"))
{ {
auto const highThreads = keyValues["high_threads"].getIntValue(); auto const highThreads = get<int>(keyValues, "high_threads");
options.env->SetBackgroundThreads (highThreads, rocksdb::Env::HIGH); options.env->SetBackgroundThreads (highThreads, rocksdb::Env::HIGH);
// If we have high-priority threads, presumably we want to // If we have high-priority threads, presumably we want to
@@ -169,29 +162,22 @@ public:
options.max_background_flushes = highThreads; options.max_background_flushes = highThreads;
} }
if (! keyValues["compression"].isEmpty ()) if (keyValues.exists ("compression") &&
{ (get<int>(keyValues, "compression") == 0))
if (keyValues["compression"].getIntValue () == 0)
{ {
options.compression = rocksdb::kNoCompression; options.compression = rocksdb::kNoCompression;
} }
}
if (! keyValues["block_size"].isEmpty ()) get_if_exists (keyValues, "block_size", table_options.block_size);
{
table_options.block_size = keyValues["block_size"].getIntValue ();
}
if (! keyValues["universal_compaction"].isEmpty ()) if (keyValues.exists ("universal_compaction") &&
{ (get<int>(keyValues, "universal_compaction") != 0))
if (keyValues["universal_compaction"].getIntValue () != 0)
{ {
options.compaction_style = rocksdb::kCompactionStyleUniversal; options.compaction_style = rocksdb::kCompactionStyleUniversal;
options.min_write_buffer_number_to_merge = 2; options.min_write_buffer_number_to_merge = 2;
options.max_write_buffer_number = 6; options.max_write_buffer_number = 6;
options.write_buffer_size = 6 * options.target_file_size_base; options.write_buffer_size = 6 * options.target_file_size_base;
} }
}
options.table_factory.reset(NewBlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
@@ -400,7 +386,7 @@ public:
std::unique_ptr <Backend> std::unique_ptr <Backend>
createInstance ( createInstance (
size_t keyBytes, size_t keyBytes,
Parameters const& keyValues, Section const& keyValues,
Scheduler& scheduler, Scheduler& scheduler,
beast::Journal journal) beast::Journal journal)
{ {

View File

@@ -95,12 +95,12 @@ public:
std::string m_name; std::string m_name;
std::unique_ptr <rocksdb::DB> m_db; std::unique_ptr <rocksdb::DB> m_db;
RocksDBQuickBackend (int keyBytes, Parameters const& keyValues, RocksDBQuickBackend (int keyBytes, Section const& keyValues,
Scheduler& scheduler, beast::Journal journal, RocksDBQuickEnv* env) Scheduler& scheduler, beast::Journal journal, RocksDBQuickEnv* env)
: m_deletePath (false) : m_deletePath (false)
, m_journal (journal) , m_journal (journal)
, m_keyBytes (keyBytes) , m_keyBytes (keyBytes)
, m_name (keyValues ["path"].toStdString ()) , m_name (get<std::string>(keyValues, "path"))
{ {
if (m_name.empty()) if (m_name.empty())
throw std::runtime_error ("Missing path in RocksDBQuickFactory backend"); throw std::runtime_error ("Missing path in RocksDBQuickFactory backend");
@@ -110,14 +110,9 @@ public:
std::string style("level"); std::string style("level");
std::uint64_t threads=4; std::uint64_t threads=4;
if (!keyValues["budget"].isEmpty()) get_if_exists (keyValues, "budget", budget);
budget = keyValues["budget"].getIntValue(); get_if_exists (keyValues, "style", style);
get_if_exists (keyValues, "threads", threads);
if (!keyValues["style"].isEmpty())
style = keyValues["style"].toStdString();
if (!keyValues["threads"].isEmpty())
threads = keyValues["threads"].getIntValue();
// Set options // Set options
@@ -163,18 +158,11 @@ public:
// options.memtable_factory.reset( // options.memtable_factory.reset(
// rocksdb::NewHashCuckooRepFactory(options.write_buffer_size)); // rocksdb::NewHashCuckooRepFactory(options.write_buffer_size));
if (! keyValues["open_files"].isEmpty()) get_if_exists (keyValues, "open_files", options.max_open_files);
{
options.max_open_files = keyValues["open_files"].getIntValue();
}
if (! keyValues["compression"].isEmpty ()) if (keyValues.exists ("compression") &&
{ (get<int>(keyValues, "compression") == 0))
if (keyValues["compression"].getIntValue () == 0)
{
options.compression = rocksdb::kNoCompression; options.compression = rocksdb::kNoCompression;
}
}
rocksdb::DB* db = nullptr; rocksdb::DB* db = nullptr;
@@ -385,7 +373,7 @@ public:
std::unique_ptr <Backend> std::unique_ptr <Backend>
createInstance ( createInstance (
size_t keyBytes, size_t keyBytes,
Parameters const& keyValues, Section const& keyValues,
Scheduler& scheduler, Scheduler& scheduler,
beast::Journal journal) beast::Journal journal)
{ {

View File

@@ -55,13 +55,13 @@ ManagerImp::~ManagerImp()
std::unique_ptr <Backend> std::unique_ptr <Backend>
ManagerImp::make_Backend ( ManagerImp::make_Backend (
Parameters const& parameters, Section const& parameters,
Scheduler& scheduler, Scheduler& scheduler,
beast::Journal journal) beast::Journal journal)
{ {
std::unique_ptr <Backend> backend; std::unique_ptr <Backend> backend;
std::string const type (parameters ["type"].toStdString ()); std::string const type (get<std::string>(parameters, "type"));
if (! type.empty ()) if (! type.empty ())
{ {
@@ -91,8 +91,8 @@ ManagerImp::make_Database (
Scheduler& scheduler, Scheduler& scheduler,
beast::Journal journal, beast::Journal journal,
int readThreads, int readThreads,
Parameters const& backendParameters, Section const& backendParameters,
Parameters fastBackendParameters) Section fastBackendParameters)
{ {
std::unique_ptr <Backend> backend (make_Backend ( std::unique_ptr <Backend> backend (make_Backend (
backendParameters, scheduler, journal)); backendParameters, scheduler, journal));
@@ -167,11 +167,8 @@ std::unique_ptr <Backend>
make_Backend (Section const& config, make_Backend (Section const& config,
Scheduler& scheduler, beast::Journal journal) Scheduler& scheduler, beast::Journal journal)
{ {
beast::StringPairArray v;
for (auto const& _ : config)
v.set (_.first, _.second);
return Manager::instance().make_Backend ( return Manager::instance().make_Backend (
v, scheduler, journal); config, scheduler, journal);
} }
} }

View File

@@ -57,7 +57,7 @@ public:
std::unique_ptr <Backend> std::unique_ptr <Backend>
make_Backend ( make_Backend (
Parameters const& parameters, Section const& parameters,
Scheduler& scheduler, Scheduler& scheduler,
beast::Journal journal) override; beast::Journal journal) override;
@@ -67,8 +67,8 @@ public:
Scheduler& scheduler, Scheduler& scheduler,
beast::Journal journal, beast::Journal journal,
int readThreads, int readThreads,
Parameters const& backendParameters, Section const& backendParameters,
Parameters fastBackendParameters) override; Section fastBackendParameters) override;
std::unique_ptr <DatabaseRotating> std::unique_ptr <DatabaseRotating>
make_DatabaseRotating ( make_DatabaseRotating (

View File

@@ -38,10 +38,10 @@ public:
testcase ("Backend type=" + type); testcase ("Backend type=" + type);
beast::StringPairArray params; Section params;
beast::UnitTestUtilities::TempDirectory path ("node_db"); beast::UnitTestUtilities::TempDirectory path ("node_db");
params.set ("type", type); params.set ("type", type);
params.set ("path", path.getFullPathName ()); params.set ("path", path.getFullPathName ().toStdString ());
// Create a batch // Create a batch
Batch batch; Batch batch;

View File

@@ -35,9 +35,9 @@ public:
DummyScheduler scheduler; DummyScheduler scheduler;
beast::UnitTestUtilities::TempDirectory node_db ("node_db"); beast::UnitTestUtilities::TempDirectory node_db ("node_db");
beast::StringPairArray srcParams; Section srcParams;
srcParams.set ("type", srcBackendType); srcParams.set ("type", srcBackendType);
srcParams.set ("path", node_db.getFullPathName ()); srcParams.set ("path", node_db.getFullPathName ().toStdString ());
// Create a batch // Create a batch
Batch batch; Batch batch;
@@ -61,9 +61,9 @@ public:
// Set up the destination database // Set up the destination database
beast::UnitTestUtilities::TempDirectory dest_db ("dest_db"); beast::UnitTestUtilities::TempDirectory dest_db ("dest_db");
beast::StringPairArray destParams; Section destParams;
destParams.set ("type", destBackendType); destParams.set ("type", destBackendType);
destParams.set ("path", dest_db.getFullPathName ()); destParams.set ("path", dest_db.getFullPathName ().toStdString ());
std::unique_ptr <Database> dest = Manager::instance().make_Database ( std::unique_ptr <Database> dest = Manager::instance().make_Database (
"test", scheduler, j, 2, destParams); "test", scheduler, j, 2, destParams);
@@ -101,16 +101,16 @@ public:
testcase (s); testcase (s);
beast::UnitTestUtilities::TempDirectory node_db ("node_db"); beast::UnitTestUtilities::TempDirectory node_db ("node_db");
beast::StringPairArray nodeParams; Section nodeParams;
nodeParams.set ("type", type); nodeParams.set ("type", type);
nodeParams.set ("path", node_db.getFullPathName ()); nodeParams.set ("path", node_db.getFullPathName ().toStdString ());
beast::UnitTestUtilities::TempDirectory temp_db ("temp_db"); beast::UnitTestUtilities::TempDirectory temp_db ("temp_db");
beast::StringPairArray tempParams; Section tempParams;
if (useEphemeralDatabase) if (useEphemeralDatabase)
{ {
tempParams.set ("type", type); tempParams.set ("type", type);
tempParams.set ("path", temp_db.getFullPathName ()); tempParams.set ("path", temp_db.getFullPathName ().toStdString ());
} }
// Create a batch // Create a batch
@@ -164,7 +164,7 @@ public:
{ {
// Verify the ephemeral db // Verify the ephemeral db
std::unique_ptr <Database> db = Manager::instance().make_Database ("test", std::unique_ptr <Database> db = Manager::instance().make_Database ("test",
scheduler, j, 2, tempParams, beast::StringPairArray ()); scheduler, j, 2, tempParams, Section ());
// Read it back in // Read it back in
Batch copy; Batch copy;

View File

@@ -50,11 +50,12 @@ public:
TestFamily (beast::Journal j) TestFamily (beast::Journal j)
: treecache_ ("TreeNodeCache", 65536, 60, clock_, j) : treecache_ ("TreeNodeCache", 65536, 60, clock_, j)
, fullbelow_ ("full_below", clock_) , fullbelow_ ("full_below", clock_)
, db_(NodeStore::Manager::instance().make_Database(
"test", scheduler_, j, 1,
parseDelimitedKeyValueString(
"type=memory|Path=SHAMap_test")))
{ {
Section testSection;
testSection.set("type", "memory");
testSection.set("Path", "SHAMap_test");
db_ = NodeStore::Manager::instance ().make_Database (
"test", scheduler_, j, 1, testSection);
} }
beast::manual_clock <std::chrono::steady_clock> beast::manual_clock <std::chrono::steady_clock>