From 85fc59b28ba678e2635d9fc3728a5faa818d4598 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Fri, 20 Sep 2013 00:42:45 -0700 Subject: [PATCH] Refactor NodeStore, add NodeStoreSchedulerService --- Builds/VisualStudio2012/RippleD.vcxproj | 109 +- .../VisualStudio2012/RippleD.vcxproj.filters | 210 ++- SConstruct | 2 +- src/ripple/frame/api/Service.cpp | 21 - src/ripple/frame/api/Service.h | 16 - src/ripple_app/main/Application.cpp | 44 +- src/ripple_app/main/Application.h | 5 +- .../main/NodeStoreSchedulerService.cpp | 39 + .../main/NodeStoreSchedulerService.h | 30 + src/ripple_app/main/RippleMain.cpp | 6 +- ...teBackendFactory.cpp => SqliteFactory.cpp} | 35 +- ...SqliteBackendFactory.h => SqliteFactory.h} | 20 +- src/ripple_app/ripple_app.cpp | 7 +- src/ripple_app/shamap/SHAMap.cpp | 2 +- src/ripple_core/functional/Config.h | 4 +- src/ripple_core/node/KeyvaDBBackendFactory.h | 32 - src/ripple_core/node/LevelDBBackendFactory.h | 35 - src/ripple_core/node/MdbBackendFactory.h | 38 - src/ripple_core/node/MemoryBackendFactory.h | 32 - src/ripple_core/node/NodeStore.cpp | 1315 ----------------- src/ripple_core/node/NodeStore.h | 448 ------ src/ripple_core/node/NullBackendFactory.cpp | 84 -- src/ripple_core/node/NullBackendFactory.h | 34 - src/ripple_core/node/SophiaBackendFactory.h | 32 - src/ripple_core/nodestore/NodeStore.cpp | 49 + src/ripple_core/nodestore/NodeStore.h | 25 + src/ripple_core/nodestore/api/Backend.h | 88 ++ src/ripple_core/nodestore/api/Database.h | 139 ++ .../nodestore/api/DummyScheduler.h | 28 + src/ripple_core/nodestore/api/Factory.h | 37 + .../{node => nodestore/api}/NodeObject.h | 11 +- src/ripple_core/nodestore/api/Scheduler.h | 36 + src/ripple_core/nodestore/api/Task.h | 26 + src/ripple_core/nodestore/api/Types.h | 38 + src/ripple_core/nodestore/api/VisitCallback.h | 24 + .../backend/HyperDBFactory.cpp} | 67 +- .../backend/HyperDBFactory.h} | 25 +- .../backend/KeyvaDBFactory.cpp} | 49 +- .../nodestore/backend/KeyvaDBFactory.h | 37 + .../backend/LevelDBFactory.cpp} | 68 +- .../nodestore/backend/LevelDBFactory.h | 40 + .../backend/MdbFactory.cpp} | 59 +- .../nodestore/backend/MdbFactory.h | 43 + .../backend/MemoryFactory.cpp} | 41 +- .../nodestore/backend/MemoryFactory.h | 37 + .../nodestore/backend/NullFactory.cpp | 81 + .../nodestore/backend/NullFactory.h | 40 + .../backend/SophiaFactory.cpp} | 62 +- .../nodestore/backend/SophiaFactory.h | 37 + .../nodestore/impl/BatchWriter.cpp | 96 ++ src/ripple_core/nodestore/impl/BatchWriter.h | 74 + src/ripple_core/nodestore/impl/DatabaseImp.h | 352 +++++ .../nodestore/impl/DecodedBlob.cpp | 83 ++ src/ripple_core/nodestore/impl/DecodedBlob.h | 47 + .../nodestore/impl/DummyScheduler.cpp | 29 + .../nodestore/impl/EncodedBlob.cpp | 38 + src/ripple_core/nodestore/impl/EncodedBlob.h | 40 + src/ripple_core/nodestore/impl/Factories.h | 50 + .../{node => nodestore/impl}/NodeObject.cpp | 16 - .../nodestore/tests/BackendTests.cpp | 97 ++ .../nodestore/tests/BasicTests.cpp | 76 + .../nodestore/tests/DatabaseTests.cpp | 228 +++ src/ripple_core/nodestore/tests/TestBase.h | 179 +++ .../nodestore/tests/TimingTests.cpp | 126 ++ src/ripple_core/ripple_core.cpp | 26 +- src/ripple_core/ripple_core.h | 7 +- 66 files changed, 2857 insertions(+), 2494 deletions(-) create mode 100644 src/ripple_app/main/NodeStoreSchedulerService.cpp create mode 100644 src/ripple_app/main/NodeStoreSchedulerService.h rename src/ripple_app/node/{SqliteBackendFactory.cpp => SqliteFactory.cpp} (87%) rename src/ripple_app/node/{SqliteBackendFactory.h => SqliteFactory.h} (55%) delete mode 100644 src/ripple_core/node/KeyvaDBBackendFactory.h delete mode 100644 src/ripple_core/node/LevelDBBackendFactory.h delete mode 100644 src/ripple_core/node/MdbBackendFactory.h delete mode 100644 src/ripple_core/node/MemoryBackendFactory.h delete mode 100644 src/ripple_core/node/NodeStore.cpp delete mode 100644 src/ripple_core/node/NodeStore.h delete mode 100644 src/ripple_core/node/NullBackendFactory.cpp delete mode 100644 src/ripple_core/node/NullBackendFactory.h delete mode 100644 src/ripple_core/node/SophiaBackendFactory.h create mode 100644 src/ripple_core/nodestore/NodeStore.cpp create mode 100644 src/ripple_core/nodestore/NodeStore.h create mode 100644 src/ripple_core/nodestore/api/Backend.h create mode 100644 src/ripple_core/nodestore/api/Database.h create mode 100644 src/ripple_core/nodestore/api/DummyScheduler.h create mode 100644 src/ripple_core/nodestore/api/Factory.h rename src/ripple_core/{node => nodestore/api}/NodeObject.h (95%) create mode 100644 src/ripple_core/nodestore/api/Scheduler.h create mode 100644 src/ripple_core/nodestore/api/Task.h create mode 100644 src/ripple_core/nodestore/api/Types.h create mode 100644 src/ripple_core/nodestore/api/VisitCallback.h rename src/ripple_core/{node/HyperLevelDBBackendFactory.cpp => nodestore/backend/HyperDBFactory.cpp} (78%) rename src/ripple_core/{node/HyperLevelDBBackendFactory.h => nodestore/backend/HyperDBFactory.h} (54%) rename src/ripple_core/{node/KeyvaDBBackendFactory.cpp => nodestore/backend/KeyvaDBFactory.cpp} (74%) create mode 100644 src/ripple_core/nodestore/backend/KeyvaDBFactory.h rename src/ripple_core/{node/LevelDBBackendFactory.cpp => nodestore/backend/LevelDBFactory.cpp} (79%) create mode 100644 src/ripple_core/nodestore/backend/LevelDBFactory.h rename src/ripple_core/{node/MdbBackendFactory.cpp => nodestore/backend/MdbFactory.cpp} (84%) create mode 100644 src/ripple_core/nodestore/backend/MdbFactory.h rename src/ripple_core/{node/MemoryBackendFactory.cpp => nodestore/backend/MemoryFactory.cpp} (66%) create mode 100644 src/ripple_core/nodestore/backend/MemoryFactory.h create mode 100644 src/ripple_core/nodestore/backend/NullFactory.cpp create mode 100644 src/ripple_core/nodestore/backend/NullFactory.h rename src/ripple_core/{node/SophiaBackendFactory.cpp => nodestore/backend/SophiaFactory.cpp} (72%) create mode 100644 src/ripple_core/nodestore/backend/SophiaFactory.h create mode 100644 src/ripple_core/nodestore/impl/BatchWriter.cpp create mode 100644 src/ripple_core/nodestore/impl/BatchWriter.h create mode 100644 src/ripple_core/nodestore/impl/DatabaseImp.h create mode 100644 src/ripple_core/nodestore/impl/DecodedBlob.cpp create mode 100644 src/ripple_core/nodestore/impl/DecodedBlob.h create mode 100644 src/ripple_core/nodestore/impl/DummyScheduler.cpp create mode 100644 src/ripple_core/nodestore/impl/EncodedBlob.cpp create mode 100644 src/ripple_core/nodestore/impl/EncodedBlob.h create mode 100644 src/ripple_core/nodestore/impl/Factories.h rename src/ripple_core/{node => nodestore/impl}/NodeObject.cpp (88%) create mode 100644 src/ripple_core/nodestore/tests/BackendTests.cpp create mode 100644 src/ripple_core/nodestore/tests/BasicTests.cpp create mode 100644 src/ripple_core/nodestore/tests/DatabaseTests.cpp create mode 100644 src/ripple_core/nodestore/tests/TestBase.h create mode 100644 src/ripple_core/nodestore/tests/TimingTests.cpp diff --git a/Builds/VisualStudio2012/RippleD.vcxproj b/Builds/VisualStudio2012/RippleD.vcxproj index dbbc2b671f..091274e01d 100644 --- a/Builds/VisualStudio2012/RippleD.vcxproj +++ b/Builds/VisualStudio2012/RippleD.vcxproj @@ -243,6 +243,12 @@ true true + + true + true + true + true + true true @@ -375,7 +381,7 @@ true true - + true true true @@ -821,55 +827,103 @@ true true - + true true true true - + true true true true - + true true true true - + true true true true - + true true true true - + true true true true - + true true true true - + true true true true - + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + true true true @@ -1499,6 +1553,7 @@ + @@ -1522,7 +1577,7 @@ - + @@ -1605,15 +1660,29 @@ - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Builds/VisualStudio2012/RippleD.vcxproj.filters b/Builds/VisualStudio2012/RippleD.vcxproj.filters index 7e8c3ad5f7..403b2989a0 100644 --- a/Builds/VisualStudio2012/RippleD.vcxproj.filters +++ b/Builds/VisualStudio2012/RippleD.vcxproj.filters @@ -103,9 +103,6 @@ {548037f2-eb8a-41bd-95dc-05f58cdbc041} - - {7270818b-573d-45b7-b10b-42f63bedcc50} - {d1648d3f-7d71-495d-afc9-576ed00d7185} @@ -190,6 +187,21 @@ {a4dd852c-651b-4ea9-a051-252dc0eaea24} + + {490d0cdb-7ce1-42f2-b234-6874dae550b7} + + + {64fa2598-39d6-46de-81b0-17131f06d943} + + + {7aa0afca-11d5-43e5-bd95-ff26626cd2fc} + + + {410e4aa6-1a25-4ec3-a7fd-b042bdf7c0de} + + + {071582fa-cf16-4e41-8791-613cfe00eef8} + @@ -768,33 +780,6 @@ [2] Old Ripple\ripple_app\tx - - [2] Old Ripple\ripple_app\node - - - [2] Old Ripple\ripple_core\node - - - [2] Old Ripple\ripple_core\node - - - [2] Old Ripple\ripple_core\node - - - [2] Old Ripple\ripple_core\node - - - [2] Old Ripple\ripple_core\node - - - [2] Old Ripple\ripple_core\node - - - [2] Old Ripple\ripple_core\node - - - [2] Old Ripple\ripple_core\node - [0] Libraries\protobuf @@ -921,9 +906,6 @@ [2] Old Ripple\ripple_net\basics - - [2] Old Ripple\ripple_core\node - [1] Ripple\sophia @@ -945,6 +927,63 @@ [1] Ripple\frame\api + + [2] Old Ripple\ripple_core\nodestore + + + [2] Old Ripple\ripple_core\nodestore\impl + + + [2] Old Ripple\ripple_core\nodestore\impl + + + [2] Old Ripple\ripple_core\nodestore\impl + + + [2] Old Ripple\ripple_core\nodestore\impl + + + [2] Old Ripple\ripple_core\nodestore\tests + + + [2] Old Ripple\ripple_core\nodestore\backend + + + [2] Old Ripple\ripple_core\nodestore\backend + + + [2] Old Ripple\ripple_core\nodestore\backend + + + [2] Old Ripple\ripple_core\nodestore\backend + + + [2] Old Ripple\ripple_core\nodestore\backend + + + [2] Old Ripple\ripple_core\nodestore\backend + + + [2] Old Ripple\ripple_core\nodestore\backend + + + [2] Old Ripple\ripple_core\nodestore\tests + + + [2] Old Ripple\ripple_core\nodestore\tests + + + [2] Old Ripple\ripple_core\nodestore\tests + + + [2] Old Ripple\ripple_core\nodestore\impl + + + [2] Old Ripple\ripple_app\main + + + [2] Old Ripple\ripple_app\node + @@ -1652,33 +1691,6 @@ [2] Old Ripple\ripple_app\tx - - [2] Old Ripple\ripple_app\node - - - [2] Old Ripple\ripple_core\node - - - [2] Old Ripple\ripple_core\node - - - [2] Old Ripple\ripple_core\node - - - [2] Old Ripple\ripple_core\node - - - [2] Old Ripple\ripple_core\node - - - [2] Old Ripple\ripple_core\node - - - [2] Old Ripple\ripple_core\node - - - [2] Old Ripple\ripple_core\node - [2] Old Ripple\ripple_app\misc @@ -1824,9 +1836,6 @@ [1] Ripple\sophia - - [2] Old Ripple\ripple_core\node - [1] Ripple\json\api @@ -1860,6 +1869,81 @@ [1] Ripple\frame\api + + [2] Old Ripple\ripple_core\nodestore + + + [2] Old Ripple\ripple_core\nodestore\api + + + [2] Old Ripple\ripple_core\nodestore\api + + + [2] Old Ripple\ripple_core\nodestore\api + + + [2] Old Ripple\ripple_core\nodestore\impl + + + [2] Old Ripple\ripple_core\nodestore\impl + + + [2] Old Ripple\ripple_core\nodestore\api + + + [2] Old Ripple\ripple_core\nodestore\api + + + [2] Old Ripple\ripple_core\nodestore\impl + + + [2] Old Ripple\ripple_core\nodestore\api + + + [2] Old Ripple\ripple_core\nodestore\api + + + [2] Old Ripple\ripple_core\nodestore\impl + + + [2] Old Ripple\ripple_core\nodestore\api + + + [2] Old Ripple\ripple_core\nodestore\backend + + + [2] Old Ripple\ripple_core\nodestore\backend + + + [2] Old Ripple\ripple_core\nodestore\backend + + + [2] Old Ripple\ripple_core\nodestore\backend + + + [2] Old Ripple\ripple_core\nodestore\backend + + + [2] Old Ripple\ripple_core\nodestore\backend + + + [2] Old Ripple\ripple_core\nodestore\backend + + + [2] Old Ripple\ripple_core\nodestore\tests + + + [2] Old Ripple\ripple_core\nodestore\impl + + + [2] Old Ripple\ripple_app\main + + + [2] Old Ripple\ripple_core\nodestore\api + + + [2] Old Ripple\ripple_app\node + diff --git a/SConstruct b/SConstruct index ea041dd94a..d033097dad 100644 --- a/SConstruct +++ b/SConstruct @@ -146,7 +146,7 @@ COMPILED_FILES.extend (['src/ripple/beast/ripple_beastc.c']) # New-style Ripple unity sources # COMPILED_FILES.extend([ - 'src/ripple/frame/ripple_frame.c', + 'src/ripple/frame/ripple_frame.cpp', 'src/ripple/json/ripple_json.cpp', 'src/ripple/sophia/ripple_sophia.c', 'src/ripple/testoverlay/ripple_testoverlay.cpp', diff --git a/src/ripple/frame/api/Service.cpp b/src/ripple/frame/api/Service.cpp index 64fd453007..75b2893b4b 100644 --- a/src/ripple/frame/api/Service.cpp +++ b/src/ripple/frame/api/Service.cpp @@ -181,25 +181,4 @@ void Service::stopRecursive (Journal::Stream stream) m_stopped = true; } -//------------------------------------------------------------------------------ - -ScopedService::ScopedService (char const* name) - : Service (name) -{ -} - -ScopedService::~ScopedService () -{ - serviceStop(); -} - -void ScopedService::onServiceStop () -{ - serviceStopped(); -} - -void ScopedService::onServiceChildrenStopped () -{ -} - } diff --git a/src/ripple/frame/api/Service.h b/src/ripple/frame/api/Service.h index 4cda60dd74..c4c4f61e8b 100644 --- a/src/ripple/frame/api/Service.h +++ b/src/ripple/frame/api/Service.h @@ -263,22 +263,6 @@ private: WaitableEvent m_stoppedEvent; }; -//------------------------------------------------------------------------------ - -/** A root Service with a short scope. - This Service takes care of stopping automatically, no additional - action is required. -*/ -class ScopedService : public Service -{ -public: - explicit ScopedService (char const* name); - ~ScopedService (); - - void onServiceStop (); - void onServiceChildrenStopped (); -}; - } #endif diff --git a/src/ripple_app/main/Application.cpp b/src/ripple_app/main/Application.cpp index cdba7f49ae..8aa20377c3 100644 --- a/src/ripple_app/main/Application.cpp +++ b/src/ripple_app/main/Application.cpp @@ -29,7 +29,6 @@ template <> char const* LogPartition::getPartitionName () { retu // VFALCO TODO Move the function definitions into the class declaration class ApplicationImp : public Application - , public NodeStore::Scheduler , public Service , public DeadlineTimer::Listener , LeakChecked @@ -80,8 +79,10 @@ public: , m_rpcServerHandler (*m_networkOPs) // passive object, not a Service - , m_nodeStore (NodeStore::New ("NodeStore.main", *m_jobQueue, - getConfig ().nodeDatabase, getConfig ().ephemeralNodeDatabase, *this)) + , m_nodeStoreScheduler (*m_jobQueue, *m_jobQueue) + + , m_nodeStore (NodeStore::Database::New ("NodeStore.main", m_nodeStoreScheduler, + getConfig ().nodeDatabase, getConfig ().ephemeralNodeDatabase)) , m_sntpClient (SNTPClient::New (*this)) @@ -158,27 +159,6 @@ public: //-------------------------------------------------------------------------- - static void callScheduledTask (NodeStore::Scheduler::Task& task, Job&) - { - task.performScheduledTask (); - } - - void scheduleTask (NodeStore::Scheduler::Task& task) - { - getJobQueue ().addJob ( - jtWRITE, - "NodeObject::store", - BIND_TYPE (&ApplicationImp::callScheduledTask, boost::ref(task), P_1)); - } - - void scheduledTasksStopped () - { - // VFALCO NOTE This is a bit of a hack - getNodeStore().serviceStopped(); - } - - //-------------------------------------------------------------------------- - LocalCredentials& getLocalCredentials () { return m_localCredentials ; @@ -214,7 +194,7 @@ public: return m_tempNodeCache; } - NodeStore& getNodeStore () + NodeStore::Database& getNodeStore () { return *m_nodeStore; } @@ -635,9 +615,6 @@ public: mShutdown = true; - // This stalls for a long time - //m_nodeStore = nullptr; - mValidations->flush (); mShutdown = false; @@ -750,7 +727,7 @@ public: &TransactionMaster::sweep, &m_txMaster)); logTimedCall (m_journal.warning, "NodeStore::sweep", __FILE__, __LINE__, boost::bind ( - &NodeStore::sweep, m_nodeStore.get ())); + &NodeStore::Database::sweep, m_nodeStore.get ())); logTimedCall (m_journal.warning, "LedgerMaster::sweep", __FILE__, __LINE__, boost::bind ( &LedgerMaster::sweep, &m_ledgerMaster)); @@ -806,7 +783,8 @@ private: ScopedPointer m_networkOPs; ScopedPointer m_deprecatedUNL; RPCServerHandler m_rpcServerHandler; - ScopedPointer m_nodeStore; + NodeStoreSchedulerService m_nodeStoreScheduler; + ScopedPointer m_nodeStore; ScopedPointer m_sntpClient; InboundLedgers m_inboundLedgers; ScopedPointer m_txQueue; @@ -1138,9 +1116,9 @@ void ApplicationImp::updateTables () if (getConfig ().importNodeDatabase.size () > 0) { - ScopedService service ("import service"); - ScopedPointer source (NodeStore::New ( - "NodeStore.import", service, getConfig ().importNodeDatabase)); + NodeStore::DummyScheduler scheduler; + ScopedPointer source (NodeStore::Database::New ( + "NodeStore.import", scheduler, getConfig ().importNodeDatabase)); WriteLog (lsWARNING, NodeObject) << "Node import from '" << source->getName () << "' to '" diff --git a/src/ripple_app/main/Application.h b/src/ripple_app/main/Application.h index d1c5c0e0d6..44e4144221 100644 --- a/src/ripple_app/main/Application.h +++ b/src/ripple_app/main/Application.h @@ -8,6 +8,7 @@ #define RIPPLE_APP_APPLICATION_H_INCLUDED namespace Validators { class Manager; } +namespace NodeStore { class Database; } // VFALCO TODO Fix forward declares required for header dependency loops class IFeatures; @@ -16,8 +17,6 @@ class IHashRouter; class ILoadFeeTrack; class Peers; class UniqueNodeList; - -class NodeStore; class JobQueue; class InboundLedgers; class LedgerMaster; @@ -90,7 +89,7 @@ public: virtual ProofOfWorkFactory& getProofOfWorkFactory () = 0; virtual UniqueNodeList& getUNL () = 0; virtual Validations& getValidations () = 0; - virtual NodeStore& getNodeStore () = 0; + virtual NodeStore::Database& getNodeStore () = 0; virtual JobQueue& getJobQueue () = 0; virtual InboundLedgers& getInboundLedgers () = 0; virtual LedgerMaster& getLedgerMaster () = 0; diff --git a/src/ripple_app/main/NodeStoreSchedulerService.cpp b/src/ripple_app/main/NodeStoreSchedulerService.cpp new file mode 100644 index 0000000000..83ff0068f0 --- /dev/null +++ b/src/ripple_app/main/NodeStoreSchedulerService.cpp @@ -0,0 +1,39 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +NodeStoreSchedulerService::NodeStoreSchedulerService (Service& parent, JobQueue& jobQueue) + : Service ("NodeStoreSchedulerService", parent) + , m_jobQueue (jobQueue) + , m_taskCount (1) // start it off at 1 +{ +} + +void NodeStoreSchedulerService::onServiceStop () +{ + if (--m_taskCount == 0) + serviceStopped(); +} + +void NodeStoreSchedulerService::onServiceChildrenStopped () +{ +} + +void NodeStoreSchedulerService::scheduleTask (NodeStore::Task& task) +{ + ++m_taskCount; + m_jobQueue.addJob ( + jtWRITE, + "NodeObject::store", + BIND_TYPE (&NodeStoreSchedulerService::doTask, + this, boost::ref(task), P_1)); +} + +void NodeStoreSchedulerService::doTask (NodeStore::Task& task, Job&) +{ + task.performScheduledTask (); + if ((--m_taskCount == 0) && isServiceStopping()) + serviceStopped(); +} diff --git a/src/ripple_app/main/NodeStoreSchedulerService.h b/src/ripple_app/main/NodeStoreSchedulerService.h new file mode 100644 index 0000000000..c9e30a9121 --- /dev/null +++ b/src/ripple_app/main/NodeStoreSchedulerService.h @@ -0,0 +1,30 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_APP_NODESTORESCHEDULERSERVICE_H_INCLUDED +#define RIPPLE_APP_NODESTORESCHEDULERSERVICE_H_INCLUDED + +/** A NodeStore::Scheduler which uses the JobQueue and implements the Service API. */ +class NodeStoreSchedulerService + : public NodeStore::Scheduler + , public Service +{ +public: + NodeStoreSchedulerService (Service& parent, JobQueue& jobQueue); + + void onServiceStop (); + void onServiceChildrenStopped (); + void scheduleTask (NodeStore::Task& task); + +private: + void doTask (NodeStore::Task& task, Job&); + + JobQueue& m_jobQueue; + Atomic m_taskCount; +}; + + +#endif diff --git a/src/ripple_app/main/RippleMain.cpp b/src/ripple_app/main/RippleMain.cpp index 209db89c2e..2bfb2665aa 100644 --- a/src/ripple_app/main/RippleMain.cpp +++ b/src/ripple_app/main/RippleMain.cpp @@ -269,12 +269,12 @@ int RippleMain::run (int argc, char const* const* argv) // NOTE: These must be added before the // Application object is created. // - NodeStore::addAvailableBackends (); + NodeStore::Database::addAvailableBackends (); - // VFALCO NOTE SqliteBackendFactory is here because it has + // VFALCO NOTE SqliteFactory is here because it has // dependencies like SqliteDatabase and DatabaseCon // - NodeStore::addBackendFactory (SqliteBackendFactory::getInstance ()); + NodeStore::Database::addFactory (SqliteFactory::getInstance ()); if (! RandomNumbers::getInstance ().initialize ()) { diff --git a/src/ripple_app/node/SqliteBackendFactory.cpp b/src/ripple_app/node/SqliteFactory.cpp similarity index 87% rename from src/ripple_app/node/SqliteBackendFactory.cpp rename to src/ripple_app/node/SqliteFactory.cpp index 89d6156032..36c6af00cf 100644 --- a/src/ripple_app/node/SqliteBackendFactory.cpp +++ b/src/ripple_app/node/SqliteFactory.cpp @@ -30,10 +30,10 @@ static int s_nodeStoreDBCount = NUMBER (s_nodeStoreDBInit); //------------------------------------------------------------------------------ -class SqliteBackendFactory::Backend : public NodeStore::Backend +class SqliteFactory::BackendImp : public NodeStore::Backend { public: - Backend (size_t keyBytes, std::string const& path, NodeStore::Scheduler& scheduler) + BackendImp (size_t keyBytes, std::string const& path, NodeStore::Scheduler& scheduler) : m_keyBytes (keyBytes) , m_name (path) , m_db (new DatabaseCon(path, s_nodeStoreDBInit, s_nodeStoreDBCount)) @@ -47,7 +47,7 @@ public: m_db->getDB()->executeSQL (s.toStdString ().c_str ()); } - ~Backend() + ~BackendImp() { } @@ -58,9 +58,9 @@ public: //-------------------------------------------------------------------------- - Status fetch (void const* key, NodeObject::Ptr* pObject) + NodeStore::Status fetch (void const* key, NodeObject::Ptr* pObject) { - Status result = ok; + NodeStore::Status result = NodeStore::ok; pObject->reset (); @@ -87,7 +87,7 @@ public: } else { - result = notFound; + result = NodeStore::notFound; } pSt.reset(); @@ -132,7 +132,7 @@ public: pStE.reset(); } - void visitAll (VisitCallback& callback) + void visitAll (NodeStore::VisitCallback& callback) { // No lock needed as per the visitAll() API @@ -165,11 +165,6 @@ public: return 0; } - void stopAsync () - { - m_scheduler.scheduledTasksStopped (); - } - //-------------------------------------------------------------------------- void doBind (SqliteStatement& statement, NodeObject::ref object) @@ -216,28 +211,28 @@ private: //------------------------------------------------------------------------------ -SqliteBackendFactory::SqliteBackendFactory () +SqliteFactory::SqliteFactory () { } -SqliteBackendFactory::~SqliteBackendFactory () +SqliteFactory::~SqliteFactory () { } -SqliteBackendFactory* SqliteBackendFactory::getInstance () +SqliteFactory* SqliteFactory::getInstance () { - return new SqliteBackendFactory; + return new SqliteFactory; } -String SqliteBackendFactory::getName () const +String SqliteFactory::getName () const { return "Sqlite"; } -NodeStore::Backend* SqliteBackendFactory::createInstance ( +NodeStore::Backend* SqliteFactory::createInstance ( size_t keyBytes, - StringPairArray const& keyValues, + NodeStore::Parameters const& keyValues, NodeStore::Scheduler& scheduler) { - return new Backend (keyBytes, keyValues ["path"].toStdString (), scheduler); + return new BackendImp (keyBytes, keyValues ["path"].toStdString (), scheduler); } diff --git a/src/ripple_app/node/SqliteBackendFactory.h b/src/ripple_app/node/SqliteFactory.h similarity index 55% rename from src/ripple_app/node/SqliteBackendFactory.h rename to src/ripple_app/node/SqliteFactory.h index 1c3889cdf4..3d238738d4 100644 --- a/src/ripple_app/node/SqliteBackendFactory.h +++ b/src/ripple_app/node/SqliteFactory.h @@ -4,28 +4,28 @@ */ //============================================================================== -#ifndef RIPPLE_SQLITEBACKENDFACTORY_H_INCLUDED -#define RIPPLE_SQLITEBACKENDFACTORY_H_INCLUDED +#ifndef RIPPLE_APP_SQLITEFACTORY_H_INCLUDED +#define RIPPLE_APP_SQLITEFACTORY_H_INCLUDED /** Factory to produce SQLite backends for the NodeStore. - @see NodeStore + @see Database */ -class SqliteBackendFactory : public NodeStore::BackendFactory +class SqliteFactory : public NodeStore::Factory { private: - class Backend; - - SqliteBackendFactory (); - ~SqliteBackendFactory (); + SqliteFactory (); + ~SqliteFactory (); public: - static SqliteBackendFactory* getInstance (); + class BackendImp; + + static SqliteFactory* getInstance (); String getName () const; NodeStore::Backend* createInstance (size_t keyBytes, - StringPairArray const& keyValues, + NodeStore::Parameters const& keyValues, NodeStore::Scheduler& scheduler); }; diff --git a/src/ripple_app/ripple_app.cpp b/src/ripple_app/ripple_app.cpp index 6a2b90a03c..8922bd6525 100644 --- a/src/ripple_app/ripple_app.cpp +++ b/src/ripple_app/ripple_app.cpp @@ -30,6 +30,9 @@ namespace ripple // Application // +# include "main/NodeStoreSchedulerService.h" +#include "main/NodeStoreSchedulerService.cpp" + # include "main/IoServicePool.h" #include "main/IoServicePool.cpp" @@ -66,8 +69,8 @@ namespace ripple // RippleMain // # include "main/RippleMain.h" -# include "node/SqliteBackendFactory.h" -#include "node/SqliteBackendFactory.cpp" +# include "node/SqliteFactory.h" +#include "node/SqliteFactory.cpp" #include "main/RippleMain.cpp" } diff --git a/src/ripple_app/shamap/SHAMap.cpp b/src/ripple_app/shamap/SHAMap.cpp index 5ab9d05fc9..291bbec55d 100644 --- a/src/ripple_app/shamap/SHAMap.cpp +++ b/src/ripple_app/shamap/SHAMap.cpp @@ -856,7 +856,7 @@ SHAMapTreeNode::pointer SHAMap::fetchNodeExternalNT (const SHAMapNode& id, uint2 // These are for diagnosing a crash on exit Application& app (getApp ()); - NodeStore& nodeStore (app.getNodeStore ()); + NodeStore::Database& nodeStore (app.getNodeStore ()); NodeObject::pointer obj (nodeStore.fetch (hash)); if (!obj) diff --git a/src/ripple_core/functional/Config.h b/src/ripple_core/functional/Config.h index 4db627e79d..f7798c43e9 100644 --- a/src/ripple_core/functional/Config.h +++ b/src/ripple_core/functional/Config.h @@ -288,7 +288,7 @@ public: This is 1 or more strings of the form = The 'type' and 'path' keys are required, see rippled-example.cfg - @see NodeStore + @see Database */ StringPairArray nodeDatabase; @@ -300,7 +300,7 @@ public: The format is the same as that for @ref nodeDatabase - @see NodeStore + @see Database */ StringPairArray ephemeralNodeDatabase; diff --git a/src/ripple_core/node/KeyvaDBBackendFactory.h b/src/ripple_core/node/KeyvaDBBackendFactory.h deleted file mode 100644 index a4e64fae6b..0000000000 --- a/src/ripple_core/node/KeyvaDBBackendFactory.h +++ /dev/null @@ -1,32 +0,0 @@ -//------------------------------------------------------------------------------ -/* - Copyright (c) 2011-2013, OpenCoin, Inc. -*/ -//============================================================================== - -#ifndef RIPPLE_CORE_NODE_KEYVABACKENDFACTORY_H_INCLUDED -#define RIPPLE_CORE_NODE_KEYVABACKENDFACTORY_H_INCLUDED - -/** Factory to produce KeyvaDB backends for the NodeStore. - - @see NodeStore -*/ -class KeyvaDBBackendFactory : public NodeStore::BackendFactory -{ -private: - class Backend; - - KeyvaDBBackendFactory (); - ~KeyvaDBBackendFactory (); - -public: - static KeyvaDBBackendFactory* getInstance (); - - String getName () const; - - NodeStore::Backend* createInstance (size_t keyBytes, - StringPairArray const& keyValues, - NodeStore::Scheduler& scheduler); -}; - -#endif diff --git a/src/ripple_core/node/LevelDBBackendFactory.h b/src/ripple_core/node/LevelDBBackendFactory.h deleted file mode 100644 index f8ab4f7531..0000000000 --- a/src/ripple_core/node/LevelDBBackendFactory.h +++ /dev/null @@ -1,35 +0,0 @@ -//------------------------------------------------------------------------------ -/* - Copyright (c) 2011-2013, OpenCoin, Inc. -*/ -//============================================================================== - -#ifndef RIPPLE_CORE_NODE_LEVELDBBACKENDFACTORY_H_INCLUDED -#define RIPPLE_CORE_NODE_LEVELDBBACKENDFACTORY_H_INCLUDED - -/** Factory to produce LevelDB backends for the NodeStore. - - @see NodeStore -*/ -class LevelDBBackendFactory : public NodeStore::BackendFactory -{ -private: - class Backend; - - LevelDBBackendFactory (); - ~LevelDBBackendFactory (); - -public: - static LevelDBBackendFactory* getInstance (); - - String getName () const; - - NodeStore::Backend* createInstance (size_t keyBytes, - StringPairArray const& keyValues, - NodeStore::Scheduler& scheduler); - -private: - void* m_lruCache; -}; - -#endif diff --git a/src/ripple_core/node/MdbBackendFactory.h b/src/ripple_core/node/MdbBackendFactory.h deleted file mode 100644 index 3a145ee85f..0000000000 --- a/src/ripple_core/node/MdbBackendFactory.h +++ /dev/null @@ -1,38 +0,0 @@ -//------------------------------------------------------------------------------ -/* - Copyright (c) 2011-2013, OpenCoin, Inc. -*/ -//============================================================================== - -#ifndef RIPPLE_CORE_NODE_MDBBACKENDFACTORY_H_INCLUDED -#define RIPPLE_CORE_NODE_MDBBACKENDFACTORY_H_INCLUDED - -#if RIPPLE_MDB_AVAILABLE - -/** Factory to produce a backend using MDB. - - @note MDB is not currently available for Win32 - - @see NodeStore -*/ -class MdbBackendFactory : public NodeStore::BackendFactory -{ -private: - class Backend; - - MdbBackendFactory (); - ~MdbBackendFactory (); - -public: - static MdbBackendFactory* getInstance (); - - String getName () const; - - NodeStore::Backend* createInstance (size_t keyBytes, - StringPairArray const& keyValues, - NodeStore::Scheduler& scheduler); -}; - -#endif - -#endif diff --git a/src/ripple_core/node/MemoryBackendFactory.h b/src/ripple_core/node/MemoryBackendFactory.h deleted file mode 100644 index db1edd5f0a..0000000000 --- a/src/ripple_core/node/MemoryBackendFactory.h +++ /dev/null @@ -1,32 +0,0 @@ -//------------------------------------------------------------------------------ -/* - Copyright (c) 2011-2013, OpenCoin, Inc. -*/ -//============================================================================== - -#ifndef RIPPLE_CORE_NODE_MEMORYBACKENDFACTORY_H_INCLUDED -#define RIPPLE_CORE_NODE_MEMORYBACKENDFACTORY_H_INCLUDED - -/** Factory to produce a RAM based backend for the NodeStore. - - @see NodeStore -*/ -class MemoryBackendFactory : public NodeStore::BackendFactory -{ -private: - class Backend; - - MemoryBackendFactory (); - ~MemoryBackendFactory (); - -public: - static MemoryBackendFactory* getInstance (); - - String getName () const; - - NodeStore::Backend* createInstance (size_t keyBytes, - StringPairArray const& keyValues, - NodeStore::Scheduler& scheduler); -}; - -#endif diff --git a/src/ripple_core/node/NodeStore.cpp b/src/ripple_core/node/NodeStore.cpp deleted file mode 100644 index 693d3f3489..0000000000 --- a/src/ripple_core/node/NodeStore.cpp +++ /dev/null @@ -1,1315 +0,0 @@ -//------------------------------------------------------------------------------ -/* - Copyright (c) 2011-2013, OpenCoin, Inc. -*/ -//============================================================================== - -// Holds the list of Backend factories -class Factories -{ -public: - typedef NodeStore::BackendFactory Factory; - - Factories () - { - } - - ~Factories () - { - } - - void add (Factory* factory) - { - m_list.add (factory); - } - - Factory* find (String name) const - { - for (int i = 0; i < m_list.size(); ++i) - if (m_list [i]->getName ().compareIgnoreCase (name) == 0) - return m_list [i]; - return nullptr; - } - - static Factories& get () - { - return *SharedSingleton ::get ( - SingletonLifetime::persistAfterCreation); - } - -private: - OwnedArray m_list; -}; - -//------------------------------------------------------------------------------ - -NodeStore::DecodedBlob::DecodedBlob (void const* key, void const* value, int valueBytes) -{ - /* Data format: - - Bytes - - 0...3 LedgerIndex 32-bit big endian integer - 4...7 Unused? An unused copy of the LedgerIndex - 8 char One of NodeObjectType - 9...end The body of the object data - */ - - m_success = false; - m_key = key; - // VFALCO NOTE Ledger indexes should have started at 1 - m_ledgerIndex = LedgerIndex (-1); - m_objectType = hotUNKNOWN; - m_objectData = nullptr; - m_dataBytes = bmax (0, valueBytes - 9); - - if (valueBytes > 4) - { - LedgerIndex const* index = static_cast (value); - m_ledgerIndex = ByteOrder::swapIfLittleEndian (*index); - } - - // VFALCO NOTE What about bytes 4 through 7 inclusive? - - if (valueBytes > 8) - { - unsigned char const* byte = static_cast (value); - m_objectType = static_cast (byte [8]); - } - - if (valueBytes > 9) - { - m_objectData = static_cast (value) + 9; - - switch (m_objectType) - { - case hotUNKNOWN: - default: - break; - - case hotLEDGER: - case hotTRANSACTION: - case hotACCOUNT_NODE: - case hotTRANSACTION_NODE: - m_success = true; - break; - } - } -} - -NodeObject::Ptr NodeStore::DecodedBlob::createObject () -{ - bassert (m_success); - - NodeObject::Ptr object; - - if (m_success) - { - Blob data (m_dataBytes); - - memcpy (data.data (), m_objectData, m_dataBytes); - - object = NodeObject::createObject ( - m_objectType, m_ledgerIndex, data, uint256::fromVoid (m_key)); - } - - return object; -} - -//------------------------------------------------------------------------------ - -void NodeStore::EncodedBlob::prepare (NodeObject::Ptr const& object) -{ - m_key = object->getHash ().begin (); - - // This is how many bytes we need in the flat data - m_size = object->getData ().size () + 9; - - m_data.ensureSize (m_size); - - // These sizes must be the same! - static_bassert (sizeof (uint32) == sizeof (object->getIndex ())); - - { - uint32* buf = static_cast (m_data.getData ()); - - buf [0] = ByteOrder::swapIfLittleEndian (object->getIndex ()); - buf [1] = ByteOrder::swapIfLittleEndian (object->getIndex ()); - } - - { - unsigned char* buf = static_cast (m_data.getData ()); - - buf [8] = static_cast (object->getType ()); - - memcpy (&buf [9], object->getData ().data (), object->getData ().size ()); - } -} - -//============================================================================== - -NodeStore::BatchWriter::BatchWriter (Callback& callback, Scheduler& scheduler) - : m_callback (callback) - , m_scheduler (scheduler) - , mWriteGeneration (0) - , mWriteLoad (0) - , mWritePending (false) -{ - mWriteSet.reserve (batchWritePreallocationSize); -} - -NodeStore::BatchWriter::~BatchWriter () -{ - waitForWriting (); -} - -void NodeStore::BatchWriter::store (NodeObject::ref object) -{ - LockType::scoped_lock sl (mWriteMutex); - - mWriteSet.push_back (object); - - if (! mWritePending) - { - mWritePending = true; - - m_scheduler.scheduleTask (*this); - } -} - -int NodeStore::BatchWriter::getWriteLoad () -{ - LockType::scoped_lock sl (mWriteMutex); - - return std::max (mWriteLoad, static_cast (mWriteSet.size ())); -} - -void NodeStore::BatchWriter::stopAsync () -{ - LockType::scoped_lock sl (mWriteMutex); - - if (! mWritePending) - m_callback.writeStopped (); -} - -void NodeStore::BatchWriter::performScheduledTask () -{ - writeBatch (); -} - -void NodeStore::BatchWriter::writeBatch () -{ - int setSize = 0; - - for (;;) - { - std::vector< boost::shared_ptr > set; - - set.reserve (batchWritePreallocationSize); - - { - LockType::scoped_lock sl (mWriteMutex); - - mWriteSet.swap (set); - assert (mWriteSet.empty ()); - ++mWriteGeneration; - mWriteCondition.notify_all (); - - if (set.empty ()) - { - mWritePending = false; - mWriteLoad = 0; - - m_callback.writeStopped (); - - // VFALCO NOTE Fix this function to not return from the middle - return; - } - - int newSetSize = mWriteSet.size(); - mWriteLoad = std::max (setSize, newSetSize); - newSetSize = set.size (); - } - - m_callback.writeBatch (set); - } -} - -void NodeStore::BatchWriter::waitForWriting () -{ - LockType::scoped_lock sl (mWriteMutex); - int gen = mWriteGeneration; - - while (mWritePending && (mWriteGeneration == gen)) - mWriteCondition.wait (sl); -} - -//============================================================================== - -class NodeStoreImp - : public NodeStore - , LeakChecked -{ -public: - NodeStoreImp (char const* name, - Service& parent, - Parameters const& backendParameters, - Parameters const& fastBackendParameters, - Scheduler& scheduler) - : NodeStore (name, parent) - , m_scheduler (scheduler) - , m_backend (createBackend (backendParameters, scheduler)) - , m_fastBackend ((fastBackendParameters.size () > 0) - ? createBackend (fastBackendParameters, scheduler) : nullptr) - , m_cache ("NodeStore", 16384, 300) - , m_negativeCache ("NoteStoreNegativeCache", 0, 120) - { - } - - ~NodeStoreImp () - { - } - - String getName () const - { - return m_backend->getName (); - } - - //------------------------------------------------------------------------------ - - NodeObject::Ptr fetch (uint256 const& hash) - { - // See if the object already exists in the cache - // - NodeObject::Ptr obj = m_cache.fetch (hash); - - if (obj == nullptr) - { - // It's not in the cache, see if we can skip checking the db. - // - if (! m_negativeCache.isPresent (hash)) - { - // There's still a chance it could be in one of the databases. - - bool foundInFastBackend = false; - - // Check the fast backend database if we have one - // - if (m_fastBackend != nullptr) - { - obj = fetchInternal (m_fastBackend, hash); - - // If we found the object, avoid storing it again later. - if (obj != nullptr) - foundInFastBackend = true; - } - - // Are we still without an object? - // - if (obj == nullptr) - { - // Yes so at last we will try the main database. - // - { - // Monitor this operation's load since it is expensive. - // - // VFALCO TODO Why is this an autoptr? Why can't it just be a plain old object? - // - // VFALCO NOTE Commented this out because it breaks the unit test! - // - //LoadEvent::autoptr event (getApp().getJobQueue ().getLoadEventAP (jtHO_READ, "HOS::retrieve")); - - obj = fetchInternal (m_backend, hash); - } - - // If it's not in the main database, remember that so we - // can skip the lookup for the same object again later. - // - if (obj == nullptr) - m_negativeCache.add (hash); - } - - // Did we finally get something? - // - if (obj != nullptr) - { - // Yes it so canonicalize. This solves the problem where - // more than one thread has its own copy of the same object. - // - m_cache.canonicalize (hash, obj); - - if (! foundInFastBackend) - { - // If we have a fast back end, store it there for later. - // - if (m_fastBackend != nullptr) - m_fastBackend->store (obj); - - // Since this was a 'hard' fetch, we will log it. - // - WriteLog (lsTRACE, NodeObject) << "HOS: " << hash << " fetch: in db"; - } - } - } - else - { - // hash is known not to be in the database - } - } - else - { - // found it! - } - - return obj; - } - - NodeObject::Ptr fetchInternal (Backend* backend, uint256 const& hash) - { - NodeObject::Ptr object; - - Backend::Status const status = backend->fetch (hash.begin (), &object); - - switch (status) - { - case Backend::ok: - case Backend::notFound: - break; - - case Backend::dataCorrupt: - // VFALCO TODO Deal with encountering corrupt data! - // - WriteLog (lsFATAL, NodeObject) << "Corrupt NodeObject #" << hash; - break; - - default: - WriteLog (lsWARNING, NodeObject) << "Unknown status=" << status; - break; - } - - return object; - } - - //------------------------------------------------------------------------------ - - void store (NodeObjectType type, - uint32 index, - Blob& data, - uint256 const& hash) - { - bool const keyFoundAndObjectCached = m_cache.refreshIfPresent (hash); - - // VFALCO NOTE What happens if the key is found, but the object - // fell out of the cache? We will end up passing it - // to the backend anyway. - // - if (! keyFoundAndObjectCached) - { - #if RIPPLE_VERIFY_NODEOBJECT_KEYS - assert (hash == Serializer::getSHA512Half (data)); - #endif - - NodeObject::Ptr object = NodeObject::createObject ( - type, index, data, hash); - - if (!m_cache.canonicalize (hash, object)) - { - m_backend->store (object); - - if (m_fastBackend) - m_fastBackend->store (object); - } - - m_negativeCache.del (hash); - } - } - - //------------------------------------------------------------------------------ - - float getCacheHitRate () - { - return m_cache.getHitRate (); - } - - void tune (int size, int age) - { - m_cache.setTargetSize (size); - m_cache.setTargetAge (age); - } - - void sweep () - { - m_cache.sweep (); - m_negativeCache.sweep (); - } - - int getWriteLoad () - { - return m_backend->getWriteLoad (); - } - - //------------------------------------------------------------------------------ - - void visitAll (Backend::VisitCallback& callback) - { - m_backend->visitAll (callback); - } - - void import (NodeStore& sourceDatabase) - { - class ImportVisitCallback : public Backend::VisitCallback - { - public: - explicit ImportVisitCallback (Backend& backend) - : m_backend (backend) - { - m_objects.reserve (batchWritePreallocationSize); - } - - ~ImportVisitCallback () - { - if (! m_objects.empty ()) - m_backend.storeBatch (m_objects); - } - - void visitObject (NodeObject::Ptr const& object) - { - if (m_objects.size () >= batchWritePreallocationSize) - { - m_backend.storeBatch (m_objects); - - m_objects.clear (); - m_objects.reserve (batchWritePreallocationSize); - } - - m_objects.push_back (object); - } - - private: - Backend& m_backend; - Batch m_objects; - }; - - //-------------------------------------------------------------------------- - - ImportVisitCallback callback (*m_backend); - - sourceDatabase.visitAll (callback); - } - - //------------------------------------------------------------------------------ - // - // Service - - void onServiceStop () - { - // notify the Backend - m_backend->stopAsync(); - - if (m_fastBackend) - m_fastBackend->stopAsync(); - } - - //------------------------------------------------------------------------------ - - static void missing_backend () - { - fatal_error ("Your rippled.cfg is missing a [node_db] entry, please see the rippled-example.cfg file!"); - } - - static NodeStore::Backend* createBackend ( - Parameters const& parameters, Scheduler& scheduler = getSynchronousScheduler ()) - { - Backend* backend = nullptr; - - String const& type = parameters ["type"]; - - if (type.isNotEmpty ()) - { - BackendFactory* factory (Factories::get().find (type)); - - if (factory != nullptr) - { - backend = factory->createInstance (NodeObject::keyBytes, parameters, scheduler); - } - else - { - missing_backend (); - } - } - else - { - missing_backend (); - } - - return backend; - } - - //------------------------------------------------------------------------------ - -private: - static Array s_factories; - - Scheduler& m_scheduler; - - // Persistent key/value storage. - ScopedPointer m_backend; - - // Larger key/value storage, but not necessarily persistent. - ScopedPointer m_fastBackend; - - // VFALCO NOTE What are these things for? We need comments. - TaggedCacheType m_cache; - KeyCache m_negativeCache; -}; - -Array NodeStoreImp::s_factories; - -//------------------------------------------------------------------------------ - -void NodeStore::addBackendFactory (BackendFactory* factory) -{ - Factories::get().add (factory); -} - -NodeStore::Scheduler& NodeStore::getSynchronousScheduler () -{ - // Simple scheduler that performs the task immediately - struct SynchronousScheduler : Scheduler - { - void scheduleTask (Task& task) - { - task.performScheduledTask (); - } - - void scheduledTasksStopped () - { - } - }; - - static SynchronousScheduler scheduler; - - return scheduler; -} - -void NodeStore::addAvailableBackends () -{ - // This is part of the ripple_app module since it has dependencies - //NodeStore::addBackendFactory (SqliteBackendFactory::getInstance ()); - - NodeStore::addBackendFactory (LevelDBBackendFactory::getInstance ()); - - NodeStore::addBackendFactory (MemoryBackendFactory::getInstance ()); - NodeStore::addBackendFactory (NullBackendFactory::getInstance ()); - -#if RIPPLE_HYPERLEVELDB_AVAILABLE - NodeStore::addBackendFactory (HyperLevelDBBackendFactory::getInstance ()); -#endif - -#if RIPPLE_MDB_AVAILABLE - NodeStore::addBackendFactory (MdbBackendFactory::getInstance ()); -#endif - -#if RIPPLE_SOPHIA_AVAILABLE - NodeStore::addBackendFactory (SophiaBackendFactory::getInstance ()); -#endif - - NodeStore::addBackendFactory (KeyvaDBBackendFactory::getInstance ()); -} - -//------------------------------------------------------------------------------ - -NodeStore::NodeStore (char const* name, Service& parent) - : Service (name, parent) -{ -} - -//------------------------------------------------------------------------------ - -NodeStore* NodeStore::New (char const* name, - Service& parent, - Parameters const& backendParameters, - Parameters fastBackendParameters, - Scheduler& scheduler) -{ - return new NodeStoreImp (name, - parent, - backendParameters, - fastBackendParameters, - scheduler); -} - -//============================================================================== - -// Some common code for the unit tests -// -class NodeStoreUnitTest : public UnitTest -{ -public: - // Tunable parameters - // - enum - { - maxPayloadBytes = 2000, - numObjectsToTest = 10000 - }; - - // Shorthand type names - // - typedef NodeStore::Backend Backend; - typedef NodeStore::Batch Batch; - - // Creates predictable objects - class PredictableObjectFactory - { - public: - explicit PredictableObjectFactory (int64 seedValue) - : m_seedValue (seedValue) - { - } - - NodeObject::Ptr createObject (int index) - { - Random r (m_seedValue + index); - - NodeObjectType type; - switch (r.nextInt (4)) - { - case 0: type = hotLEDGER; break; - case 1: type = hotTRANSACTION; break; - case 2: type = hotACCOUNT_NODE; break; - case 3: type = hotTRANSACTION_NODE; break; - default: - type = hotUNKNOWN; - break; - }; - - LedgerIndex ledgerIndex = 1 + r.nextInt (1024 * 1024); - - uint256 hash; - r.fillBitsRandomly (hash.begin (), hash.size ()); - - int const payloadBytes = 1 + r.nextInt (maxPayloadBytes); - - Blob data (payloadBytes); - - r.fillBitsRandomly (data.data (), payloadBytes); - - return NodeObject::createObject (type, ledgerIndex, data, hash); - } - - private: - int64 const m_seedValue; - }; - -public: - NodeStoreUnitTest (String name, UnitTest::When when = UnitTest::runNormal) - : UnitTest (name, "ripple", when) - { - } - - // Create a predictable batch of objects - static void createPredictableBatch (Batch& batch, int startingIndex, int numObjects, int64 seedValue) - { - batch.reserve (numObjects); - - PredictableObjectFactory factory (seedValue); - - for (int i = 0; i < numObjects; ++i) - batch.push_back (factory.createObject (startingIndex + i)); - } - - // Compare two batches for equality - static bool areBatchesEqual (Batch const& lhs, Batch const& rhs) - { - bool result = true; - - if (lhs.size () == rhs.size ()) - { - for (int i = 0; i < lhs.size (); ++i) - { - if (! lhs [i]->isCloneOf (rhs [i])) - { - result = false; - break; - } - } - } - else - { - result = false; - } - - return result; - } - - // Store a batch in a backend - void storeBatch (Backend& backend, Batch const& batch) - { - for (int i = 0; i < batch.size (); ++i) - { - backend.store (batch [i]); - } - } - - // Get a copy of a batch in a backend - void fetchCopyOfBatch (Backend& backend, Batch* pCopy, Batch const& batch) - { - pCopy->clear (); - pCopy->reserve (batch.size ()); - - for (int i = 0; i < batch.size (); ++i) - { - NodeObject::Ptr object; - - Backend::Status const status = backend.fetch ( - batch [i]->getHash ().cbegin (), &object); - - expect (status == Backend::ok, "Should be ok"); - - if (status == Backend::ok) - { - expect (object != nullptr, "Should not be null"); - - pCopy->push_back (object); - } - } - } - - // Store all objects in a batch - static void storeBatch (NodeStore& db, NodeStore::Batch const& batch) - { - for (int i = 0; i < batch.size (); ++i) - { - NodeObject::Ptr const object (batch [i]); - - Blob data (object->getData ()); - - db.store (object->getType (), - object->getIndex (), - data, - object->getHash ()); - } - } - - // Fetch all the hashes in one batch, into another batch. - static void fetchCopyOfBatch (NodeStore& db, - NodeStore::Batch* pCopy, - NodeStore::Batch const& batch) - { - pCopy->clear (); - pCopy->reserve (batch.size ()); - - for (int i = 0; i < batch.size (); ++i) - { - NodeObject::Ptr object = db.fetch (batch [i]->getHash ()); - - if (object != nullptr) - pCopy->push_back (object); - } - } -}; - -//------------------------------------------------------------------------------ - -// Tests predictable batches, and NodeObject blob encoding -// -class NodeStoreBasicsTests : public NodeStoreUnitTest -{ -public: - typedef NodeStore::EncodedBlob EncodedBlob; - typedef NodeStore::DecodedBlob DecodedBlob; - - NodeStoreBasicsTests () : NodeStoreUnitTest ("NodeStoreBasics") - { - } - - // Make sure predictable object generation works! - void testBatches (int64 const seedValue) - { - beginTestCase ("batch"); - - Batch batch1; - createPredictableBatch (batch1, 0, numObjectsToTest, seedValue); - - Batch batch2; - createPredictableBatch (batch2, 0, numObjectsToTest, seedValue); - - expect (areBatchesEqual (batch1, batch2), "Should be equal"); - - Batch batch3; - createPredictableBatch (batch3, 1, numObjectsToTest, seedValue); - - expect (! areBatchesEqual (batch1, batch3), "Should not be equal"); - } - - // Checks encoding/decoding blobs - void testBlobs (int64 const seedValue) - { - beginTestCase ("encoding"); - - Batch batch; - createPredictableBatch (batch, 0, numObjectsToTest, seedValue); - - EncodedBlob encoded; - for (int i = 0; i < batch.size (); ++i) - { - encoded.prepare (batch [i]); - - DecodedBlob decoded (encoded.getKey (), encoded.getData (), encoded.getSize ()); - - expect (decoded.wasOk (), "Should be ok"); - - if (decoded.wasOk ()) - { - NodeObject::Ptr const object (decoded.createObject ()); - - expect (batch [i]->isCloneOf (object), "Should be clones"); - } - } - } - - void runTest () - { - int64 const seedValue = 50; - - testBatches (seedValue); - - testBlobs (seedValue); - } -}; - -static NodeStoreBasicsTests nodeStoreBasicsTests; - -//------------------------------------------------------------------------------ - -// Tests the NodeStore::Backend interface -// -class NodeStoreBackendTests : public NodeStoreUnitTest -{ -public: - NodeStoreBackendTests () : NodeStoreUnitTest ("NodeStoreBackend") - { - } - - //-------------------------------------------------------------------------- - - void testBackend (String type, int64 const seedValue) - { - beginTestCase (String ("NodeStore::Backend type=") + type); - - StringPairArray params; - File const path (File::createTempFile ("node_db")); - params.set ("type", type); - params.set ("path", path.getFullPathName ()); - - // Create a batch - NodeStore::Batch batch; - createPredictableBatch (batch, 0, numObjectsToTest, seedValue); - - { - // Open the backend - ScopedPointer backend (NodeStoreImp::createBackend (params)); - - // Write the batch - storeBatch (*backend, batch); - - { - // Read it back in - NodeStore::Batch copy; - fetchCopyOfBatch (*backend, ©, batch); - expect (areBatchesEqual (batch, copy), "Should be equal"); - } - - { - // Reorder and read the copy again - NodeStore::Batch copy; - UnitTestUtilities::repeatableShuffle (batch.size (), batch, seedValue); - fetchCopyOfBatch (*backend, ©, batch); - expect (areBatchesEqual (batch, copy), "Should be equal"); - } - } - - { - // Re-open the backend - ScopedPointer backend (NodeStoreImp::createBackend (params)); - - // Read it back in - NodeStore::Batch copy; - fetchCopyOfBatch (*backend, ©, batch); - // Canonicalize the source and destination batches - std::sort (batch.begin (), batch.end (), NodeObject::LessThan ()); - std::sort (copy.begin (), copy.end (), NodeObject::LessThan ()); - expect (areBatchesEqual (batch, copy), "Should be equal"); - } - } - - //-------------------------------------------------------------------------- - - void runTest () - { - int const seedValue = 50; - - testBackend ("leveldb", seedValue); - - testBackend ("sqlite", seedValue); - - #if RIPPLE_HYPERLEVELDB_AVAILABLE - testBackend ("hyperleveldb", seedValue); - #endif - - #if RIPPLE_MDB_AVAILABLE - testBackend ("mdb", seedValue); - #endif - - #if RIPPLE_SOPHIA_AVAILABLE - testBackend ("sophia", seedValue); - #endif - } -}; - -static NodeStoreBackendTests nodeStoreBackendTests; - -//------------------------------------------------------------------------------ - -class NodeStoreTimingTests : public NodeStoreUnitTest -{ -public: - enum - { - numObjectsToTest = 20000 - }; - - NodeStoreTimingTests () - : NodeStoreUnitTest ("NodeStoreTiming", UnitTest::runManual) - { - } - - class Stopwatch - { - public: - Stopwatch () - { - } - - void start () - { - m_startTime = Time::getHighResolutionTicks (); - } - - double getElapsed () - { - int64 const now = Time::getHighResolutionTicks(); - - return Time::highResolutionTicksToSeconds (now - m_startTime); - } - - private: - int64 m_startTime; - }; - - //-------------------------------------------------------------------------- - - void testBackend (String type, int64 const seedValue) - { - String s; - s << "Testing backend '" << type << "' performance"; - beginTestCase (s); - - StringPairArray params; - File const path (File::createTempFile ("node_db")); - params.set ("type", type); - params.set ("path", path.getFullPathName ()); - - // Create batches - NodeStore::Batch batch1; - createPredictableBatch (batch1, 0, numObjectsToTest, seedValue); - NodeStore::Batch batch2; - createPredictableBatch (batch2, 0, numObjectsToTest, seedValue); - - // Open the backend - ScopedPointer backend (NodeStoreImp::createBackend (params)); - - Stopwatch t; - - // Individual write batch test - t.start (); - storeBatch (*backend, batch1); - s = ""; - s << " Single write: " << String (t.getElapsed (), 2) << " seconds"; - logMessage (s); - - // Bulk write batch test - t.start (); - backend->storeBatch (batch2); - s = ""; - s << " Batch write: " << String (t.getElapsed (), 2) << " seconds"; - logMessage (s); - - // Read test - Batch copy; - t.start (); - fetchCopyOfBatch (*backend, ©, batch1); - fetchCopyOfBatch (*backend, ©, batch2); - s = ""; - s << " Batch read: " << String (t.getElapsed (), 2) << " seconds"; - logMessage (s); - } - - //-------------------------------------------------------------------------- - - void runTest () - { - int const seedValue = 50; - - testBackend ("leveldb", seedValue); - - #if RIPPLE_HYPERLEVELDB_AVAILABLE - testBackend ("hyperleveldb", seedValue); - #endif - - /* - #if RIPPLE_MDB_AVAILABLE - testBackend ("mdb", seedValue); - #endif - */ - #if RIPPLE_SOPHIA_AVAILABLE - testBackend ("sophia", seedValue); - #endif - - /* - testBackend ("sqlite", seedValue); - */ - } -}; - -static NodeStoreTimingTests nodeStoreTimingTests; - -//------------------------------------------------------------------------------ - -class NodeStoreTests : public NodeStoreUnitTest -{ -public: - NodeStoreTests () - : NodeStoreUnitTest ("NodeStore") - { - } - - ~NodeStoreTests () - { - } - - //-------------------------------------------------------------------------- - - void testImport (String destBackendType, String srcBackendType, int64 seedValue) - { - File const node_db (File::createTempFile ("node_db")); - StringPairArray srcParams; - srcParams.set ("type", srcBackendType); - srcParams.set ("path", node_db.getFullPathName ()); - - // Create a batch - NodeStore::Batch batch; - createPredictableBatch (batch, 0, numObjectsToTest, seedValue); - - // Write to source db - { - ScopedService service ("test"); - ScopedPointer src (NodeStore::New ("test", service, srcParams)); - storeBatch (*src, batch); - service.serviceStop(); - } - - NodeStore::Batch copy; - - { - ScopedService service ("test"); - - // Re-open the db - ScopedPointer src (NodeStore::New ("test", service, srcParams)); - - // Set up the destination database - File const dest_db (File::createTempFile ("dest_db")); - StringPairArray destParams; - destParams.set ("type", destBackendType); - destParams.set ("path", dest_db.getFullPathName ()); - - ScopedPointer dest (NodeStore::New ("test", service, destParams)); - - beginTestCase (String ("import into '") + destBackendType + "' from '" + srcBackendType + "'"); - - // Do the import - dest->import (*src); - - // Get the results of the import - fetchCopyOfBatch (*dest, ©, batch); - - service.serviceStop(); - } - - // Canonicalize the source and destination batches - std::sort (batch.begin (), batch.end (), NodeObject::LessThan ()); - std::sort (copy.begin (), copy.end (), NodeObject::LessThan ()); - expect (areBatchesEqual (batch, copy), "Should be equal"); - - } - - //-------------------------------------------------------------------------- - - void testNodeStore (String type, - bool const useEphemeralDatabase, - bool const testPersistence, - int64 const seedValue) - { - String s; - s << String ("NodeStore backend '") + type + "'"; - if (useEphemeralDatabase) - s << " (with ephemeral database)"; - - beginTestCase (s); - - File const node_db (File::createTempFile ("node_db")); - StringPairArray nodeParams; - nodeParams.set ("type", type); - nodeParams.set ("path", node_db.getFullPathName ()); - - File const temp_db (File::createTempFile ("temp_db")); - StringPairArray tempParams; - if (useEphemeralDatabase) - { - tempParams.set ("type", type); - tempParams.set ("path", temp_db.getFullPathName ()); - } - - // Create a batch - NodeStore::Batch batch; - createPredictableBatch (batch, 0, numObjectsToTest, seedValue); - - { - ScopedService service ("test"); - - // Open the database - ScopedPointer db (NodeStore::New ("test", service, nodeParams, tempParams)); - - // Write the batch - storeBatch (*db, batch); - - { - // Read it back in - NodeStore::Batch copy; - fetchCopyOfBatch (*db, ©, batch); - expect (areBatchesEqual (batch, copy), "Should be equal"); - } - - { - // Reorder and read the copy again - NodeStore::Batch copy; - UnitTestUtilities::repeatableShuffle (batch.size (), batch, seedValue); - fetchCopyOfBatch (*db, ©, batch); - expect (areBatchesEqual (batch, copy), "Should be equal"); - } - - service.serviceStop(); - } - - if (testPersistence) - { - { - ScopedService service ("test"); - - // Re-open the database without the ephemeral DB - ScopedPointer db (NodeStore::New ("test", service, nodeParams)); - - // Read it back in - NodeStore::Batch copy; - fetchCopyOfBatch (*db, ©, batch); - - // Canonicalize the source and destination batches - std::sort (batch.begin (), batch.end (), NodeObject::LessThan ()); - std::sort (copy.begin (), copy.end (), NodeObject::LessThan ()); - expect (areBatchesEqual (batch, copy), "Should be equal"); - - service.serviceStop(); - } - - if (useEphemeralDatabase) - { - ScopedService service ("test"); - - // Verify the ephemeral db - ScopedPointer db (NodeStore::New ("test", - service, tempParams, StringPairArray ())); - - // Read it back in - NodeStore::Batch copy; - fetchCopyOfBatch (*db, ©, batch); - - // Canonicalize the source and destination batches - std::sort (batch.begin (), batch.end (), NodeObject::LessThan ()); - std::sort (copy.begin (), copy.end (), NodeObject::LessThan ()); - expect (areBatchesEqual (batch, copy), "Should be equal"); - - service.serviceStop(); - } - } - } - - //-------------------------------------------------------------------------- - - void runBackendTests (bool useEphemeralDatabase, int64 const seedValue) - { - testNodeStore ("leveldb", useEphemeralDatabase, true, seedValue); - - testNodeStore ("sqlite", useEphemeralDatabase, true, seedValue); - - #if RIPPLE_HYPERLEVELDB_AVAILABLE - testNodeStore ("hyperleveldb", useEphemeralDatabase, true, seedValue); - #endif - - #if RIPPLE_MDB_AVAILABLE - testNodeStore ("mdb", useEphemeralDatabase, true, seedValue); - #endif - - #if RIPPLE_SOPHIA_AVAILABLE - testNodeStore ("sophia", useEphemeralDatabase, true, seedValue); - #endif - } - - //-------------------------------------------------------------------------- - - void runImportTests (int64 const seedValue) - { - testImport ("leveldb", "leveldb", seedValue); - - #if RIPPLE_HYPERLEVELDB_AVAILABLE - testImport ("hyperleveldb", "hyperleveldb", seedValue); - #endif - - #if RIPPLE_MDB_AVAILABLE - //testImport ("mdb", "mdb", seedValue); - #endif - - #if RIPPLE_SOPHIA_AVAILABLE - //testImport ("sophia", "sophia", seedValue); - #endif - - testImport ("sqlite", "sqlite", seedValue); - } - - //-------------------------------------------------------------------------- - - void runTest () - { - int64 const seedValue = 50; - - testNodeStore ("memory", false, false, seedValue); - - runBackendTests (false, seedValue); - - runBackendTests (true, seedValue); - - runImportTests (seedValue); - } -}; - -static NodeStoreTests nodeStoreTests; diff --git a/src/ripple_core/node/NodeStore.h b/src/ripple_core/node/NodeStore.h deleted file mode 100644 index 6b62f7456e..0000000000 --- a/src/ripple_core/node/NodeStore.h +++ /dev/null @@ -1,448 +0,0 @@ -//------------------------------------------------------------------------------ -/* - Copyright (c) 2011-2013, OpenCoin, Inc. -*/ -//============================================================================== - -#ifndef RIPPLE_CORE_NODE_NODESTORE_H_INCLUDED -#define RIPPLE_CORE_NODE_NODESTORE_H_INCLUDED - -/** Persistency layer for NodeObject - - A Node is a ledger object which is uniquely identified by a key, which is - the 256-bit hash of the body of the node. The payload is a variable length - block of serialized data. - - All ledger data is stored as node objects and as such, needs to be persisted - between launches. Furthermore, since the set of node objects will in - general be larger than the amount of available memory, purged node objects - which are later accessed must be retrieved from the node store. - - @see NodeObject -*/ -class NodeStore : public Service -{ -protected: - NodeStore (char const* name, Service& parent); - -public: - enum - { - // This is only used to pre-allocate the array for - // batch objects and does not affect the amount written. - // - batchWritePreallocationSize = 128 - }; - - typedef std::vector Batch; - - typedef StringPairArray Parameters; - - //-------------------------------------------------------------------------- - - /** Parsed key/value blob into NodeObject components. - - This will extract the information required to construct a NodeObject. It - also does consistency checking and returns the result, so it is possible - to determine if the data is corrupted without throwing an exception. Not - all forms of corruption are detected so further analysis will be needed - to eliminate false negatives. - - @note This defines the database format of a NodeObject! - */ - class DecodedBlob - { - public: - /** Construct the decoded blob from raw data. */ - DecodedBlob (void const* key, void const* value, int valueBytes); - - /** Determine if the decoding was successful. */ - bool wasOk () const noexcept { return m_success; } - - /** Create a NodeObject from this data. */ - NodeObject::Ptr createObject (); - - private: - bool m_success; - - void const* m_key; - LedgerIndex m_ledgerIndex; - NodeObjectType m_objectType; - unsigned char const* m_objectData; - int m_dataBytes; - }; - - //-------------------------------------------------------------------------- - - /** Utility for producing flattened node objects. - - These get recycled to prevent many small allocations. - - @note This defines the database format of a NodeObject! - */ - struct EncodedBlob - { - typedef RecycledObjectPool Pool; - - void prepare (NodeObject::Ptr const& object); - - void const* getKey () const noexcept { return m_key; } - - size_t getSize () const noexcept { return m_size; } - - void const* getData () const noexcept { return m_data.getData (); } - - private: - void const* m_key; - MemoryBlock m_data; - size_t m_size; - }; - - //-------------------------------------------------------------------------- - - /** Provides optional asynchronous scheduling for backends. - - For improved performance, a backend has the option of performing writes - in batches. These writes can be scheduled using the provided scheduler - object. - - @see BatchWriter - */ - class Scheduler - { - public: - /** Derived classes perform scheduled tasks. */ - struct Task - { - virtual ~Task () { } - - /** Performs the task. - The call may take place on a foreign thread. - */ - virtual void performScheduledTask () = 0; - }; - - /** Schedules a task. - - Depending on the implementation, this could happen - immediately or get deferred. - */ - virtual void scheduleTask (Task& task) = 0; - - /** Notifies the scheduler that all tasks are complete. */ - virtual void scheduledTasksStopped () = 0; - }; - - //-------------------------------------------------------------------------- - - /** Helps with batch writing. - - The batch writes are performed with a scheduled task. Use of the - class it not required. A backend can implement its own write batching, - or skip write batching if doing so yields a performance benefit. - - @see Scheduler - */ - // VFALCO NOTE I'm not entirely happy having placed this here, - // because whoever needs to use NodeStore certainly doesn't - // need to see the implementation details of BatchWriter. - // - class BatchWriter : private Scheduler::Task - { - public: - /** This callback does the actual writing. */ - struct Callback - { - virtual void writeBatch (Batch const& batch) = 0; - - // Called after stopAsync when there is no more pending write - virtual void writeStopped () = 0; - }; - - /** Create a batch writer. */ - BatchWriter (Callback& callback, Scheduler& scheduler); - - /** Destroy a batch writer. - - Anything pending in the batch is written out before this returns. - */ - ~BatchWriter (); - - /** Store the object. - - This will add to the batch and initiate a scheduled task to - write the batch out. - */ - void store (NodeObject::Ptr const& object); - - /** Get an estimate of the amount of writing I/O pending. */ - int getWriteLoad (); - - /** Called to notify that the NodeStore wants to stop. */ - void stopAsync (); - - private: - void performScheduledTask (); - void writeBatch (); - void waitForWriting (); - - private: - typedef boost::recursive_mutex LockType; - typedef boost::condition_variable_any CondvarType; - - Callback& m_callback; - Scheduler& m_scheduler; - LockType mWriteMutex; - CondvarType mWriteCondition; - int mWriteGeneration; - int mWriteLoad; - bool mWritePending; - Batch mWriteSet; - }; - - //-------------------------------------------------------------------------- - - /** A backend used for the store. - - The NodeStore uses a swappable backend so that other database systems - can be tried. Different databases may offer various features such - as improved performance, fault tolerant or distributed storage, or - all in-memory operation. - - A given instance of a backend is fixed to a particular key size. - */ - class Backend - { - public: - /** Return codes from operations. */ - enum Status - { - ok, - notFound, - dataCorrupt, - unknown - }; - - /** Destroy the backend. - - All open files are closed and flushed. If there are batched writes - or other tasks scheduled, they will be completed before this call - returns. - */ - virtual ~Backend () { } - - /** Get the human-readable name of this backend. - - This is used for diagnostic output. - */ - virtual std::string getName() = 0; - - /** Fetch a single object. - - If the object is not found or an error is encountered, the - result will indicate the condition. - - @note This will be called concurrently. - - @param key A pointer to the key data. - @param pObject [out] The created object if successful. - - @return The result of the operation. - */ - virtual Status fetch (void const* key, NodeObject::Ptr* pObject) = 0; - - /** Store a single object. - - Depending on the implementation this may happen immediately - or deferred using a scheduled task. - - @note This will be called concurrently. - - @param object The object to store. - */ - virtual void store (NodeObject::Ptr const& object) = 0; - - /** Store a group of objects. - - @note This function will not be called concurrently with - itself or @ref store. - */ - virtual void storeBatch (Batch const& batch) = 0; - - /** Callback for iterating through objects. - - @see visitAll - */ - struct VisitCallback - { - virtual void visitObject (NodeObject::Ptr const& object) = 0; - }; - - /** Visit every object in the database - - This is usually called during import. - - @note This routine will not be called concurrently with itself - or other methods. - - @see import, VisitCallback - */ - virtual void visitAll (VisitCallback& callback) = 0; - - /** Estimate the number of write operations pending. */ - virtual int getWriteLoad () = 0; - - /** Called to notify the Backend that the NodeStore wants to stop. */ - virtual void stopAsync () = 0; - }; - - //-------------------------------------------------------------------------- - - /** Factory to produce backends. - */ - class BackendFactory - { - public: - virtual ~BackendFactory () { } - - /** Retrieve the name of this factory. */ - virtual String getName () const = 0; - - /** Create an instance of this factory's backend. - - @param keyBytes The fixed number of bytes per key. - @param keyValues A set of key/value configuration pairs. - @param scheduler The scheduler to use for running tasks. - - @return A pointer to the Backend object. - */ - virtual Backend* createInstance (size_t keyBytes, - Parameters const& parameters, - Scheduler& scheduler) = 0; - }; - - //-------------------------------------------------------------------------- - - /** Construct a node store. - - The parameters are key value pairs passed to the backend. The - 'type' key must exist, it defines the choice of backend. Most - backends also require a 'path' field. - - Some choices for 'type' are: - HyperLevelDB, LevelDB, SQLite, KeyvaDB, MDB - - If the fastBackendParameter is omitted or empty, no ephemeral database - is used. If the scheduler parameter is omited or unspecified, a - synchronous scheduler is used which performs all tasks immediately on - the caller's thread. - - @note If the database cannot be opened or created, an exception is thrown. - - @param backendParameters The parameter string for the persistent backend. - @param fastBackendParameters [optional] The parameter string for the ephemeral backend. - @param scheduler [optional The scheduler to use for performing asynchronous tasks. - - @return The opened database. - */ - static NodeStore* New (char const* name, - Service& parent, - Parameters const& backendParameters, - Parameters fastBackendParameters = Parameters (), - Scheduler& scheduler = getSynchronousScheduler ()); - - /** Get the synchronous scheduler. - - The synchronous scheduler performs all tasks immediately, before - returning to the caller, using the caller's thread. - */ - static Scheduler& getSynchronousScheduler (); - - /** Destroy the node store. - - All pending operations are completed, pending writes flushed, - and files closed before this returns. - */ - virtual ~NodeStore () { } - - /** Retrieve the name associated with this backend. - - This is used for diagnostics and may not reflect the actual path - or paths used by the underlying backend. - */ - virtual String getName () const = 0; - - /** Add the specified backend factory to the list of available factories. - The names of available factories are compared against the "type" - value in the parameter list on construction. Ownership of the object - is transferred. The object must be allocated using new. - @param factory The factory to add. - */ - static void addBackendFactory (BackendFactory* factory); - - /** Fetch an object. - - If the object is known to be not in the database, isn't found in the - database during the fetch, or failed to load correctly during the fetch, - `nullptr` is returned. - - @note This can be called concurrently. - - @param hash The key of the object to retrieve. - - @return The object, or nullptr if it couldn't be retrieved. - */ - virtual NodeObject::pointer fetch (uint256 const& hash) = 0; - - /** Store the object. - - The caller's Blob parameter is overwritten. - - @param type The type of object. - @param ledgerIndex The ledger in which the object appears. - @param data The payload of the object. The caller's - variable is overwritten. - @param hash The 256-bit hash of the payload data. - - @return `true` if the object was stored? - */ - virtual void store (NodeObjectType type, - uint32 ledgerIndex, - Blob& data, - uint256 const& hash) = 0; - - /** Visit every object in the database - - This is usually called during import. - - @note This routine will not be called concurrently with itself - or other methods. - - @see import - */ - virtual void visitAll (Backend::VisitCallback& callback) = 0; - - /** Import objects from another database. */ - virtual void import (NodeStore& sourceDatabase) = 0; - - /** Retrieve the estimated number of pending write operations. - This is used for diagnostics. - */ - virtual int getWriteLoad () = 0; - - // VFALCO TODO Document this. - virtual float getCacheHitRate () = 0; - - // VFALCO TODO Document this. - // TODO Document the parameter meanings. - virtual void tune (int size, int age) = 0; - - // VFALCO TODO Document this. - virtual void sweep () = 0; - - /** Add the known Backend factories to the singleton. - */ - static void addAvailableBackends (); -}; - -#endif diff --git a/src/ripple_core/node/NullBackendFactory.cpp b/src/ripple_core/node/NullBackendFactory.cpp deleted file mode 100644 index 8479480f22..0000000000 --- a/src/ripple_core/node/NullBackendFactory.cpp +++ /dev/null @@ -1,84 +0,0 @@ -//------------------------------------------------------------------------------ -/* - Copyright (c) 2011-2013, OpenCoin, Inc. -*/ -//============================================================================== - -class NullBackendFactory::Backend : public NodeStore::Backend -{ -public: - explicit Backend (NodeStore::Scheduler& scheduler) - : m_scheduler (scheduler) - { - } - - ~Backend () - { - } - - std::string getName() - { - return std::string (); - } - - Status fetch (void const*, NodeObject::Ptr*) - { - return notFound; - } - - void store (NodeObject::ref object) - { - } - - void storeBatch (NodeStore::Batch const& batch) - { - } - - void visitAll (VisitCallback& callback) - { - } - - int getWriteLoad () - { - return 0; - } - - void stopAsync () - { - m_scheduler.scheduledTasksStopped (); - } - -private: - NodeStore::Scheduler& m_scheduler; -}; - -//------------------------------------------------------------------------------ - -NullBackendFactory::NullBackendFactory () -{ -} - -NullBackendFactory::~NullBackendFactory () -{ -} - -NullBackendFactory* NullBackendFactory::getInstance () -{ - return new NullBackendFactory; -} - -String NullBackendFactory::getName () const -{ - return "none"; -} - -NodeStore::Backend* NullBackendFactory::createInstance ( - size_t, - StringPairArray const&, - NodeStore::Scheduler& scheduler) -{ - return new NullBackendFactory::Backend (scheduler); -} - -//------------------------------------------------------------------------------ - diff --git a/src/ripple_core/node/NullBackendFactory.h b/src/ripple_core/node/NullBackendFactory.h deleted file mode 100644 index 2d4bc0dab3..0000000000 --- a/src/ripple_core/node/NullBackendFactory.h +++ /dev/null @@ -1,34 +0,0 @@ -//------------------------------------------------------------------------------ -/* - Copyright (c) 2011-2013, OpenCoin, Inc. -*/ -//============================================================================== - -#ifndef RIPPLE_CORE_NODE_NULLBACKENDFACTORY_H_INCLUDED -#define RIPPLE_CORE_NODE_NULLBACKENDFACTORY_H_INCLUDED - -/** Factory to produce a null backend. - - This is for standalone / testing mode. - - @see NodeStore -*/ -class NullBackendFactory : public NodeStore::BackendFactory -{ -private: - class Backend; - - NullBackendFactory (); - ~NullBackendFactory (); - -public: - static NullBackendFactory* getInstance (); - - String getName () const; - - NodeStore::Backend* createInstance (size_t keyBytes, - StringPairArray const& keyValues, - NodeStore::Scheduler& scheduler); -}; - -#endif diff --git a/src/ripple_core/node/SophiaBackendFactory.h b/src/ripple_core/node/SophiaBackendFactory.h deleted file mode 100644 index 8163104f14..0000000000 --- a/src/ripple_core/node/SophiaBackendFactory.h +++ /dev/null @@ -1,32 +0,0 @@ -//------------------------------------------------------------------------------ -/* - Copyright (c) 2011-2013, OpenCoin, Inc. -*/ -//============================================================================== - -#ifndef RIPPLE_CORE_SOPHIABACKENDFACTORY_H_INCLUDED -#define RIPPLE_CORE_SOPHIABACKENDFACTORY_H_INCLUDED - -/** Factory to produce Sophia backends for the NodeStore. - - @see NodeStore -*/ -class SophiaBackendFactory : public NodeStore::BackendFactory -{ -private: - class Backend; - - SophiaBackendFactory (); - ~SophiaBackendFactory (); - -public: - static SophiaBackendFactory* getInstance (); - - String getName () const; - - NodeStore::Backend* createInstance (size_t keyBytes, - StringPairArray const& keyValues, - NodeStore::Scheduler& scheduler); -}; - -#endif diff --git a/src/ripple_core/nodestore/NodeStore.cpp b/src/ripple_core/nodestore/NodeStore.cpp new file mode 100644 index 0000000000..d607a23040 --- /dev/null +++ b/src/ripple_core/nodestore/NodeStore.cpp @@ -0,0 +1,49 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +// backend support +#include "beast/modules/beast_db/beast_db.h" +#include "../ripple_hyperleveldb/ripple_hyperleveldb.h" +#include "../ripple_leveldb/ripple_leveldb.h" +#include "../ripple_mdb/ripple_mdb.h" +#include "../ripple/sophia/ripple_sophia.h" + +namespace ripple +{ + +# include "impl/DecodedBlob.h" +# include "impl/EncodedBlob.h" +# include "impl/BatchWriter.h" +# include "backend/HyperDBFactory.h" +#include "backend/HyperDBFactory.cpp" +# include "backend/KeyvaDBFactory.h" +#include "backend/KeyvaDBFactory.cpp" +# include "backend/LevelDBFactory.h" +#include "backend/LevelDBFactory.cpp" +# include "backend/MemoryFactory.h" +#include "backend/MemoryFactory.cpp" +# include "backend/NullFactory.h" +#include "backend/NullFactory.cpp" +# include "backend/MdbFactory.h" +#include "backend/MdbFactory.cpp" +# include "backend/SophiaFactory.h" +#include "backend/SophiaFactory.cpp" + +#include "impl/BatchWriter.cpp" +# include "impl/Factories.h" +# include "impl/DatabaseImp.h" +#include "impl/DummyScheduler.cpp" +#include "impl/DecodedBlob.cpp" +#include "impl/EncodedBlob.cpp" +#include "impl/NodeObject.cpp" + +# include "tests/TestBase.h" +#include "tests/BackendTests.cpp" +#include "tests/BasicTests.cpp" +#include "tests/DatabaseTests.cpp" +#include "tests/TimingTests.cpp" + +} diff --git a/src/ripple_core/nodestore/NodeStore.h b/src/ripple_core/nodestore/NodeStore.h new file mode 100644 index 0000000000..33b4440832 --- /dev/null +++ b/src/ripple_core/nodestore/NodeStore.h @@ -0,0 +1,25 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_H_INCLUDED +#define RIPPLE_NODESTORE_H_INCLUDED + +namespace ripple +{ + +#include "api/NodeObject.h" +#include "api/Types.h" +#include "api/VisitCallback.h" +#include "api/Backend.h" +# include "api/Task.h" +# include "api/Scheduler.h" +#include "api/DummyScheduler.h" +#include "api/Factory.h" +#include "api/Database.h" + +} + +#endif diff --git a/src/ripple_core/nodestore/api/Backend.h b/src/ripple_core/nodestore/api/Backend.h new file mode 100644 index 0000000000..d4292e7ed9 --- /dev/null +++ b/src/ripple_core/nodestore/api/Backend.h @@ -0,0 +1,88 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_BACKEND_H_INCLUDED +#define RIPPLE_NODESTORE_BACKEND_H_INCLUDED + +namespace NodeStore +{ + +/** A backend used for the store. + + The NodeStore uses a swappable backend so that other database systems + can be tried. Different databases may offer various features such + as improved performance, fault tolerant or distributed storage, or + all in-memory operation. + + A given instance of a backend is fixed to a particular key size. +*/ +class Backend +{ +public: + /** Destroy the backend. + + All open files are closed and flushed. If there are batched writes + or other tasks scheduled, they will be completed before this call + returns. + */ + virtual ~Backend () { } + + /** Get the human-readable name of this backend. + + This is used for diagnostic output. + */ + virtual std::string getName() = 0; + + /** Fetch a single object. + + If the object is not found or an error is encountered, the + result will indicate the condition. + + @note This will be called concurrently. + + @param key A pointer to the key data. + @param pObject [out] The created object if successful. + + @return The result of the operation. + */ + virtual Status fetch (void const* key, NodeObject::Ptr* pObject) = 0; + + /** Store a single object. + + Depending on the implementation this may happen immediately + or deferred using a scheduled task. + + @note This will be called concurrently. + + @param object The object to store. + */ + virtual void store (NodeObject::Ptr const& object) = 0; + + /** Store a group of objects. + + @note This function will not be called concurrently with + itself or @ref store. + */ + virtual void storeBatch (Batch const& batch) = 0; + + /** Visit every object in the database + + This is usually called during import. + + @note This routine will not be called concurrently with itself + or other methods. + + @see import, VisitCallback + */ + virtual void visitAll (VisitCallback& callback) = 0; + + /** Estimate the number of write operations pending. */ + virtual int getWriteLoad () = 0; +}; + +} + +#endif diff --git a/src/ripple_core/nodestore/api/Database.h b/src/ripple_core/nodestore/api/Database.h new file mode 100644 index 0000000000..21bec95d93 --- /dev/null +++ b/src/ripple_core/nodestore/api/Database.h @@ -0,0 +1,139 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_DATABASE_H_INCLUDED +#define RIPPLE_NODESTORE_DATABASE_H_INCLUDED + +namespace NodeStore +{ + +/** Persistency layer for NodeObject + + A Node is a ledger object which is uniquely identified by a key, which is + the 256-bit hash of the body of the node. The payload is a variable length + block of serialized data. + + All ledger data is stored as node objects and as such, needs to be persisted + between launches. Furthermore, since the set of node objects will in + general be larger than the amount of available memory, purged node objects + which are later accessed must be retrieved from the node store. + + @see NodeObject +*/ +class Database +{ +public: + /** Construct a node store database. + + The parameters are key value pairs passed to the backend. The + 'type' key must exist, it defines the choice of backend. Most + backends also require a 'path' field. + + Some choices for 'type' are: + HyperLevelDB, LevelDBFactory, SQLite, KeyvaDB, MDB + + If the fastBackendParameter is omitted or empty, no ephemeral database + is used. If the scheduler parameter is omited or unspecified, a + synchronous scheduler is used which performs all tasks immediately on + the caller's thread. + + @note If the database cannot be opened or created, an exception is thrown. + + @param name A diagnostic label for the database. + @param scheduler The scheduler to use for performing asynchronous tasks. + @param backendParameters The parameter string for the persistent backend. + @param fastBackendParameters [optional] The parameter string for the ephemeral backend. + + @return The opened database. + */ + static Database* New (char const* name, + Scheduler& scheduler, + Parameters const& backendParameters, + Parameters fastBackendParameters = Parameters ()); + + /** Destroy the node store. + All pending operations are completed, pending writes flushed, + and files closed before this returns. + */ + virtual ~Database () { } + + /** Retrieve the name associated with this backend. + This is used for diagnostics and may not reflect the actual path + or paths used by the underlying backend. + */ + virtual String getName () const = 0; + + /** Add the specified backend factory to the list of available factories. + The names of available factories are compared against the "type" + value in the parameter list on construction. Ownership of the object + is transferred. The object must be allocated using new. + @param factory The factory to add. + */ + static void addFactory (Factory* factory); + + /** Fetch an object. + If the object is known to be not in the database, isn't found in the + database during the fetch, or failed to load correctly during the fetch, + `nullptr` is returned. + + @note This can be called concurrently. + @param hash The key of the object to retrieve. + @return The object, or nullptr if it couldn't be retrieved. + */ + virtual NodeObject::pointer fetch (uint256 const& hash) = 0; + + /** Store the object. + + The caller's Blob parameter is overwritten. + + @param type The type of object. + @param ledgerIndex The ledger in which the object appears. + @param data The payload of the object. The caller's + variable is overwritten. + @param hash The 256-bit hash of the payload data. + + @return `true` if the object was stored? + */ + virtual void store (NodeObjectType type, + uint32 ledgerIndex, + Blob& data, + uint256 const& hash) = 0; + + /** Visit every object in the database + This is usually called during import. + + @note This routine will not be called concurrently with itself + or other methods. + @see import + */ + virtual void visitAll (VisitCallback& callback) = 0; + + /** Import objects from another database. */ + virtual void import (Database& sourceDatabase) = 0; + + /** Retrieve the estimated number of pending write operations. + This is used for diagnostics. + */ + virtual int getWriteLoad () = 0; + + // VFALCO TODO Document this. + virtual float getCacheHitRate () = 0; + + // VFALCO TODO Document this. + // TODO Document the parameter meanings. + virtual void tune (int size, int age) = 0; + + // VFALCO TODO Document this. + virtual void sweep () = 0; + + /** Add the known Backend factories to the singleton. + */ + static void addAvailableBackends (); +}; + +} + +#endif diff --git a/src/ripple_core/nodestore/api/DummyScheduler.h b/src/ripple_core/nodestore/api/DummyScheduler.h new file mode 100644 index 0000000000..8d327126c7 --- /dev/null +++ b/src/ripple_core/nodestore/api/DummyScheduler.h @@ -0,0 +1,28 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_DUMMYSCHEDULER_H_INCLUDED +#define RIPPLE_NODESTORE_DUMMYSCHEDULER_H_INCLUDED + +namespace NodeStore +{ + +// Simple Scheduler that just peforms the tasks synchronously. +class DummyScheduler : public Scheduler +{ +public: + DummyScheduler (); + + ~DummyScheduler (); + + void scheduleTask (Task& task); + + void scheduledTasksStopped (); +}; + +} + +#endif diff --git a/src/ripple_core/nodestore/api/Factory.h b/src/ripple_core/nodestore/api/Factory.h new file mode 100644 index 0000000000..c6dbda4ac0 --- /dev/null +++ b/src/ripple_core/nodestore/api/Factory.h @@ -0,0 +1,37 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_FACTORY_H_INCLUDED +#define RIPPLE_NODESTORE_FACTORY_H_INCLUDED + +namespace NodeStore +{ + +/** Factory to produce backends. */ +class Factory +{ +public: + virtual ~Factory () { } + + /** Retrieve the name of this factory. */ + virtual String getName () const = 0; + + /** Create an instance of this factory's backend. + + @param keyBytes The fixed number of bytes per key. + @param keyValues A set of key/value configuration pairs. + @param scheduler The scheduler to use for running tasks. + + @return A pointer to the Backend object. + */ + virtual Backend* createInstance (size_t keyBytes, + Parameters const& parameters, + Scheduler& scheduler) = 0; +}; + +} + +#endif diff --git a/src/ripple_core/node/NodeObject.h b/src/ripple_core/nodestore/api/NodeObject.h similarity index 95% rename from src/ripple_core/node/NodeObject.h rename to src/ripple_core/nodestore/api/NodeObject.h index 8d1b089b14..754ce7c579 100644 --- a/src/ripple_core/node/NodeObject.h +++ b/src/ripple_core/nodestore/api/NodeObject.h @@ -4,11 +4,14 @@ */ //============================================================================== -#ifndef RIPPLE_CORE_NODE_NODEOBJECT_H_INCLUDED -#define RIPPLE_CORE_NODE_NODEOBJECT_H_INCLUDED +#ifndef RIPPLE_NODESTORE_NODEOBJECT_H_INCLUDED +#define RIPPLE_NODESTORE_NODEOBJECT_H_INCLUDED -/** The types of node objects. -*/ +// VFALCO NOTE Intentionally not in the NodeStore namespace + + + +/** The types of node objects. */ enum NodeObjectType { hotUNKNOWN = 0, diff --git a/src/ripple_core/nodestore/api/Scheduler.h b/src/ripple_core/nodestore/api/Scheduler.h new file mode 100644 index 0000000000..8d3a16624c --- /dev/null +++ b/src/ripple_core/nodestore/api/Scheduler.h @@ -0,0 +1,36 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_SCHEDULER_H_INCLUDED +#define RIPPLE_NODESTORE_SCHEDULER_H_INCLUDED + +namespace NodeStore +{ + +/** Scheduling for asynchronous backend activity + + For improved performance, a backend has the option of performing writes + in batches. These writes can be scheduled using the provided scheduler + object. + + @see BatchWriter +*/ +class Scheduler +{ +public: + virtual ~Scheduler() { } + + /** Schedules a task. + Depending on the implementation, the task may be invoked either on + the current thread of execution, or an unspecified implementation-defined + foreign thread. + */ + virtual void scheduleTask (Task& task) = 0; +}; + +} + +#endif diff --git a/src/ripple_core/nodestore/api/Task.h b/src/ripple_core/nodestore/api/Task.h new file mode 100644 index 0000000000..4534bfb872 --- /dev/null +++ b/src/ripple_core/nodestore/api/Task.h @@ -0,0 +1,26 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_TASK_H_INCLUDED +#define RIPPLE_NODESTORE_TASK_H_INCLUDED + +namespace NodeStore +{ + +/** Derived classes perform scheduled tasks. */ +struct Task +{ + virtual ~Task () { } + + /** Performs the task. + The call may take place on a foreign thread. + */ + virtual void performScheduledTask () = 0; +}; + +} + +#endif diff --git a/src/ripple_core/nodestore/api/Types.h b/src/ripple_core/nodestore/api/Types.h new file mode 100644 index 0000000000..94e7ceaf07 --- /dev/null +++ b/src/ripple_core/nodestore/api/Types.h @@ -0,0 +1,38 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_TYPES_H_INCLUDED +#define RIPPLE_NODESTORE_TYPES_H_INCLUDED + +namespace NodeStore +{ + +enum +{ + // This is only used to pre-allocate the array for + // batch objects and does not affect the amount written. + // + batchWritePreallocationSize = 128 +}; + +/** Return codes from Backend operations. */ +enum Status +{ + ok, + notFound, + dataCorrupt, + unknown +}; + +/** A batch of NodeObjects to write at once. */ +typedef std::vector Batch; + +/** A list of key/value parameter pairs passed to the backend. */ +typedef StringPairArray Parameters; + +} + +#endif diff --git a/src/ripple_core/nodestore/api/VisitCallback.h b/src/ripple_core/nodestore/api/VisitCallback.h new file mode 100644 index 0000000000..3e2ed6d5b1 --- /dev/null +++ b/src/ripple_core/nodestore/api/VisitCallback.h @@ -0,0 +1,24 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_VISITCALLBACK_H_INCLUDED +#define RIPPLE_NODESTORE_VISITCALLBACK_H_INCLUDED + +namespace NodeStore +{ + +/** Callback for iterating through objects. + + @see visitAll +*/ +struct VisitCallback +{ + virtual void visitObject (NodeObject::Ptr const& object) = 0; +}; + +} + +#endif diff --git a/src/ripple_core/node/HyperLevelDBBackendFactory.cpp b/src/ripple_core/nodestore/backend/HyperDBFactory.cpp similarity index 78% rename from src/ripple_core/node/HyperLevelDBBackendFactory.cpp rename to src/ripple_core/nodestore/backend/HyperDBFactory.cpp index 1ef07cbc50..71a7ee10fd 100644 --- a/src/ripple_core/node/HyperLevelDBBackendFactory.cpp +++ b/src/ripple_core/nodestore/backend/HyperDBFactory.cpp @@ -6,24 +6,27 @@ #if RIPPLE_HYPERLEVELDB_AVAILABLE -class HyperLevelDBBackendFactory::Backend - : public NodeStore::Backend - , public NodeStore::BatchWriter::Callback - , LeakChecked +namespace NodeStore +{ + +class HyperDBFactory::BackendImp + : public Backend + , public BatchWriter::Callback + , public LeakChecked { public: typedef RecycledObjectPool StringPool; - Backend (size_t keyBytes, - StringPairArray const& keyValues, - NodeStore::Scheduler& scheduler) + BackendImp (size_t keyBytes, + Parameters const& keyValues, + Scheduler& scheduler) : m_keyBytes (keyBytes) , m_scheduler (scheduler) , m_batch (*this, scheduler) , m_name (keyValues ["path"].toStdString ()) { if (m_name.empty ()) - Throw (std::runtime_error ("Missing path in LevelDB backend")); + Throw (std::runtime_error ("Missing path in LevelDBFactory backend")); hyperleveldb::Options options; options.create_if_missing = true; @@ -60,7 +63,7 @@ public: m_db = db; } - ~Backend () + ~BackendImp () { } @@ -91,7 +94,7 @@ public: if (getStatus.ok ()) { - NodeStore::DecodedBlob decoded (key, string.data (), string.size ()); + DecodedBlob decoded (key, string.data (), string.size ()); if (decoded.wasOk ()) { @@ -129,12 +132,12 @@ public: m_batch.store (object); } - void storeBatch (NodeStore::Batch const& batch) + void storeBatch (Batch const& batch) { hyperleveldb::WriteBatch wb; { - NodeStore::EncodedBlob::Pool::ScopedItem item (m_blobPool); + EncodedBlob::Pool::ScopedItem item (m_blobPool); BOOST_FOREACH (NodeObject::ref object, batch) { @@ -163,7 +166,7 @@ public: { if (it->key ().size () == m_keyBytes) { - NodeStore::DecodedBlob decoded (it->key ().data (), + DecodedBlob decoded (it->key ().data (), it->value ().data (), it->value ().size ()); @@ -193,61 +196,51 @@ public: return m_batch.getWriteLoad (); } - void stopAsync () - { - m_batch.stopAsync(); - } - //-------------------------------------------------------------------------- - void writeBatch (NodeStore::Batch const& batch) + void writeBatch (Batch const& batch) { storeBatch (batch); } - void writeStopped () - { - m_scheduler.scheduledTasksStopped (); - } - private: size_t const m_keyBytes; - NodeStore::Scheduler& m_scheduler; - NodeStore::BatchWriter m_batch; + Scheduler& m_scheduler; + BatchWriter m_batch; StringPool m_stringPool; - NodeStore::EncodedBlob::Pool m_blobPool; + EncodedBlob::Pool m_blobPool; std::string m_name; ScopedPointer m_db; }; //------------------------------------------------------------------------------ -HyperLevelDBBackendFactory::HyperLevelDBBackendFactory () +HyperDBFactory::HyperDBFactory () { } -HyperLevelDBBackendFactory::~HyperLevelDBBackendFactory () +HyperDBFactory::~HyperDBFactory () { } -HyperLevelDBBackendFactory* HyperLevelDBBackendFactory::getInstance () +HyperDBFactory* HyperDBFactory::getInstance () { - return new HyperLevelDBBackendFactory; + return new HyperDBFactory; } -String HyperLevelDBBackendFactory::getName () const +String HyperDBFactory::getName () const { return "HyperLevelDB"; } -NodeStore::Backend* HyperLevelDBBackendFactory::createInstance ( +Backend* HyperDBFactory::createInstance ( size_t keyBytes, - StringPairArray const& keyValues, - NodeStore::Scheduler& scheduler) + Parameters const& keyValues, + Scheduler& scheduler) { - return new HyperLevelDBBackendFactory::Backend (keyBytes, keyValues, scheduler); + return new HyperDBFactory::BackendImp (keyBytes, keyValues, scheduler); } -//------------------------------------------------------------------------------ +} #endif diff --git a/src/ripple_core/node/HyperLevelDBBackendFactory.h b/src/ripple_core/nodestore/backend/HyperDBFactory.h similarity index 54% rename from src/ripple_core/node/HyperLevelDBBackendFactory.h rename to src/ripple_core/nodestore/backend/HyperDBFactory.h index 1d8338f3ee..42f4015820 100644 --- a/src/ripple_core/node/HyperLevelDBBackendFactory.h +++ b/src/ripple_core/nodestore/backend/HyperDBFactory.h @@ -4,33 +4,38 @@ */ //============================================================================== -#ifndef RIPPLE_CORE_NODE_HYPERLEVELDBBACKENDFACTORY_H_INCLUDED -#define RIPPLE_CORE_NODE_HYPERLEVELDBBACKENDFACTORY_H_INCLUDED +#ifndef RIPPLE_NODESTORE_HYPERDBFACTORY_H_INCLUDED +#define RIPPLE_NODESTORE_HYPERDBFACTORY_H_INCLUDED #if RIPPLE_HYPERLEVELDB_AVAILABLE +namespace NodeStore +{ + /** Factory to produce HyperLevelDB backends for the NodeStore. - @see NodeStore + @see Database */ -class HyperLevelDBBackendFactory : public NodeStore::BackendFactory +class HyperDBFactory : public NodeStore::Factory { private: - class Backend; - - HyperLevelDBBackendFactory (); - ~HyperLevelDBBackendFactory (); + HyperDBFactory (); + ~HyperDBFactory (); public: - static HyperLevelDBBackendFactory* getInstance (); + class BackendImp; + + static HyperDBFactory* getInstance (); String getName () const; NodeStore::Backend* createInstance (size_t keyBytes, - StringPairArray const& keyValues, + Parameters const& keyValues, NodeStore::Scheduler& scheduler); }; +} + #endif #endif diff --git a/src/ripple_core/node/KeyvaDBBackendFactory.cpp b/src/ripple_core/nodestore/backend/KeyvaDBFactory.cpp similarity index 74% rename from src/ripple_core/node/KeyvaDBBackendFactory.cpp rename to src/ripple_core/nodestore/backend/KeyvaDBFactory.cpp index 8ec405d350..8cf577e9a6 100644 --- a/src/ripple_core/node/KeyvaDBBackendFactory.cpp +++ b/src/ripple_core/nodestore/backend/KeyvaDBFactory.cpp @@ -4,16 +4,19 @@ */ //============================================================================== -class KeyvaDBBackendFactory::Backend : public NodeStore::Backend +namespace NodeStore +{ + +class KeyvaDBFactory::BackendImp : public Backend { private: typedef RecycledObjectPool MemoryPool; - typedef RecycledObjectPool EncodedBlobPool; + typedef RecycledObjectPool EncodedBlobPool; public: - Backend (size_t keyBytes, - StringPairArray const& keyValues, - NodeStore::Scheduler& scheduler) + BackendImp (size_t keyBytes, + Parameters const& keyValues, + Scheduler& scheduler) : m_keyBytes (keyBytes) , m_scheduler (scheduler) , m_path (keyValues ["path"]) @@ -25,7 +28,7 @@ public: { } - ~Backend () + ~BackendImp () { } @@ -83,7 +86,7 @@ public: if (found) { - NodeStore::DecodedBlob decoded (key, cb.getData (), cb.getSize ()); + DecodedBlob decoded (key, cb.getData (), cb.getSize ()); if (decoded.wasOk ()) { @@ -107,14 +110,14 @@ public: void store (NodeObject::ref object) { EncodedBlobPool::ScopedItem item (m_blobPool); - NodeStore::EncodedBlob& encoded (item.getObject ()); + EncodedBlob& encoded (item.getObject ()); encoded.prepare (object); m_db->put (encoded.getKey (), encoded.getData (), encoded.getSize ()); } - void storeBatch (NodeStore::Batch const& batch) + void storeBatch (Batch const& batch) { for (std::size_t i = 0; i < batch.size (); ++i) store (batch [i]); @@ -134,16 +137,11 @@ public: return 0; } - void stopAsync () - { - m_scheduler.scheduledTasksStopped (); - } - //-------------------------------------------------------------------------- private: size_t const m_keyBytes; - NodeStore::Scheduler& m_scheduler; + Scheduler& m_scheduler; String m_path; ScopedPointer m_db; MemoryPool m_memoryPool; @@ -152,31 +150,30 @@ private: //------------------------------------------------------------------------------ -KeyvaDBBackendFactory::KeyvaDBBackendFactory () +KeyvaDBFactory::KeyvaDBFactory () { } -KeyvaDBBackendFactory::~KeyvaDBBackendFactory () +KeyvaDBFactory::~KeyvaDBFactory () { } -KeyvaDBBackendFactory* KeyvaDBBackendFactory::getInstance () +KeyvaDBFactory* KeyvaDBFactory::getInstance () { - return new KeyvaDBBackendFactory; + return new KeyvaDBFactory; } -String KeyvaDBBackendFactory::getName () const +String KeyvaDBFactory::getName () const { return "KeyvaDB"; } -NodeStore::Backend* KeyvaDBBackendFactory::createInstance ( +Backend* KeyvaDBFactory::createInstance ( size_t keyBytes, - StringPairArray const& keyValues, - NodeStore::Scheduler& scheduler) + Parameters const& keyValues, + Scheduler& scheduler) { - return new KeyvaDBBackendFactory::Backend (keyBytes, keyValues, scheduler); + return new KeyvaDBFactory::BackendImp (keyBytes, keyValues, scheduler); } -//------------------------------------------------------------------------------ - +} diff --git a/src/ripple_core/nodestore/backend/KeyvaDBFactory.h b/src/ripple_core/nodestore/backend/KeyvaDBFactory.h new file mode 100644 index 0000000000..4361ec3bd9 --- /dev/null +++ b/src/ripple_core/nodestore/backend/KeyvaDBFactory.h @@ -0,0 +1,37 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_KEYVADBFACTORY_H_INCLUDED +#define RIPPLE_NODESTORE_KEYVADBFACTORY_H_INCLUDED + +namespace NodeStore +{ + +/** Factory to produce KeyvaDB backends for the NodeStore. + + @see Database +*/ +class KeyvaDBFactory : public Factory +{ +private: + KeyvaDBFactory (); + ~KeyvaDBFactory (); + +public: + class BackendImp; + + static KeyvaDBFactory* getInstance (); + + String getName () const; + + Backend* createInstance (size_t keyBytes, + Parameters const& keyValues, + Scheduler& scheduler); +}; + +} + +#endif diff --git a/src/ripple_core/node/LevelDBBackendFactory.cpp b/src/ripple_core/nodestore/backend/LevelDBFactory.cpp similarity index 79% rename from src/ripple_core/node/LevelDBBackendFactory.cpp rename to src/ripple_core/nodestore/backend/LevelDBFactory.cpp index 1bf4818c15..2fd20c9889 100644 --- a/src/ripple_core/node/LevelDBBackendFactory.cpp +++ b/src/ripple_core/nodestore/backend/LevelDBFactory.cpp @@ -4,26 +4,29 @@ */ //============================================================================== -class LevelDBBackendFactory::Backend - : public NodeStore::Backend - , public NodeStore::BatchWriter::Callback - , LeakChecked +namespace NodeStore +{ + +class LevelDBFactory::BackendImp + : public Backend + , public BatchWriter::Callback + , public LeakChecked { public: typedef RecycledObjectPool StringPool; //-------------------------------------------------------------------------- - Backend (int keyBytes, - StringPairArray const& keyValues, - NodeStore::Scheduler& scheduler) + BackendImp (int keyBytes, + Parameters const& keyValues, + Scheduler& scheduler) : m_keyBytes (keyBytes) , m_scheduler (scheduler) , m_batch (*this, scheduler) , m_name (keyValues ["path"].toStdString ()) { if (m_name.empty()) - Throw (std::runtime_error ("Missing path in LevelDB backend")); + Throw (std::runtime_error ("Missing path in LevelDBFactory backend")); leveldb::Options options; options.create_if_missing = true; @@ -60,7 +63,7 @@ public: m_db = db; } - ~Backend () + ~BackendImp () { } @@ -91,7 +94,7 @@ public: if (getStatus.ok ()) { - NodeStore::DecodedBlob decoded (key, string.data (), string.size ()); + DecodedBlob decoded (key, string.data (), string.size ()); if (decoded.wasOk ()) { @@ -129,12 +132,12 @@ public: m_batch.store (object); } - void storeBatch (NodeStore::Batch const& batch) + void storeBatch (Batch const& batch) { leveldb::WriteBatch wb; { - NodeStore::EncodedBlob::Pool::ScopedItem item (m_blobPool); + EncodedBlob::Pool::ScopedItem item (m_blobPool); BOOST_FOREACH (NodeObject::ref object, batch) { @@ -163,7 +166,7 @@ public: { if (it->key ().size () == m_keyBytes) { - NodeStore::DecodedBlob decoded (it->key ().data (), + DecodedBlob decoded (it->key ().data (), it->value ().data (), it->value ().size ()); @@ -193,36 +196,26 @@ public: return m_batch.getWriteLoad (); } - void stopAsync () - { - m_batch.stopAsync (); - } - //-------------------------------------------------------------------------- - void writeBatch (NodeStore::Batch const& batch) + void writeBatch (Batch const& batch) { storeBatch (batch); } - void writeStopped () - { - m_scheduler.scheduledTasksStopped (); - } - private: size_t const m_keyBytes; - NodeStore::Scheduler& m_scheduler; - NodeStore::BatchWriter m_batch; + Scheduler& m_scheduler; + BatchWriter m_batch; StringPool m_stringPool; - NodeStore::EncodedBlob::Pool m_blobPool; + EncodedBlob::Pool m_blobPool; std::string m_name; ScopedPointer m_db; }; //------------------------------------------------------------------------------ -LevelDBBackendFactory::LevelDBBackendFactory () +LevelDBFactory::LevelDBFactory () : m_lruCache (nullptr) { leveldb::Options options; @@ -233,29 +226,28 @@ LevelDBBackendFactory::LevelDBBackendFactory () m_lruCache = options.block_cache; } -LevelDBBackendFactory::~LevelDBBackendFactory () +LevelDBFactory::~LevelDBFactory () { leveldb::Cache* cache (reinterpret_cast (m_lruCache)); delete cache; } -LevelDBBackendFactory* LevelDBBackendFactory::getInstance () +LevelDBFactory* LevelDBFactory::getInstance () { - return new LevelDBBackendFactory; + return new LevelDBFactory; } -String LevelDBBackendFactory::getName () const +String LevelDBFactory::getName () const { return "LevelDB"; } -NodeStore::Backend* LevelDBBackendFactory::createInstance ( +Backend* LevelDBFactory::createInstance ( size_t keyBytes, - StringPairArray const& keyValues, - NodeStore::Scheduler& scheduler) + Parameters const& keyValues, + Scheduler& scheduler) { - return new LevelDBBackendFactory::Backend (keyBytes, keyValues, scheduler); + return new LevelDBFactory::BackendImp (keyBytes, keyValues, scheduler); } -//------------------------------------------------------------------------------ - +} diff --git a/src/ripple_core/nodestore/backend/LevelDBFactory.h b/src/ripple_core/nodestore/backend/LevelDBFactory.h new file mode 100644 index 0000000000..6bb08acf06 --- /dev/null +++ b/src/ripple_core/nodestore/backend/LevelDBFactory.h @@ -0,0 +1,40 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_LEVELDBFACTORY_H_INCLUDED +#define RIPPLE_NODESTORE_LEVELDBFACTORY_H_INCLUDED + +namespace NodeStore +{ + +/** Factory to produce LevelDBFactory backends for the NodeStore. + + @see Database +*/ +class LevelDBFactory : public Factory +{ +private: + LevelDBFactory (); + ~LevelDBFactory (); + +public: + class BackendImp; + + static LevelDBFactory* getInstance (); + + String getName () const; + + Backend* createInstance (size_t keyBytes, + Parameters const& keyValues, + Scheduler& scheduler); + +private: + void* m_lruCache; +}; + +} + +#endif diff --git a/src/ripple_core/node/MdbBackendFactory.cpp b/src/ripple_core/nodestore/backend/MdbFactory.cpp similarity index 84% rename from src/ripple_core/node/MdbBackendFactory.cpp rename to src/ripple_core/nodestore/backend/MdbFactory.cpp index fddabad097..79c96c71e3 100644 --- a/src/ripple_core/node/MdbBackendFactory.cpp +++ b/src/ripple_core/nodestore/backend/MdbFactory.cpp @@ -6,19 +6,18 @@ #if RIPPLE_MDB_AVAILABLE -class MdbBackendFactory::Backend - : public NodeStore::Backend - , public NodeStore::BatchWriter::Callback - , LeakChecked +namespace NodeStore +{ + +class MdbFactory::BackendImp + : public Backend + , public BatchWriter::Callback + , public LeakChecked { public: - typedef NodeStore::Batch Batch; - typedef NodeStore::EncodedBlob EncodedBlob; - typedef NodeStore::DecodedBlob DecodedBlob; - - explicit Backend (size_t keyBytes, - StringPairArray const& keyValues, - NodeStore::Scheduler& scheduler) + explicit BackendImp (size_t keyBytes, + Parameters const& keyValues, + Scheduler& scheduler) : m_keyBytes (keyBytes) , m_scheduler (scheduler) , m_batch (*this, scheduler) @@ -78,7 +77,7 @@ public: } } - ~Backend () + ~BackendImp () { if (m_env != nullptr) { @@ -229,11 +228,6 @@ public: return m_batch.getWriteLoad (); } - void stopAsync () - { - m_batch.stopAsync(); - } - //-------------------------------------------------------------------------- void writeBatch (Batch const& batch) @@ -241,16 +235,11 @@ public: storeBatch (batch); } - void writeStopped () - { - m_scheduler.scheduledTasksStopped (); - } - private: size_t const m_keyBytes; - NodeStore::Scheduler& m_scheduler; - NodeStore::BatchWriter m_batch; - NodeStore::EncodedBlob::Pool m_blobPool; + Scheduler& m_scheduler; + BatchWriter m_batch; + EncodedBlob::Pool m_blobPool; std::string m_basePath; MDB_env* m_env; MDB_dbi m_dbi; @@ -258,30 +247,32 @@ private: //------------------------------------------------------------------------------ -MdbBackendFactory::MdbBackendFactory () +MdbFactory::MdbFactory () { } -MdbBackendFactory::~MdbBackendFactory () +MdbFactory::~MdbFactory () { } -MdbBackendFactory* MdbBackendFactory::getInstance () +MdbFactory* MdbFactory::getInstance () { - return new MdbBackendFactory; + return new MdbFactory; } -String MdbBackendFactory::getName () const +String MdbFactory::getName () const { return "mdb"; } -NodeStore::Backend* MdbBackendFactory::createInstance ( +Backend* MdbFactory::createInstance ( size_t keyBytes, - StringPairArray const& keyValues, - NodeStore::Scheduler& scheduler) + Parameters const& keyValues, + Scheduler& scheduler) { - return new MdbBackendFactory::Backend (keyBytes, keyValues, scheduler); + return new MdbFactory::BackendImp (keyBytes, keyValues, scheduler); +} + } #endif diff --git a/src/ripple_core/nodestore/backend/MdbFactory.h b/src/ripple_core/nodestore/backend/MdbFactory.h new file mode 100644 index 0000000000..b78385448c --- /dev/null +++ b/src/ripple_core/nodestore/backend/MdbFactory.h @@ -0,0 +1,43 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_MDBFACTORY_H_INCLUDED +#define RIPPLE_NODESTORE_MDBFACTORY_H_INCLUDED + +#if RIPPLE_MDB_AVAILABLE + +namespace NodeStore +{ + +/** Factory to produce a backend using MDB. + + @note MDB is not currently available for Win32 + + @see Database +*/ +class MdbFactory : public Factory +{ +private: + MdbFactory (); + ~MdbFactory (); + +public: + class BackendImp; + + static MdbFactory* getInstance (); + + String getName () const; + + Backend* createInstance (size_t keyBytes, + Parameters const& keyValues, + Scheduler& scheduler); +}; + +} + +#endif + +#endif diff --git a/src/ripple_core/node/MemoryBackendFactory.cpp b/src/ripple_core/nodestore/backend/MemoryFactory.cpp similarity index 66% rename from src/ripple_core/node/MemoryBackendFactory.cpp rename to src/ripple_core/nodestore/backend/MemoryFactory.cpp index 5e7fe929ef..ec59339f82 100644 --- a/src/ripple_core/node/MemoryBackendFactory.cpp +++ b/src/ripple_core/nodestore/backend/MemoryFactory.cpp @@ -4,20 +4,23 @@ */ //============================================================================== -class MemoryBackendFactory::Backend : public NodeStore::Backend +namespace NodeStore +{ + +class MemoryFactory::BackendImp : public Backend { private: typedef std::map Map; public: - Backend (size_t keyBytes, StringPairArray const& keyValues, - NodeStore::Scheduler& scheduler) + BackendImp (size_t keyBytes, Parameters const& keyValues, + Scheduler& scheduler) : m_keyBytes (keyBytes) , m_scheduler (scheduler) { } - ~Backend () + ~BackendImp () { } @@ -56,7 +59,7 @@ public: } } - void storeBatch (NodeStore::Batch const& batch) + void storeBatch (Batch const& batch) { for (std::size_t i = 0; i < batch.size (); ++i) store (batch [i]); @@ -73,47 +76,41 @@ public: return 0; } - void stopAsync () - { - m_scheduler.scheduledTasksStopped (); - } - //-------------------------------------------------------------------------- private: size_t const m_keyBytes; Map m_map; - NodeStore::Scheduler& m_scheduler; + Scheduler& m_scheduler; }; //------------------------------------------------------------------------------ -MemoryBackendFactory::MemoryBackendFactory () +MemoryFactory::MemoryFactory () { } -MemoryBackendFactory::~MemoryBackendFactory () +MemoryFactory::~MemoryFactory () { } -MemoryBackendFactory* MemoryBackendFactory::getInstance () +MemoryFactory* MemoryFactory::getInstance () { - return new MemoryBackendFactory; + return new MemoryFactory; } -String MemoryBackendFactory::getName () const +String MemoryFactory::getName () const { return "Memory"; } -NodeStore::Backend* MemoryBackendFactory::createInstance ( +Backend* MemoryFactory::createInstance ( size_t keyBytes, - StringPairArray const& keyValues, - NodeStore::Scheduler& scheduler) + Parameters const& keyValues, + Scheduler& scheduler) { - return new MemoryBackendFactory::Backend (keyBytes, keyValues, scheduler); + return new MemoryFactory::BackendImp (keyBytes, keyValues, scheduler); } -//------------------------------------------------------------------------------ - +} diff --git a/src/ripple_core/nodestore/backend/MemoryFactory.h b/src/ripple_core/nodestore/backend/MemoryFactory.h new file mode 100644 index 0000000000..7e0ad74b20 --- /dev/null +++ b/src/ripple_core/nodestore/backend/MemoryFactory.h @@ -0,0 +1,37 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_MEMORYFACTORY_H_INCLUDED +#define RIPPLE_NODESTORE_MEMORYFACTORY_H_INCLUDED + +namespace NodeStore +{ + +/** Factory to produce a RAM based backend for the NodeStore. + + @see Database +*/ +class MemoryFactory : public Factory +{ +private: + MemoryFactory (); + ~MemoryFactory (); + +public: + class BackendImp; + + static MemoryFactory* getInstance (); + + String getName () const; + + Backend* createInstance (size_t keyBytes, + Parameters const& keyValues, + Scheduler& scheduler); +}; + +} + +#endif diff --git a/src/ripple_core/nodestore/backend/NullFactory.cpp b/src/ripple_core/nodestore/backend/NullFactory.cpp new file mode 100644 index 0000000000..fd6fd63a46 --- /dev/null +++ b/src/ripple_core/nodestore/backend/NullFactory.cpp @@ -0,0 +1,81 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +namespace NodeStore +{ + +class NullFactory::BackendImp : public Backend +{ +public: + explicit BackendImp (Scheduler& scheduler) + : m_scheduler (scheduler) + { + } + + ~BackendImp () + { + } + + std::string getName() + { + return std::string (); + } + + Status fetch (void const*, NodeObject::Ptr*) + { + return notFound; + } + + void store (NodeObject::ref object) + { + } + + void storeBatch (Batch const& batch) + { + } + + void visitAll (VisitCallback& callback) + { + } + + int getWriteLoad () + { + return 0; + } + +private: + Scheduler& m_scheduler; +}; + +//------------------------------------------------------------------------------ + +NullFactory::NullFactory () +{ +} + +NullFactory::~NullFactory () +{ +} + +NullFactory* NullFactory::getInstance () +{ + return new NullFactory; +} + +String NullFactory::getName () const +{ + return "none"; +} + +Backend* NullFactory::createInstance ( + size_t, + Parameters const&, + Scheduler& scheduler) +{ + return new NullFactory::BackendImp (scheduler); +} + +} diff --git a/src/ripple_core/nodestore/backend/NullFactory.h b/src/ripple_core/nodestore/backend/NullFactory.h new file mode 100644 index 0000000000..95e4431be8 --- /dev/null +++ b/src/ripple_core/nodestore/backend/NullFactory.h @@ -0,0 +1,40 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_NULLFACTORY_H_INCLUDED +#define RIPPLE_NODESTORE_NULLFACTORY_H_INCLUDED + +namespace NodeStore +{ + +/** Factory to produce a null backend. + + This is for standalone / testing mode. + + @see Database +*/ +class NullFactory : public Factory +{ +private: + NullFactory (); + ~NullFactory (); + +public: + class BackendImp; + + static NullFactory* getInstance (); + + String getName () const; + + Backend* createInstance (size_t keyBytes, + Parameters const& keyValues, + Scheduler& scheduler); +}; + +} + +#endif + diff --git a/src/ripple_core/node/SophiaBackendFactory.cpp b/src/ripple_core/nodestore/backend/SophiaFactory.cpp similarity index 72% rename from src/ripple_core/node/SophiaBackendFactory.cpp rename to src/ripple_core/nodestore/backend/SophiaFactory.cpp index f2056ac735..ffd8d4af1e 100644 --- a/src/ripple_core/node/SophiaBackendFactory.cpp +++ b/src/ripple_core/nodestore/backend/SophiaFactory.cpp @@ -6,22 +6,22 @@ #if RIPPLE_SOPHIA_AVAILABLE -class SophiaBackendFactory::Backend - : public NodeStore::Backend - , public NodeStore::BatchWriter::Callback - , public LeakChecked +namespace NodeStore +{ + +class SophiaFactory::BackendImp + : public Backend + , public BatchWriter::Callback + , public LeakChecked { public: typedef RecycledObjectPool StringPool; - typedef NodeStore::Batch Batch; - typedef NodeStore::EncodedBlob EncodedBlob; - typedef NodeStore::DecodedBlob DecodedBlob; //-------------------------------------------------------------------------- - Backend (int keyBytes, - StringPairArray const& keyValues, - NodeStore::Scheduler& scheduler) + BackendImp (int keyBytes, + Parameters const& keyValues, + Scheduler& scheduler) : m_keyBytes (keyBytes) , m_scheduler (scheduler) , m_batch (*this, scheduler) @@ -41,7 +41,7 @@ public: } } - ~Backend () + ~BackendImp () { if (m_db != nullptr) sp_destroy (m_db); @@ -108,7 +108,7 @@ public: { EncodedBlob::Pool::ScopedItem item (m_blobPool); - for (NodeStore::Batch::const_iterator iter (batch.begin()); + for (Batch::const_iterator iter (batch.begin()); iter != batch.end(); ++iter) { EncodedBlob& encoded (item.getObject ()); @@ -136,29 +136,19 @@ public: return m_batch.getWriteLoad (); } - void stopAsync () - { - m_batch.stopAsync (); - } - //-------------------------------------------------------------------------- - void writeBatch (NodeStore::Batch const& batch) + void writeBatch (Batch const& batch) { storeBatch (batch); } - void writeStopped () - { - m_scheduler.scheduledTasksStopped (); - } - private: size_t const m_keyBytes; - NodeStore::Scheduler& m_scheduler; - NodeStore::BatchWriter m_batch; + Scheduler& m_scheduler; + BatchWriter m_batch; StringPool m_stringPool; - NodeStore::EncodedBlob::Pool m_blobPool; + EncodedBlob::Pool m_blobPool; std::string m_name; void* m_env; void* m_db; @@ -166,7 +156,7 @@ private: //------------------------------------------------------------------------------ -SophiaBackendFactory::SophiaBackendFactory () +SophiaFactory::SophiaFactory () { leveldb::Options options; options.create_if_missing = true; @@ -174,28 +164,28 @@ SophiaBackendFactory::SophiaBackendFactory () siHashNodeDBCache) * 1024 * 1024); } -SophiaBackendFactory::~SophiaBackendFactory () +SophiaFactory::~SophiaFactory () { } -SophiaBackendFactory* SophiaBackendFactory::getInstance () +SophiaFactory* SophiaFactory::getInstance () { - return new SophiaBackendFactory; + return new SophiaFactory; } -String SophiaBackendFactory::getName () const +String SophiaFactory::getName () const { return "sophia"; } -NodeStore::Backend* SophiaBackendFactory::createInstance ( +Backend* SophiaFactory::createInstance ( size_t keyBytes, - StringPairArray const& keyValues, - NodeStore::Scheduler& scheduler) + Parameters const& keyValues, + Scheduler& scheduler) { - return new SophiaBackendFactory::Backend (keyBytes, keyValues, scheduler); + return new SophiaFactory::BackendImp (keyBytes, keyValues, scheduler); } -//------------------------------------------------------------------------------ +} #endif diff --git a/src/ripple_core/nodestore/backend/SophiaFactory.h b/src/ripple_core/nodestore/backend/SophiaFactory.h new file mode 100644 index 0000000000..ed6430a142 --- /dev/null +++ b/src/ripple_core/nodestore/backend/SophiaFactory.h @@ -0,0 +1,37 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_SOPHIAFACTORY_H_INCLUDED +#define RIPPLE_NODESTORE_SOPHIAFACTORY_H_INCLUDED + +namespace NodeStore +{ + +/** Factory to produce Sophia backends for the NodeStore. + + @see Database +*/ +class SophiaFactory : public Factory +{ +private: + SophiaFactory (); + ~SophiaFactory (); + +public: + class BackendImp; + + static SophiaFactory* getInstance (); + + String getName () const; + + NodeStore::Backend* createInstance (size_t keyBytes, + Parameters const& keyValues, + Scheduler& scheduler); +}; + +} + +#endif diff --git a/src/ripple_core/nodestore/impl/BatchWriter.cpp b/src/ripple_core/nodestore/impl/BatchWriter.cpp new file mode 100644 index 0000000000..09a941271b --- /dev/null +++ b/src/ripple_core/nodestore/impl/BatchWriter.cpp @@ -0,0 +1,96 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +namespace NodeStore +{ + +BatchWriter::BatchWriter (Callback& callback, Scheduler& scheduler) + : m_callback (callback) + , m_scheduler (scheduler) + , mWriteGeneration (0) + , mWriteLoad (0) + , mWritePending (false) +{ + mWriteSet.reserve (batchWritePreallocationSize); +} + +BatchWriter::~BatchWriter () +{ + waitForWriting (); +} + +void BatchWriter::store (NodeObject::ref object) +{ + LockType::scoped_lock sl (mWriteMutex); + + mWriteSet.push_back (object); + + if (! mWritePending) + { + mWritePending = true; + + m_scheduler.scheduleTask (*this); + } +} + +int BatchWriter::getWriteLoad () +{ + LockType::scoped_lock sl (mWriteMutex); + + return std::max (mWriteLoad, static_cast (mWriteSet.size ())); +} + +void BatchWriter::performScheduledTask () +{ + writeBatch (); +} + +void BatchWriter::writeBatch () +{ + int setSize = 0; + + for (;;) + { + std::vector< boost::shared_ptr > set; + + set.reserve (batchWritePreallocationSize); + + { + LockType::scoped_lock sl (mWriteMutex); + + mWriteSet.swap (set); + assert (mWriteSet.empty ()); + ++mWriteGeneration; + mWriteCondition.notify_all (); + + if (set.empty ()) + { + mWritePending = false; + mWriteLoad = 0; + + // VFALCO NOTE Fix this function to not return from the middle + return; + } + + int newSetSize = mWriteSet.size(); + mWriteLoad = std::max (setSize, newSetSize); + newSetSize = set.size (); + } + + m_callback.writeBatch (set); + } +} + +void BatchWriter::waitForWriting () +{ + LockType::scoped_lock sl (mWriteMutex); + int gen = mWriteGeneration; + + while (mWritePending && (mWriteGeneration == gen)) + mWriteCondition.wait (sl); +} + +} diff --git a/src/ripple_core/nodestore/impl/BatchWriter.h b/src/ripple_core/nodestore/impl/BatchWriter.h new file mode 100644 index 0000000000..b8bc113d87 --- /dev/null +++ b/src/ripple_core/nodestore/impl/BatchWriter.h @@ -0,0 +1,74 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_BATCHWRITER_H_INCLUDED +#define RIPPLE_NODESTORE_BATCHWRITER_H_INCLUDED + +namespace NodeStore +{ + +/** Batch-writing assist logic. + + The batch writes are performed with a scheduled task. Use of the + class it not required. A backend can implement its own write batching, + or skip write batching if doing so yields a performance benefit. + + @see Scheduler +*/ +// VFALCO NOTE I'm not entirely happy having placed this here, +// because whoever needs to use NodeStore certainly doesn't +// need to see the implementation details of BatchWriter. +// +class BatchWriter : private Task +{ +public: + /** This callback does the actual writing. */ + struct Callback + { + virtual void writeBatch (Batch const& batch) = 0; + }; + + /** Create a batch writer. */ + BatchWriter (Callback& callback, Scheduler& scheduler); + + /** Destroy a batch writer. + + Anything pending in the batch is written out before this returns. + */ + ~BatchWriter (); + + /** Store the object. + + This will add to the batch and initiate a scheduled task to + write the batch out. + */ + void store (NodeObject::Ptr const& object); + + /** Get an estimate of the amount of writing I/O pending. */ + int getWriteLoad (); + +private: + void performScheduledTask (); + void writeBatch (); + void waitForWriting (); + +private: + typedef boost::recursive_mutex LockType; + typedef boost::condition_variable_any CondvarType; + + Callback& m_callback; + Scheduler& m_scheduler; + LockType mWriteMutex; + CondvarType mWriteCondition; + int mWriteGeneration; + int mWriteLoad; + bool mWritePending; + Batch mWriteSet; +}; + +} + +#endif diff --git a/src/ripple_core/nodestore/impl/DatabaseImp.h b/src/ripple_core/nodestore/impl/DatabaseImp.h new file mode 100644 index 0000000000..f2c0154c6b --- /dev/null +++ b/src/ripple_core/nodestore/impl/DatabaseImp.h @@ -0,0 +1,352 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +namespace NodeStore +{ + +class DatabaseImp + : public Database + , LeakChecked +{ +public: + DatabaseImp (char const* name, + Scheduler& scheduler, + Parameters const& backendParameters, + Parameters const& fastBackendParameters) + : m_scheduler (scheduler) + , m_backend (createBackend (backendParameters, scheduler)) + , m_fastBackend ((fastBackendParameters.size () > 0) + ? createBackend (fastBackendParameters, scheduler) : nullptr) + , m_cache ("NodeStore", 16384, 300) + , m_negativeCache ("NoteStoreNegativeCache", 0, 120) + { + } + + ~DatabaseImp () + { + } + + String getName () const + { + return m_backend->getName (); + } + + //------------------------------------------------------------------------------ + + NodeObject::Ptr fetch (uint256 const& hash) + { + // See if the object already exists in the cache + // + NodeObject::Ptr obj = m_cache.fetch (hash); + + if (obj == nullptr) + { + // It's not in the cache, see if we can skip checking the db. + // + if (! m_negativeCache.isPresent (hash)) + { + // There's still a chance it could be in one of the databases. + + bool foundInFastBackend = false; + + // Check the fast backend database if we have one + // + if (m_fastBackend != nullptr) + { + obj = fetchInternal (m_fastBackend, hash); + + // If we found the object, avoid storing it again later. + if (obj != nullptr) + foundInFastBackend = true; + } + + // Are we still without an object? + // + if (obj == nullptr) + { + // Yes so at last we will try the main database. + // + { + // Monitor this operation's load since it is expensive. + // + // VFALCO TODO Why is this an autoptr? Why can't it just be a plain old object? + // + // VFALCO NOTE Commented this out because it breaks the unit test! + // + //LoadEvent::autoptr event (getApp().getJobQueue ().getLoadEventAP (jtHO_READ, "HOS::retrieve")); + + obj = fetchInternal (m_backend, hash); + } + + // If it's not in the main database, remember that so we + // can skip the lookup for the same object again later. + // + if (obj == nullptr) + m_negativeCache.add (hash); + } + + // Did we finally get something? + // + if (obj != nullptr) + { + // Yes it so canonicalize. This solves the problem where + // more than one thread has its own copy of the same object. + // + m_cache.canonicalize (hash, obj); + + if (! foundInFastBackend) + { + // If we have a fast back end, store it there for later. + // + if (m_fastBackend != nullptr) + m_fastBackend->store (obj); + + // Since this was a 'hard' fetch, we will log it. + // + WriteLog (lsTRACE, NodeObject) << "HOS: " << hash << " fetch: in db"; + } + } + } + else + { + // hash is known not to be in the database + } + } + else + { + // found it! + } + + return obj; + } + + NodeObject::Ptr fetchInternal (Backend* backend, uint256 const& hash) + { + NodeObject::Ptr object; + + Status const status = backend->fetch (hash.begin (), &object); + + switch (status) + { + case ok: + case notFound: + break; + + case dataCorrupt: + // VFALCO TODO Deal with encountering corrupt data! + // + WriteLog (lsFATAL, NodeObject) << "Corrupt NodeObject #" << hash; + break; + + default: + WriteLog (lsWARNING, NodeObject) << "Unknown status=" << status; + break; + } + + return object; + } + + //------------------------------------------------------------------------------ + + void store (NodeObjectType type, + uint32 index, + Blob& data, + uint256 const& hash) + { + bool const keyFoundAndObjectCached = m_cache.refreshIfPresent (hash); + + // VFALCO NOTE What happens if the key is found, but the object + // fell out of the cache? We will end up passing it + // to the backend anyway. + // + if (! keyFoundAndObjectCached) + { + #if RIPPLE_VERIFY_NODEOBJECT_KEYS + assert (hash == Serializer::getSHA512Half (data)); + #endif + + NodeObject::Ptr object = NodeObject::createObject ( + type, index, data, hash); + + if (!m_cache.canonicalize (hash, object)) + { + m_backend->store (object); + + if (m_fastBackend) + m_fastBackend->store (object); + } + + m_negativeCache.del (hash); + } + } + + //------------------------------------------------------------------------------ + + float getCacheHitRate () + { + return m_cache.getHitRate (); + } + + void tune (int size, int age) + { + m_cache.setTargetSize (size); + m_cache.setTargetAge (age); + } + + void sweep () + { + m_cache.sweep (); + m_negativeCache.sweep (); + } + + int getWriteLoad () + { + return m_backend->getWriteLoad (); + } + + //------------------------------------------------------------------------------ + + void visitAll (VisitCallback& callback) + { + m_backend->visitAll (callback); + } + + void import (Database& sourceDatabase) + { + class ImportVisitCallback : public VisitCallback + { + public: + explicit ImportVisitCallback (Backend& backend) + : m_backend (backend) + { + m_objects.reserve (batchWritePreallocationSize); + } + + ~ImportVisitCallback () + { + if (! m_objects.empty ()) + m_backend.storeBatch (m_objects); + } + + void visitObject (NodeObject::Ptr const& object) + { + if (m_objects.size () >= batchWritePreallocationSize) + { + m_backend.storeBatch (m_objects); + + m_objects.clear (); + m_objects.reserve (batchWritePreallocationSize); + } + + m_objects.push_back (object); + } + + private: + Backend& m_backend; + Batch m_objects; + }; + + //-------------------------------------------------------------------------- + + ImportVisitCallback callback (*m_backend); + + sourceDatabase.visitAll (callback); + } + + //------------------------------------------------------------------------------ + + static void missing_backend () + { + fatal_error ("Your rippled.cfg is missing a [node_db] entry, please see the rippled-example.cfg file!"); + } + + static Backend* createBackend (Parameters const& parameters, Scheduler& scheduler) + { + Backend* backend = nullptr; + + String const& type = parameters ["type"]; + + if (type.isNotEmpty ()) + { + Factory* factory (Factories::get().find (type)); + + if (factory != nullptr) + { + backend = factory->createInstance (NodeObject::keyBytes, parameters, scheduler); + } + else + { + missing_backend (); + } + } + else + { + missing_backend (); + } + + return backend; + } + + //------------------------------------------------------------------------------ + +private: + Scheduler& m_scheduler; + + // Persistent key/value storage. + ScopedPointer m_backend; + + // Larger key/value storage, but not necessarily persistent. + ScopedPointer m_fastBackend; + + // VFALCO NOTE What are these things for? We need comments. + TaggedCacheType m_cache; + KeyCache m_negativeCache; +}; + +//------------------------------------------------------------------------------ + +void Database::addFactory (Factory* factory) +{ + Factories::get().add (factory); +} + +void Database::addAvailableBackends () +{ + // This is part of the ripple_app module since it has dependencies + //addFactory (SqliteFactory::getInstance ()); + + addFactory (LevelDBFactory::getInstance ()); + + addFactory (MemoryFactory::getInstance ()); + addFactory (NullFactory::getInstance ()); + +#if RIPPLE_HYPERLEVELDB_AVAILABLE + addFactory (HyperDBFactory::getInstance ()); +#endif + +#if RIPPLE_MDB_AVAILABLE + addFactory (MdbFactory::getInstance ()); +#endif + +#if RIPPLE_SOPHIA_AVAILABLE + addFactory (SophiaFactory::getInstance ()); +#endif + + addFactory (KeyvaDBFactory::getInstance ()); +} + +//------------------------------------------------------------------------------ + +Database* Database::New (char const* name, + Scheduler& scheduler, + Parameters const& backendParameters, + Parameters fastBackendParameters) +{ + return new DatabaseImp (name, + scheduler, backendParameters, fastBackendParameters); +} + +} diff --git a/src/ripple_core/nodestore/impl/DecodedBlob.cpp b/src/ripple_core/nodestore/impl/DecodedBlob.cpp new file mode 100644 index 0000000000..8ef1a1bcdc --- /dev/null +++ b/src/ripple_core/nodestore/impl/DecodedBlob.cpp @@ -0,0 +1,83 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +namespace NodeStore +{ + +DecodedBlob::DecodedBlob (void const* key, void const* value, int valueBytes) +{ + /* Data format: + + Bytes + + 0...3 LedgerIndex 32-bit big endian integer + 4...7 Unused? An unused copy of the LedgerIndex + 8 char One of NodeObjectType + 9...end The body of the object data + */ + + m_success = false; + m_key = key; + // VFALCO NOTE Ledger indexes should have started at 1 + m_ledgerIndex = LedgerIndex (-1); + m_objectType = hotUNKNOWN; + m_objectData = nullptr; + m_dataBytes = bmax (0, valueBytes - 9); + + if (valueBytes > 4) + { + LedgerIndex const* index = static_cast (value); + m_ledgerIndex = ByteOrder::swapIfLittleEndian (*index); + } + + // VFALCO NOTE What about bytes 4 through 7 inclusive? + + if (valueBytes > 8) + { + unsigned char const* byte = static_cast (value); + m_objectType = static_cast (byte [8]); + } + + if (valueBytes > 9) + { + m_objectData = static_cast (value) + 9; + + switch (m_objectType) + { + case hotUNKNOWN: + default: + break; + + case hotLEDGER: + case hotTRANSACTION: + case hotACCOUNT_NODE: + case hotTRANSACTION_NODE: + m_success = true; + break; + } + } +} + +NodeObject::Ptr DecodedBlob::createObject () +{ + bassert (m_success); + + NodeObject::Ptr object; + + if (m_success) + { + Blob data (m_dataBytes); + + memcpy (data.data (), m_objectData, m_dataBytes); + + object = NodeObject::createObject ( + m_objectType, m_ledgerIndex, data, uint256::fromVoid (m_key)); + } + + return object; +} + +} diff --git a/src/ripple_core/nodestore/impl/DecodedBlob.h b/src/ripple_core/nodestore/impl/DecodedBlob.h new file mode 100644 index 0000000000..9e48786fef --- /dev/null +++ b/src/ripple_core/nodestore/impl/DecodedBlob.h @@ -0,0 +1,47 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_DECODEDBLOB_H_INCLUDED +#define RIPPLE_NODESTORE_DECODEDBLOB_H_INCLUDED + +namespace NodeStore +{ + +/** Parsed key/value blob into NodeObject components. + + This will extract the information required to construct a NodeObject. It + also does consistency checking and returns the result, so it is possible + to determine if the data is corrupted without throwing an exception. Not + all forms of corruption are detected so further analysis will be needed + to eliminate false negatives. + + @note This defines the database format of a NodeObject! +*/ +class DecodedBlob +{ +public: + /** Construct the decoded blob from raw data. */ + DecodedBlob (void const* key, void const* value, int valueBytes); + + /** Determine if the decoding was successful. */ + bool wasOk () const noexcept { return m_success; } + + /** Create a NodeObject from this data. */ + NodeObject::Ptr createObject (); + +private: + bool m_success; + + void const* m_key; + LedgerIndex m_ledgerIndex; + NodeObjectType m_objectType; + unsigned char const* m_objectData; + int m_dataBytes; +}; + +} + +#endif diff --git a/src/ripple_core/nodestore/impl/DummyScheduler.cpp b/src/ripple_core/nodestore/impl/DummyScheduler.cpp new file mode 100644 index 0000000000..b6923c259d --- /dev/null +++ b/src/ripple_core/nodestore/impl/DummyScheduler.cpp @@ -0,0 +1,29 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +namespace NodeStore +{ + +DummyScheduler::DummyScheduler () +{ +} + +DummyScheduler::~DummyScheduler () +{ +} + +void DummyScheduler::scheduleTask (Task& task) +{ + // Invoke the task synchronously. + task.performScheduledTask(); +} + +void DummyScheduler::scheduledTasksStopped () +{ +} + +} + diff --git a/src/ripple_core/nodestore/impl/EncodedBlob.cpp b/src/ripple_core/nodestore/impl/EncodedBlob.cpp new file mode 100644 index 0000000000..10e59aedfe --- /dev/null +++ b/src/ripple_core/nodestore/impl/EncodedBlob.cpp @@ -0,0 +1,38 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +namespace NodeStore +{ + +void EncodedBlob::prepare (NodeObject::Ptr const& object) +{ + m_key = object->getHash ().begin (); + + // This is how many bytes we need in the flat data + m_size = object->getData ().size () + 9; + + m_data.ensureSize (m_size); + + // These sizes must be the same! + static_bassert (sizeof (uint32) == sizeof (object->getIndex ())); + + { + uint32* buf = static_cast (m_data.getData ()); + + buf [0] = ByteOrder::swapIfLittleEndian (object->getIndex ()); + buf [1] = ByteOrder::swapIfLittleEndian (object->getIndex ()); + } + + { + unsigned char* buf = static_cast (m_data.getData ()); + + buf [8] = static_cast (object->getType ()); + + memcpy (&buf [9], object->getData ().data (), object->getData ().size ()); + } +} + +} diff --git a/src/ripple_core/nodestore/impl/EncodedBlob.h b/src/ripple_core/nodestore/impl/EncodedBlob.h new file mode 100644 index 0000000000..400a745562 --- /dev/null +++ b/src/ripple_core/nodestore/impl/EncodedBlob.h @@ -0,0 +1,40 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_ENCODEDBLOB_H_INCLUDED +#define RIPPLE_NODESTORE_ENCODEDBLOB_H_INCLUDED + +namespace NodeStore +{ + +/** Utility for producing flattened node objects. + + These get recycled to prevent many small allocations. + + @note This defines the database format of a NodeObject! +*/ +struct EncodedBlob +{ +public: + typedef RecycledObjectPool Pool; + + void prepare (NodeObject::Ptr const& object); + + void const* getKey () const noexcept { return m_key; } + + size_t getSize () const noexcept { return m_size; } + + void const* getData () const noexcept { return m_data.getData (); } + +private: + void const* m_key; + MemoryBlock m_data; + size_t m_size; +}; + +} + +#endif diff --git a/src/ripple_core/nodestore/impl/Factories.h b/src/ripple_core/nodestore/impl/Factories.h new file mode 100644 index 0000000000..80f3f965e0 --- /dev/null +++ b/src/ripple_core/nodestore/impl/Factories.h @@ -0,0 +1,50 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_FACTORIES_H_INCLUDED +#define RIPPLE_NODESTORE_FACTORIES_H_INCLUDED + +namespace NodeStore +{ + +// Holds the list of Backend factories +class Factories +{ +public: + Factories () + { + } + + ~Factories () + { + } + + void add (Factory* factory) + { + m_list.add (factory); + } + + Factory* find (String name) const + { + for (int i = 0; i < m_list.size(); ++i) + if (m_list [i]->getName ().compareIgnoreCase (name) == 0) + return m_list [i]; + return nullptr; + } + + static Factories& get () + { + return *SharedSingleton ::get ( + SingletonLifetime::persistAfterCreation); + } + +private: + OwnedArray m_list; +}; + +} + +#endif diff --git a/src/ripple_core/node/NodeObject.cpp b/src/ripple_core/nodestore/impl/NodeObject.cpp similarity index 88% rename from src/ripple_core/node/NodeObject.cpp rename to src/ripple_core/nodestore/impl/NodeObject.cpp index 1d3f282762..a102f05079 100644 --- a/src/ripple_core/node/NodeObject.cpp +++ b/src/ripple_core/nodestore/impl/NodeObject.cpp @@ -73,19 +73,3 @@ bool NodeObject::isCloneOf (NodeObject::Ptr const& other) const //------------------------------------------------------------------------------ -class NodeObjectTests : public UnitTest -{ -public: - - NodeObjectTests () : UnitTest ("NodeObject", "ripple") - { - } - - - void runTest () - { - } -}; - -static NodeObjectTests nodeObjectTests; - diff --git a/src/ripple_core/nodestore/tests/BackendTests.cpp b/src/ripple_core/nodestore/tests/BackendTests.cpp new file mode 100644 index 0000000000..c45a43e085 --- /dev/null +++ b/src/ripple_core/nodestore/tests/BackendTests.cpp @@ -0,0 +1,97 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +namespace NodeStore +{ + +// Tests the Backend interface +// +class BackendTests : public TestBase +{ +public: + void testBackend (String type, int64 const seedValue, int numObjectsToTest = 2000) + { + DummyScheduler scheduler; + + beginTestCase (String ("Backend type=") + type); + + StringPairArray params; + File const path (File::createTempFile ("node_db")); + params.set ("type", type); + params.set ("path", path.getFullPathName ()); + + // Create a batch + Batch batch; + createPredictableBatch (batch, 0, numObjectsToTest, seedValue); + + { + // Open the backend + ScopedPointer backend (DatabaseImp::createBackend (params, scheduler)); + + // Write the batch + storeBatch (*backend, batch); + + { + // Read it back in + Batch copy; + fetchCopyOfBatch (*backend, ©, batch); + expect (areBatchesEqual (batch, copy), "Should be equal"); + } + + { + // Reorder and read the copy again + Batch copy; + UnitTestUtilities::repeatableShuffle (batch.size (), batch, seedValue); + fetchCopyOfBatch (*backend, ©, batch); + expect (areBatchesEqual (batch, copy), "Should be equal"); + } + } + + { + // Re-open the backend + ScopedPointer backend (DatabaseImp::createBackend (params, scheduler)); + + // Read it back in + Batch copy; + fetchCopyOfBatch (*backend, ©, batch); + // Canonicalize the source and destination batches + std::sort (batch.begin (), batch.end (), NodeObject::LessThan ()); + std::sort (copy.begin (), copy.end (), NodeObject::LessThan ()); + expect (areBatchesEqual (batch, copy), "Should be equal"); + } + } + + //-------------------------------------------------------------------------- + + void runTest () + { + int const seedValue = 50; + + testBackend ("leveldb", seedValue); + + testBackend ("sqlite", seedValue); + + #if RIPPLE_HYPERLEVELDB_AVAILABLE + testBackend ("hyperleveldb", seedValue); + #endif + + #if RIPPLE_MDB_AVAILABLE + testBackend ("mdb", seedValue, 200); + #endif + + #if RIPPLE_SOPHIA_AVAILABLE + testBackend ("sophia", seedValue); + #endif + } + + BackendTests () : TestBase ("NodeStoreBackend") + { + } +}; + +static BackendTests backendTests; + +} diff --git a/src/ripple_core/nodestore/tests/BasicTests.cpp b/src/ripple_core/nodestore/tests/BasicTests.cpp new file mode 100644 index 0000000000..0aba5e1660 --- /dev/null +++ b/src/ripple_core/nodestore/tests/BasicTests.cpp @@ -0,0 +1,76 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +namespace NodeStore +{ + +// Tests predictable batches, and NodeObject blob encoding +// +class BasicTests : public TestBase +{ +public: + BasicTests () : TestBase ("NodeStoreBasics") + { + } + + // Make sure predictable object generation works! + void testBatches (int64 const seedValue) + { + beginTestCase ("batch"); + + Batch batch1; + createPredictableBatch (batch1, 0, numObjectsToTest, seedValue); + + Batch batch2; + createPredictableBatch (batch2, 0, numObjectsToTest, seedValue); + + expect (areBatchesEqual (batch1, batch2), "Should be equal"); + + Batch batch3; + createPredictableBatch (batch3, 1, numObjectsToTest, seedValue); + + expect (! areBatchesEqual (batch1, batch3), "Should not be equal"); + } + + // Checks encoding/decoding blobs + void testBlobs (int64 const seedValue) + { + beginTestCase ("encoding"); + + Batch batch; + createPredictableBatch (batch, 0, numObjectsToTest, seedValue); + + EncodedBlob encoded; + for (int i = 0; i < batch.size (); ++i) + { + encoded.prepare (batch [i]); + + DecodedBlob decoded (encoded.getKey (), encoded.getData (), encoded.getSize ()); + + expect (decoded.wasOk (), "Should be ok"); + + if (decoded.wasOk ()) + { + NodeObject::Ptr const object (decoded.createObject ()); + + expect (batch [i]->isCloneOf (object), "Should be clones"); + } + } + } + + void runTest () + { + int64 const seedValue = 50; + + testBatches (seedValue); + + testBlobs (seedValue); + } +}; + +static BasicTests basicTests; + +} diff --git a/src/ripple_core/nodestore/tests/DatabaseTests.cpp b/src/ripple_core/nodestore/tests/DatabaseTests.cpp new file mode 100644 index 0000000000..c493e7b00c --- /dev/null +++ b/src/ripple_core/nodestore/tests/DatabaseTests.cpp @@ -0,0 +1,228 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +namespace NodeStore +{ + +class DatabaseTests : public TestBase +{ +public: + DatabaseTests () + : TestBase ("NodeStore") + { + } + + ~DatabaseTests () + { + } + + //-------------------------------------------------------------------------- + + void testImport (String destBackendType, String srcBackendType, int64 seedValue) + { + DummyScheduler scheduler; + + File const node_db (File::createTempFile ("node_db")); + StringPairArray srcParams; + srcParams.set ("type", srcBackendType); + srcParams.set ("path", node_db.getFullPathName ()); + + // Create a batch + Batch batch; + createPredictableBatch (batch, 0, numObjectsToTest, seedValue); + + // Write to source db + { + ScopedPointer src (Database::New ("test", scheduler, srcParams)); + storeBatch (*src, batch); + } + + Batch copy; + + { + // Re-open the db + ScopedPointer src (Database::New ("test", scheduler, srcParams)); + + // Set up the destination database + File const dest_db (File::createTempFile ("dest_db")); + StringPairArray destParams; + destParams.set ("type", destBackendType); + destParams.set ("path", dest_db.getFullPathName ()); + + ScopedPointer dest (Database::New ("test", scheduler, destParams)); + + beginTestCase (String ("import into '") + destBackendType + "' from '" + srcBackendType + "'"); + + // Do the import + dest->import (*src); + + // Get the results of the import + fetchCopyOfBatch (*dest, ©, batch); + } + + // Canonicalize the source and destination batches + std::sort (batch.begin (), batch.end (), NodeObject::LessThan ()); + std::sort (copy.begin (), copy.end (), NodeObject::LessThan ()); + expect (areBatchesEqual (batch, copy), "Should be equal"); + + } + + //-------------------------------------------------------------------------- + + void testNodeStore (String type, + bool const useEphemeralDatabase, + bool const testPersistence, + int64 const seedValue, + int numObjectsToTest = 2000) + { + DummyScheduler scheduler; + + String s; + s << String ("NodeStore backend '") + type + "'"; + if (useEphemeralDatabase) + s << " (with ephemeral database)"; + + beginTestCase (s); + + File const node_db (File::createTempFile ("node_db")); + StringPairArray nodeParams; + nodeParams.set ("type", type); + nodeParams.set ("path", node_db.getFullPathName ()); + + File const temp_db (File::createTempFile ("temp_db")); + StringPairArray tempParams; + if (useEphemeralDatabase) + { + tempParams.set ("type", type); + tempParams.set ("path", temp_db.getFullPathName ()); + } + + // Create a batch + Batch batch; + createPredictableBatch (batch, 0, numObjectsToTest, seedValue); + + { + // Open the database + ScopedPointer db (Database::New ("test", scheduler, nodeParams, tempParams)); + + // Write the batch + storeBatch (*db, batch); + + { + // Read it back in + Batch copy; + fetchCopyOfBatch (*db, ©, batch); + expect (areBatchesEqual (batch, copy), "Should be equal"); + } + + { + // Reorder and read the copy again + Batch copy; + UnitTestUtilities::repeatableShuffle (batch.size (), batch, seedValue); + fetchCopyOfBatch (*db, ©, batch); + expect (areBatchesEqual (batch, copy), "Should be equal"); + } + } + + if (testPersistence) + { + { + // Re-open the database without the ephemeral DB + ScopedPointer db (Database::New ("test", scheduler, nodeParams)); + + // Read it back in + Batch copy; + fetchCopyOfBatch (*db, ©, batch); + + // Canonicalize the source and destination batches + std::sort (batch.begin (), batch.end (), NodeObject::LessThan ()); + std::sort (copy.begin (), copy.end (), NodeObject::LessThan ()); + expect (areBatchesEqual (batch, copy), "Should be equal"); + } + + if (useEphemeralDatabase) + { + // Verify the ephemeral db + ScopedPointer db (Database::New ("test", + scheduler, tempParams, StringPairArray ())); + + // Read it back in + Batch copy; + fetchCopyOfBatch (*db, ©, batch); + + // Canonicalize the source and destination batches + std::sort (batch.begin (), batch.end (), NodeObject::LessThan ()); + std::sort (copy.begin (), copy.end (), NodeObject::LessThan ()); + expect (areBatchesEqual (batch, copy), "Should be equal"); + } + } + } + + //-------------------------------------------------------------------------- + + void runBackendTests (bool useEphemeralDatabase, int64 const seedValue) + { + testNodeStore ("leveldb", useEphemeralDatabase, true, seedValue); + + #if RIPPLE_HYPERLEVELDB_AVAILABLE + testNodeStore ("hyperleveldb", useEphemeralDatabase, true, seedValue); + #endif + + #if RIPPLE_MDB_AVAILABLE + testNodeStore ("mdb", useEphemeralDatabase, true, seedValue, 200); + #endif + + #if RIPPLE_SOPHIA_AVAILABLE + testNodeStore ("sophia", useEphemeralDatabase, true, seedValue); + #endif + + testNodeStore ("sqlite", useEphemeralDatabase, true, seedValue); + } + + //-------------------------------------------------------------------------- + + void runImportTests (int64 const seedValue) + { + testImport ("leveldb", "leveldb", seedValue); + + #if RIPPLE_HYPERLEVELDB_AVAILABLE + testImport ("hyperleveldb", "hyperleveldb", seedValue); + #endif + + /* + #if RIPPLE_MDB_AVAILABLE + testImport ("mdb", "mdb", seedValue); + #endif + */ + + /* + #if RIPPLE_SOPHIA_AVAILABLE + testImport ("sophia", "sophia", seedValue); + #endif + */ + + testImport ("sqlite", "sqlite", seedValue); + } + + //-------------------------------------------------------------------------- + + void runTest () + { + int64 const seedValue = 50; + + testNodeStore ("memory", false, false, seedValue); + + runBackendTests (false, seedValue); + + runBackendTests (true, seedValue); + + runImportTests (seedValue); + } +}; + +static DatabaseTests databaseTests; + +} diff --git a/src/ripple_core/nodestore/tests/TestBase.h b/src/ripple_core/nodestore/tests/TestBase.h new file mode 100644 index 0000000000..18a0ea7dea --- /dev/null +++ b/src/ripple_core/nodestore/tests/TestBase.h @@ -0,0 +1,179 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_NODESTORE_TESTBASE_H_INCLUDED +#define RIPPLE_NODESTORE_TESTBASE_H_INCLUDED + +namespace NodeStore +{ + +// Some common code for the unit tests +// +class TestBase : public UnitTest +{ +public: + // Tunable parameters + // + enum + { + maxPayloadBytes = 2000, + numObjectsToTest = 2000 + }; + + // Creates predictable objects + class PredictableObjectFactory + { + public: + explicit PredictableObjectFactory (int64 seedValue) + : m_seedValue (seedValue) + { + } + + NodeObject::Ptr createObject (int index) + { + Random r (m_seedValue + index); + + NodeObjectType type; + switch (r.nextInt (4)) + { + case 0: type = hotLEDGER; break; + case 1: type = hotTRANSACTION; break; + case 2: type = hotACCOUNT_NODE; break; + case 3: type = hotTRANSACTION_NODE; break; + default: + type = hotUNKNOWN; + break; + }; + + LedgerIndex ledgerIndex = 1 + r.nextInt (1024 * 1024); + + uint256 hash; + r.fillBitsRandomly (hash.begin (), hash.size ()); + + int const payloadBytes = 1 + r.nextInt (maxPayloadBytes); + + Blob data (payloadBytes); + + r.fillBitsRandomly (data.data (), payloadBytes); + + return NodeObject::createObject (type, ledgerIndex, data, hash); + } + + private: + int64 const m_seedValue; + }; + +public: + // Create a predictable batch of objects + static void createPredictableBatch (Batch& batch, int startingIndex, int numObjects, int64 seedValue) + { + batch.reserve (numObjects); + + PredictableObjectFactory factory (seedValue); + + for (int i = 0; i < numObjects; ++i) + batch.push_back (factory.createObject (startingIndex + i)); + } + + // Compare two batches for equality + static bool areBatchesEqual (Batch const& lhs, Batch const& rhs) + { + bool result = true; + + if (lhs.size () == rhs.size ()) + { + for (int i = 0; i < lhs.size (); ++i) + { + if (! lhs [i]->isCloneOf (rhs [i])) + { + result = false; + break; + } + } + } + else + { + result = false; + } + + return result; + } + + // Store a batch in a backend + void storeBatch (Backend& backend, Batch const& batch) + { + for (int i = 0; i < batch.size (); ++i) + { + backend.store (batch [i]); + } + } + + // Get a copy of a batch in a backend + void fetchCopyOfBatch (Backend& backend, Batch* pCopy, Batch const& batch) + { + pCopy->clear (); + pCopy->reserve (batch.size ()); + + for (int i = 0; i < batch.size (); ++i) + { + NodeObject::Ptr object; + + Status const status = backend.fetch ( + batch [i]->getHash ().cbegin (), &object); + + expect (status == ok, "Should be ok"); + + if (status == ok) + { + expect (object != nullptr, "Should not be null"); + + pCopy->push_back (object); + } + } + } + + // Store all objects in a batch + static void storeBatch (Database& db, Batch const& batch) + { + for (int i = 0; i < batch.size (); ++i) + { + NodeObject::Ptr const object (batch [i]); + + Blob data (object->getData ()); + + db.store (object->getType (), + object->getIndex (), + data, + object->getHash ()); + } + } + + // Fetch all the hashes in one batch, into another batch. + static void fetchCopyOfBatch (Database& db, + Batch* pCopy, + Batch const& batch) + { + pCopy->clear (); + pCopy->reserve (batch.size ()); + + for (int i = 0; i < batch.size (); ++i) + { + NodeObject::Ptr object = db.fetch (batch [i]->getHash ()); + + if (object != nullptr) + pCopy->push_back (object); + } + } + + TestBase (String name, UnitTest::When when = UnitTest::runNormal) + : UnitTest (name, "ripple", when) + { + } +}; + +} + +#endif diff --git a/src/ripple_core/nodestore/tests/TimingTests.cpp b/src/ripple_core/nodestore/tests/TimingTests.cpp new file mode 100644 index 0000000000..cb5858bdd4 --- /dev/null +++ b/src/ripple_core/nodestore/tests/TimingTests.cpp @@ -0,0 +1,126 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +namespace NodeStore +{ + +class TimingTests : public TestBase +{ +public: + enum + { + numObjectsToTest = 10000 + }; + + TimingTests () + : TestBase ("NodeStoreTiming", UnitTest::runManual) + { + } + + class Stopwatch + { + public: + Stopwatch () + { + } + + void start () + { + m_startTime = Time::getHighResolutionTicks (); + } + + double getElapsed () + { + int64 const now = Time::getHighResolutionTicks(); + + return Time::highResolutionTicksToSeconds (now - m_startTime); + } + + private: + int64 m_startTime; + }; + + //-------------------------------------------------------------------------- + + void testBackend (String type, int64 const seedValue) + { + DummyScheduler scheduler; + + String s; + s << "Testing backend '" << type << "' performance"; + beginTestCase (s); + + StringPairArray params; + File const path (File::createTempFile ("node_db")); + params.set ("type", type); + params.set ("path", path.getFullPathName ()); + + // Create batches + NodeStore::Batch batch1; + createPredictableBatch (batch1, 0, numObjectsToTest, seedValue); + NodeStore::Batch batch2; + createPredictableBatch (batch2, 0, numObjectsToTest, seedValue); + + // Open the backend + ScopedPointer backend (DatabaseImp::createBackend (params, scheduler)); + + Stopwatch t; + + // Individual write batch test + t.start (); + storeBatch (*backend, batch1); + s = ""; + s << " Single write: " << String (t.getElapsed (), 2) << " seconds"; + logMessage (s); + + // Bulk write batch test + t.start (); + backend->storeBatch (batch2); + s = ""; + s << " Batch write: " << String (t.getElapsed (), 2) << " seconds"; + logMessage (s); + + // Read test + Batch copy; + t.start (); + fetchCopyOfBatch (*backend, ©, batch1); + fetchCopyOfBatch (*backend, ©, batch2); + s = ""; + s << " Batch read: " << String (t.getElapsed (), 2) << " seconds"; + logMessage (s); + } + + //-------------------------------------------------------------------------- + + void runTest () + { + int const seedValue = 50; + + testBackend ("leveldb", seedValue); + + #if RIPPLE_HYPERLEVELDB_AVAILABLE + testBackend ("hyperleveldb", seedValue); + #endif + + /* + #if RIPPLE_MDB_AVAILABLE + testBackend ("mdb", seedValue); + #endif + */ + + #if RIPPLE_SOPHIA_AVAILABLE + testBackend ("sophia", seedValue); + #endif + + /* + testBackend ("sqlite", seedValue); + */ + } +}; + +static TimingTests timingTests; + +} diff --git a/src/ripple_core/ripple_core.cpp b/src/ripple_core/ripple_core.cpp index dc2a1cc0f3..f662309a91 100644 --- a/src/ripple_core/ripple_core.cpp +++ b/src/ripple_core/ripple_core.cpp @@ -19,12 +19,9 @@ #include #include -// For NodeStore backends -#include "beast/modules/beast_db/beast_db.h" -#include "../ripple_hyperleveldb/ripple_hyperleveldb.h" -#include "../ripple_leveldb/ripple_leveldb.h" -#include "../ripple_mdb/ripple_mdb.h" -#include "../ripple/sophia/ripple_sophia.h" + + +#include "nodestore/NodeStore.cpp" namespace ripple { @@ -37,23 +34,6 @@ namespace ripple #include "functional/LoadEvent.cpp" #include "functional/LoadMonitor.cpp" -# include "node/HyperLevelDBBackendFactory.h" -# include "node/HyperLevelDBBackendFactory.cpp" -# include "node/KeyvaDBBackendFactory.h" -# include "node/KeyvaDBBackendFactory.cpp" -# include "node/LevelDBBackendFactory.h" -# include "node/LevelDBBackendFactory.cpp" -# include "node/MemoryBackendFactory.h" -# include "node/MemoryBackendFactory.cpp" -# include "node/NullBackendFactory.h" -# include "node/NullBackendFactory.cpp" -# include "node/MdbBackendFactory.h" -# include "node/MdbBackendFactory.cpp" -# include "node/SophiaBackendFactory.h" -# include "node/SophiaBackendFactory.cpp" -#include "node/NodeStore.cpp" -#include "node/NodeObject.cpp" - #include "peerfinder/PeerFinder.cpp" } diff --git a/src/ripple_core/ripple_core.h b/src/ripple_core/ripple_core.h index dbcb12c48d..e5da96c839 100644 --- a/src/ripple_core/ripple_core.h +++ b/src/ripple_core/ripple_core.h @@ -14,6 +14,10 @@ #include "../ripple_basics/ripple_basics.h" #include "../ripple_data/ripple_data.h" + + +#include "nodestore/NodeStore.h" + namespace ripple { @@ -29,9 +33,6 @@ namespace ripple # include "functional/LoadType.h" #include "functional/LoadSource.h" -#include "node/NodeObject.h" -#include "node/NodeStore.h" - #include "peerfinder/PeerFinder.h" }