Refactor NodeStore, add NodeStoreSchedulerService

Builds/VisualStudio2012/RippleD.vcxproj

@@ -243,6 +243,12 @@
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
     </ClCompile>
+    <ClCompile Include="..\..\src\ripple_app\main\NodeStoreSchedulerService.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
+    </ClCompile>
     <ClCompile Include="..\..\src\ripple_app\main\ParameterTable.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
@@ -375,7 +381,7 @@
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
     </ClCompile>
-    <ClCompile Include="..\..\src\ripple_app\node\SqliteBackendFactory.cpp">
+    <ClCompile Include="..\..\src\ripple_app\node\SqliteFactory.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
@@ -821,55 +827,103 @@
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
     </ClCompile>
-    <ClCompile Include="..\..\src\ripple_core\node\HyperLevelDBBackendFactory.cpp">
+    <ClCompile Include="..\..\src\ripple_core\nodestore\backend\HyperDBFactory.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
     </ClCompile>
-    <ClCompile Include="..\..\src\ripple_core\node\KeyvaDBBackendFactory.cpp">
+    <ClCompile Include="..\..\src\ripple_core\nodestore\backend\KeyvaDBFactory.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
     </ClCompile>
-    <ClCompile Include="..\..\src\ripple_core\node\LevelDBBackendFactory.cpp">
+    <ClCompile Include="..\..\src\ripple_core\nodestore\backend\LevelDBFactory.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
     </ClCompile>
-    <ClCompile Include="..\..\src\ripple_core\node\MdbBackendFactory.cpp">
+    <ClCompile Include="..\..\src\ripple_core\nodestore\backend\MdbFactory.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
     </ClCompile>
-    <ClCompile Include="..\..\src\ripple_core\node\MemoryBackendFactory.cpp">
+    <ClCompile Include="..\..\src\ripple_core\nodestore\backend\MemoryFactory.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
     </ClCompile>
-    <ClCompile Include="..\..\src\ripple_core\node\NodeObject.cpp">
+    <ClCompile Include="..\..\src\ripple_core\nodestore\backend\NullFactory.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
     </ClCompile>
-    <ClCompile Include="..\..\src\ripple_core\node\NodeStore.cpp">
+    <ClCompile Include="..\..\src\ripple_core\nodestore\backend\SophiaFactory.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
     </ClCompile>
-    <ClCompile Include="..\..\src\ripple_core\node\NullBackendFactory.cpp">
+    <ClCompile Include="..\..\src\ripple_core\nodestore\impl\BatchWriter.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
     </ClCompile>
-    <ClCompile Include="..\..\src\ripple_core\node\SophiaBackendFactory.cpp">
+    <ClCompile Include="..\..\src\ripple_core\nodestore\impl\DecodedBlob.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\impl\DummyScheduler.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\impl\EncodedBlob.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\impl\NodeObject.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\NodeStore.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\tests\BackendTests.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\tests\BasicTests.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\tests\DatabaseTests.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\tests\TimingTests.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
@@ -1499,6 +1553,7 @@
     <ClInclude Include="..\..\src\ripple_app\ledger\LedgerHistory.h" />
     <ClInclude Include="..\..\src\ripple_app\ledger\SerializedValidation.h" />
     <ClInclude Include="..\..\src\ripple_app\main\IoServicePool.h" />
+    <ClInclude Include="..\..\src\ripple_app\main\NodeStoreSchedulerService.h" />
     <ClInclude Include="..\..\src\ripple_app\main\ParameterTable.h" />
     <ClInclude Include="..\..\src\ripple_app\main\Application.h" />
     <ClInclude Include="..\..\src\ripple_app\main\FatalErrorReporter.h" />
@@ -1522,7 +1577,7 @@
     <ClInclude Include="..\..\src\ripple_app\misc\SerializedLedger.h" />
     <ClInclude Include="..\..\src\ripple_app\misc\SerializedTransaction.h" />
     <ClInclude Include="..\..\src\ripple_app\misc\Validations.h" />
-    <ClInclude Include="..\..\src\ripple_app\node\SqliteBackendFactory.h" />
+    <ClInclude Include="..\..\src\ripple_app\node\SqliteFactory.h" />
     <ClInclude Include="..\..\src\ripple_app\paths\Pathfinder.h" />
     <ClInclude Include="..\..\src\ripple_app\paths\PathRequest.h" />
     <ClInclude Include="..\..\src\ripple_app\paths\PathState.h" />
@@ -1605,15 +1660,29 @@
     <ClInclude Include="..\..\src\ripple_core\functional\LoadEvent.h" />
     <ClInclude Include="..\..\src\ripple_core\functional\LoadFeeTrack.h" />
     <ClInclude Include="..\..\src\ripple_core\functional\LoadMonitor.h" />
-    <ClInclude Include="..\..\src\ripple_core\node\HyperLevelDBBackendFactory.h" />
+    <ClInclude Include="..\..\src\ripple_core\nodestore\api\Backend.h" />
-    <ClInclude Include="..\..\src\ripple_core\node\KeyvaDBBackendFactory.h" />
+    <ClInclude Include="..\..\src\ripple_core\nodestore\api\Database.h" />
-    <ClInclude Include="..\..\src\ripple_core\node\LevelDBBackendFactory.h" />
+    <ClInclude Include="..\..\src\ripple_core\nodestore\api\DummyScheduler.h" />
-    <ClInclude Include="..\..\src\ripple_core\node\MdbBackendFactory.h" />
+    <ClInclude Include="..\..\src\ripple_core\nodestore\api\Factory.h" />
-    <ClInclude Include="..\..\src\ripple_core\node\MemoryBackendFactory.h" />
+    <ClInclude Include="..\..\src\ripple_core\nodestore\api\NodeObject.h" />
-    <ClInclude Include="..\..\src\ripple_core\node\NodeObject.h" />
+    <ClInclude Include="..\..\src\ripple_core\nodestore\api\Scheduler.h" />
-    <ClInclude Include="..\..\src\ripple_core\node\NodeStore.h" />
+    <ClInclude Include="..\..\src\ripple_core\nodestore\api\Task.h" />
-    <ClInclude Include="..\..\src\ripple_core\node\NullBackendFactory.h" />
+    <ClInclude Include="..\..\src\ripple_core\nodestore\api\Types.h" />
-    <ClInclude Include="..\..\src\ripple_core\node\SophiaBackendFactory.h" />
+    <ClInclude Include="..\..\src\ripple_core\nodestore\api\VisitCallback.h" />
+    <ClInclude Include="..\..\src\ripple_core\nodestore\backend\HyperDBFactory.h" />
+    <ClInclude Include="..\..\src\ripple_core\nodestore\backend\KeyvaDBFactory.h" />
+    <ClInclude Include="..\..\src\ripple_core\nodestore\backend\LevelDBFactory.h" />
+    <ClInclude Include="..\..\src\ripple_core\nodestore\backend\MdbFactory.h" />
+    <ClInclude Include="..\..\src\ripple_core\nodestore\backend\MemoryFactory.h" />
+    <ClInclude Include="..\..\src\ripple_core\nodestore\backend\NullFactory.h" />
+    <ClInclude Include="..\..\src\ripple_core\nodestore\backend\SophiaFactory.h" />
+    <ClInclude Include="..\..\src\ripple_core\nodestore\impl\BatchWriter.h" />
+    <ClInclude Include="..\..\src\ripple_core\nodestore\impl\DatabaseImp.h" />
+    <ClInclude Include="..\..\src\ripple_core\nodestore\impl\DecodedBlob.h" />
+    <ClInclude Include="..\..\src\ripple_core\nodestore\impl\EncodedBlob.h" />
+    <ClInclude Include="..\..\src\ripple_core\nodestore\impl\Factories.h" />
+    <ClInclude Include="..\..\src\ripple_core\nodestore\NodeStore.h" />
+    <ClInclude Include="..\..\src\ripple_core\nodestore\tests\TestBase.h" />
     <ClInclude Include="..\..\src\ripple_core\peerfinder\PeerFinder.h" />
     <ClInclude Include="..\..\src\ripple_core\ripple_core.h" />
     <ClInclude Include="..\..\src\ripple_data\crypto\Base58.h" />

Builds/VisualStudio2012/RippleD.vcxproj.filters

@@ -103,9 +103,6 @@
     <Filter Include="[2] Old Ripple\ripple_core\functional">
       <UniqueIdentifier>{548037f2-eb8a-41bd-95dc-05f58cdbc041}</UniqueIdentifier>
     </Filter>
-    <Filter Include="[2] Old Ripple\ripple_core\node">
-      <UniqueIdentifier>{7270818b-573d-45b7-b10b-42f63bedcc50}</UniqueIdentifier>
-    </Filter>
     <Filter Include="[2] Old Ripple\ripple_core\peerfinder">
       <UniqueIdentifier>{d1648d3f-7d71-495d-afc9-576ed00d7185}</UniqueIdentifier>
     </Filter>
@@ -190,6 +187,21 @@
     <Filter Include="[1] Ripple\frame\api">
       <UniqueIdentifier>{a4dd852c-651b-4ea9-a051-252dc0eaea24}</UniqueIdentifier>
     </Filter>
+    <Filter Include="[2] Old Ripple\ripple_core\nodestore">
+      <UniqueIdentifier>{490d0cdb-7ce1-42f2-b234-6874dae550b7}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="[2] Old Ripple\ripple_core\nodestore\api">
+      <UniqueIdentifier>{64fa2598-39d6-46de-81b0-17131f06d943}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="[2] Old Ripple\ripple_core\nodestore\impl">
+      <UniqueIdentifier>{7aa0afca-11d5-43e5-bd95-ff26626cd2fc}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="[2] Old Ripple\ripple_core\nodestore\backend">
+      <UniqueIdentifier>{410e4aa6-1a25-4ec3-a7fd-b042bdf7c0de}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="[2] Old Ripple\ripple_core\nodestore\tests">
+      <UniqueIdentifier>{071582fa-cf16-4e41-8791-613cfe00eef8}</UniqueIdentifier>
+    </Filter>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\..\src\ripple_basics\containers\RangeSet.cpp">
@@ -768,33 +780,6 @@
     <ClCompile Include="..\..\src\ripple_app\tx\TxQueueEntry.cpp">
       <Filter>[2] Old Ripple\ripple_app\tx</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\src\ripple_app\node\SqliteBackendFactory.cpp">
-      <Filter>[2] Old Ripple\ripple_app\node</Filter>
-    </ClCompile>
-    <ClCompile Include="..\..\src\ripple_core\node\HyperLevelDBBackendFactory.cpp">
-      <Filter>[2] Old Ripple\ripple_core\node</Filter>
-    </ClCompile>
-    <ClCompile Include="..\..\src\ripple_core\node\KeyvaDBBackendFactory.cpp">
-      <Filter>[2] Old Ripple\ripple_core\node</Filter>
-    </ClCompile>
-    <ClCompile Include="..\..\src\ripple_core\node\LevelDBBackendFactory.cpp">
-      <Filter>[2] Old Ripple\ripple_core\node</Filter>
-    </ClCompile>
-    <ClCompile Include="..\..\src\ripple_core\node\MdbBackendFactory.cpp">
-      <Filter>[2] Old Ripple\ripple_core\node</Filter>
-    </ClCompile>
-    <ClCompile Include="..\..\src\ripple_core\node\MemoryBackendFactory.cpp">
-      <Filter>[2] Old Ripple\ripple_core\node</Filter>
-    </ClCompile>
-    <ClCompile Include="..\..\src\ripple_core\node\NodeObject.cpp">
-      <Filter>[2] Old Ripple\ripple_core\node</Filter>
-    </ClCompile>
-    <ClCompile Include="..\..\src\ripple_core\node\NodeStore.cpp">
-      <Filter>[2] Old Ripple\ripple_core\node</Filter>
-    </ClCompile>
-    <ClCompile Include="..\..\src\ripple_core\node\NullBackendFactory.cpp">
-      <Filter>[2] Old Ripple\ripple_core\node</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\src\protobuf_core.cpp">
       <Filter>[0] Libraries\protobuf</Filter>
     </ClCompile>
@@ -921,9 +906,6 @@
     <ClCompile Include="..\..\src\ripple_net\basics\AsyncService.cpp">
       <Filter>[2] Old Ripple\ripple_net\basics</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\src\ripple_core\node\SophiaBackendFactory.cpp">
-      <Filter>[2] Old Ripple\ripple_core\node</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\src\ripple\sophia\ripple_sophia.c">
       <Filter>[1] Ripple\sophia</Filter>
     </ClCompile>
@@ -945,6 +927,63 @@
     <ClCompile Include="..\..\src\ripple\frame\api\Service.cpp">
       <Filter>[1] Ripple\frame\api</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\NodeStore.cpp">
+      <Filter>[2] Old Ripple\ripple_core\nodestore</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\impl\NodeObject.cpp">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\impl</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\impl\BatchWriter.cpp">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\impl</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\impl\DecodedBlob.cpp">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\impl</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\impl\EncodedBlob.cpp">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\impl</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\tests\TimingTests.cpp">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\tests</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\backend\SophiaFactory.cpp">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\backend</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\backend\NullFactory.cpp">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\backend</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\backend\MemoryFactory.cpp">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\backend</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\backend\MdbFactory.cpp">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\backend</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\backend\LevelDBFactory.cpp">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\backend</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\backend\KeyvaDBFactory.cpp">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\backend</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\backend\HyperDBFactory.cpp">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\backend</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\tests\BackendTests.cpp">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\tests</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\tests\BasicTests.cpp">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\tests</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\tests\DatabaseTests.cpp">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\tests</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_core\nodestore\impl\DummyScheduler.cpp">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\impl</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_app\main\NodeStoreSchedulerService.cpp">
+      <Filter>[2] Old Ripple\ripple_app\main</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_app\node\SqliteFactory.cpp">
+      <Filter>[2] Old Ripple\ripple_app\node</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\src\ripple_basics\containers\KeyCache.h">
@@ -1652,33 +1691,6 @@
     <ClInclude Include="..\..\src\ripple_app\tx\TxQueueEntry.h">
       <Filter>[2] Old Ripple\ripple_app\tx</Filter>
     </ClInclude>
-    <ClInclude Include="..\..\src\ripple_app\node\SqliteBackendFactory.h">
-      <Filter>[2] Old Ripple\ripple_app\node</Filter>
-    </ClInclude>
-    <ClInclude Include="..\..\src\ripple_core\node\HyperLevelDBBackendFactory.h">
-      <Filter>[2] Old Ripple\ripple_core\node</Filter>
-    </ClInclude>
-    <ClInclude Include="..\..\src\ripple_core\node\KeyvaDBBackendFactory.h">
-      <Filter>[2] Old Ripple\ripple_core\node</Filter>
-    </ClInclude>
-    <ClInclude Include="..\..\src\ripple_core\node\LevelDBBackendFactory.h">
-      <Filter>[2] Old Ripple\ripple_core\node</Filter>
-    </ClInclude>
-    <ClInclude Include="..\..\src\ripple_core\node\MdbBackendFactory.h">
-      <Filter>[2] Old Ripple\ripple_core\node</Filter>
-    </ClInclude>
-    <ClInclude Include="..\..\src\ripple_core\node\MemoryBackendFactory.h">
-      <Filter>[2] Old Ripple\ripple_core\node</Filter>
-    </ClInclude>
-    <ClInclude Include="..\..\src\ripple_core\node\NodeObject.h">
-      <Filter>[2] Old Ripple\ripple_core\node</Filter>
-    </ClInclude>
-    <ClInclude Include="..\..\src\ripple_core\node\NodeStore.h">
-      <Filter>[2] Old Ripple\ripple_core\node</Filter>
-    </ClInclude>
-    <ClInclude Include="..\..\src\ripple_core\node\NullBackendFactory.h">
-      <Filter>[2] Old Ripple\ripple_core\node</Filter>
-    </ClInclude>
     <ClInclude Include="..\..\src\ripple_app\misc\Validations.h">
       <Filter>[2] Old Ripple\ripple_app\misc</Filter>
     </ClInclude>
@@ -1824,9 +1836,6 @@
     <ClInclude Include="..\..\src\ripple\sophia\ripple_sophia.h">
       <Filter>[1] Ripple\sophia</Filter>
     </ClInclude>
-    <ClInclude Include="..\..\src\ripple_core\node\SophiaBackendFactory.h">
-      <Filter>[2] Old Ripple\ripple_core\node</Filter>
-    </ClInclude>
     <ClInclude Include="..\..\src\ripple\json\api\json_features.h">
       <Filter>[1] Ripple\json\api</Filter>
     </ClInclude>
@@ -1860,6 +1869,81 @@
     <ClInclude Include="..\..\src\ripple\frame\api\Service.h">
       <Filter>[1] Ripple\frame\api</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\src\ripple_core\nodestore\NodeStore.h">
+      <Filter>[2] Old Ripple\ripple_core\nodestore</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_core\nodestore\api\Database.h">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\api</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_core\nodestore\api\NodeObject.h">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\api</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_core\nodestore\api\Types.h">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\api</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_core\nodestore\impl\DecodedBlob.h">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\impl</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_core\nodestore\impl\EncodedBlob.h">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\impl</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_core\nodestore\api\Scheduler.h">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\api</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_core\nodestore\api\DummyScheduler.h">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\api</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_core\nodestore\impl\BatchWriter.h">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\impl</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_core\nodestore\api\Backend.h">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\api</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_core\nodestore\api\Factory.h">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\api</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_core\nodestore\impl\Factories.h">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\impl</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_core\nodestore\api\VisitCallback.h">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\api</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_core\nodestore\backend\SophiaFactory.h">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\backend</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_core\nodestore\backend\NullFactory.h">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\backend</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_core\nodestore\backend\MemoryFactory.h">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\backend</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_core\nodestore\backend\MdbFactory.h">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\backend</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_core\nodestore\backend\LevelDBFactory.h">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\backend</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_core\nodestore\backend\KeyvaDBFactory.h">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\backend</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_core\nodestore\backend\HyperDBFactory.h">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\backend</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_core\nodestore\tests\TestBase.h">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\tests</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_core\nodestore\impl\DatabaseImp.h">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\impl</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_app\main\NodeStoreSchedulerService.h">
+      <Filter>[2] Old Ripple\ripple_app\main</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_core\nodestore\api\Task.h">
+      <Filter>[2] Old Ripple\ripple_core\nodestore\api</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\ripple_app\node\SqliteFactory.h">
+      <Filter>[2] Old Ripple\ripple_app\node</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <CustomBuild Include="..\..\src\ripple_data\protocol\ripple.proto">

SConstruct

@@ -146,7 +146,7 @@ COMPILED_FILES.extend (['src/ripple/beast/ripple_beastc.c'])
 # New-style Ripple unity sources
 #
 COMPILED_FILES.extend([
-    'src/ripple/frame/ripple_frame.c',
+    'src/ripple/frame/ripple_frame.cpp',
     'src/ripple/json/ripple_json.cpp',
     'src/ripple/sophia/ripple_sophia.c',
     'src/ripple/testoverlay/ripple_testoverlay.cpp',

src/ripple/frame/api/Service.cpp

@@ -181,25 +181,4 @@ void Service::stopRecursive (Journal::Stream stream)
     m_stopped = true;
 }
 
-//------------------------------------------------------------------------------
-
-ScopedService::ScopedService (char const* name)
-    : Service (name)
-{
-}
-
-ScopedService::~ScopedService ()
-{
-    serviceStop();
-}
-
-void ScopedService::onServiceStop ()
-{
-    serviceStopped();
-}
-
-void ScopedService::onServiceChildrenStopped ()
-{
-}
-
 }

src/ripple/frame/api/Service.h

@@ -263,22 +263,6 @@ private:
     WaitableEvent m_stoppedEvent;
 };
 
-//------------------------------------------------------------------------------
-
-/** A root Service with a short scope.
-    This Service takes care of stopping automatically, no additional
-    action is required.
-*/
-class ScopedService : public Service
-{
-public:
-    explicit ScopedService (char const* name);
-    ~ScopedService ();
-
-    void onServiceStop ();
-    void onServiceChildrenStopped ();
-};
-
 }
 
 #endif

src/ripple_app/main/Application.cpp

@@ -29,7 +29,6 @@ template <> char const* LogPartition::getPartitionName <NetworkOPsLog> () { retu
 // VFALCO TODO Move the function definitions into the class declaration
 class ApplicationImp
     : public Application
-    , public NodeStore::Scheduler
     , public Service
     , public DeadlineTimer::Listener
     , LeakChecked <ApplicationImp>
@@ -80,8 +79,10 @@ public:
 
         , m_rpcServerHandler (*m_networkOPs) // passive object, not a Service
 
-        , m_nodeStore (NodeStore::New ("NodeStore.main", *m_jobQueue,
+        , m_nodeStoreScheduler (*m_jobQueue, *m_jobQueue)
-            getConfig ().nodeDatabase, getConfig ().ephemeralNodeDatabase, *this))
+
+        , m_nodeStore (NodeStore::Database::New ("NodeStore.main", m_nodeStoreScheduler,
+            getConfig ().nodeDatabase, getConfig ().ephemeralNodeDatabase))
 
         , m_sntpClient (SNTPClient::New (*this))
 
@@ -158,27 +159,6 @@ public:
 
     //--------------------------------------------------------------------------
 
-    static void callScheduledTask (NodeStore::Scheduler::Task& task, Job&)
-    {
-        task.performScheduledTask ();
-    }
-
-    void scheduleTask (NodeStore::Scheduler::Task& task)
-    {
-        getJobQueue ().addJob (
-            jtWRITE,
-            "NodeObject::store",
-            BIND_TYPE (&ApplicationImp::callScheduledTask, boost::ref(task), P_1));
-    }
-
-    void scheduledTasksStopped ()
-    {
-        // VFALCO NOTE This is a bit of a hack
-        getNodeStore().serviceStopped();
-    }
-
-    //--------------------------------------------------------------------------
-
     LocalCredentials& getLocalCredentials ()
     {
         return m_localCredentials ;
@@ -214,7 +194,7 @@ public:
         return m_tempNodeCache;
     }
 
-    NodeStore& getNodeStore ()
+    NodeStore::Database& getNodeStore ()
     {
         return *m_nodeStore;
     }
@@ -635,9 +615,6 @@ public:
 
         mShutdown = true;
 
-        // This stalls for a long time
-        //m_nodeStore = nullptr;
-
         mValidations->flush ();
         mShutdown = false;
 
@@ -750,7 +727,7 @@ public:
             &TransactionMaster::sweep, &m_txMaster));
 
         logTimedCall (m_journal.warning, "NodeStore::sweep", __FILE__, __LINE__, boost::bind (
-            &NodeStore::sweep, m_nodeStore.get ()));
+            &NodeStore::Database::sweep, m_nodeStore.get ()));
 
         logTimedCall (m_journal.warning, "LedgerMaster::sweep", __FILE__, __LINE__, boost::bind (
             &LedgerMaster::sweep, &m_ledgerMaster));
@@ -806,7 +783,8 @@ private:
     ScopedPointer <NetworkOPs> m_networkOPs;
     ScopedPointer <UniqueNodeList> m_deprecatedUNL;
     RPCServerHandler m_rpcServerHandler;
-    ScopedPointer <NodeStore> m_nodeStore;
+    NodeStoreSchedulerService m_nodeStoreScheduler;
+    ScopedPointer <NodeStore::Database> m_nodeStore;
     ScopedPointer <SNTPClient> m_sntpClient;
     InboundLedgers m_inboundLedgers;
     ScopedPointer <TxQueue> m_txQueue;
@@ -1138,9 +1116,9 @@ void ApplicationImp::updateTables ()
 
     if (getConfig ().importNodeDatabase.size () > 0)
     {
-        ScopedService service ("import service");
+        NodeStore::DummyScheduler scheduler;
-        ScopedPointer <NodeStore> source (NodeStore::New (
+        ScopedPointer <NodeStore::Database> source (NodeStore::Database::New (
-            "NodeStore.import", service, getConfig ().importNodeDatabase));
+            "NodeStore.import", scheduler, getConfig ().importNodeDatabase));
 
         WriteLog (lsWARNING, NodeObject) <<
             "Node import from '" << source->getName () << "' to '"

src/ripple_app/main/Application.h

@@ -8,6 +8,7 @@
 #define RIPPLE_APP_APPLICATION_H_INCLUDED
 
 namespace Validators { class Manager; }
+namespace NodeStore { class Database; }
 
 // VFALCO TODO Fix forward declares required for header dependency loops
 class IFeatures;
@@ -16,8 +17,6 @@ class IHashRouter;
 class ILoadFeeTrack;
 class Peers;
 class UniqueNodeList;
-
-class NodeStore;
 class JobQueue;
 class InboundLedgers;
 class LedgerMaster;
@@ -90,7 +89,7 @@ public:
     virtual ProofOfWorkFactory&     getProofOfWorkFactory () = 0;
     virtual UniqueNodeList&         getUNL () = 0;
     virtual Validations&            getValidations () = 0;
-    virtual NodeStore&              getNodeStore () = 0;
+    virtual NodeStore::Database&    getNodeStore () = 0;
     virtual JobQueue&               getJobQueue () = 0;
     virtual InboundLedgers&         getInboundLedgers () = 0;
     virtual LedgerMaster&           getLedgerMaster () = 0;

src/ripple_app/main/NodeStoreSchedulerService.cpp

@@ -0,0 +1,39 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+NodeStoreSchedulerService::NodeStoreSchedulerService (Service& parent, JobQueue& jobQueue)
+    : Service ("NodeStoreSchedulerService", parent)
+    , m_jobQueue (jobQueue)
+    , m_taskCount (1) // start it off at 1
+{
+}
+
+void NodeStoreSchedulerService::onServiceStop ()
+{
+    if (--m_taskCount == 0)
+        serviceStopped();
+}
+
+void NodeStoreSchedulerService::onServiceChildrenStopped ()
+{
+}
+
+void NodeStoreSchedulerService::scheduleTask (NodeStore::Task& task)
+{
+    ++m_taskCount;
+    m_jobQueue.addJob (
+        jtWRITE,
+        "NodeObject::store",
+        BIND_TYPE (&NodeStoreSchedulerService::doTask,
+            this, boost::ref(task), P_1));
+}
+
+void NodeStoreSchedulerService::doTask (NodeStore::Task& task, Job&)
+{
+    task.performScheduledTask ();
+    if ((--m_taskCount == 0) && isServiceStopping())
+        serviceStopped();
+}

src/ripple_app/main/NodeStoreSchedulerService.h

@@ -0,0 +1,30 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_APP_NODESTORESCHEDULERSERVICE_H_INCLUDED
+#define RIPPLE_APP_NODESTORESCHEDULERSERVICE_H_INCLUDED
+
+/** A NodeStore::Scheduler which uses the JobQueue and implements the Service API. */
+class NodeStoreSchedulerService
+    : public NodeStore::Scheduler
+    , public Service
+{
+public:
+    NodeStoreSchedulerService (Service& parent, JobQueue& jobQueue);
+
+    void onServiceStop ();
+    void onServiceChildrenStopped ();
+    void scheduleTask (NodeStore::Task& task);
+
+private:
+    void doTask (NodeStore::Task& task, Job&);
+
+    JobQueue& m_jobQueue;
+    Atomic <int> m_taskCount;
+};
+
+
+#endif

src/ripple_app/main/RippleMain.cpp

@@ -269,12 +269,12 @@ int RippleMain::run (int argc, char const* const* argv)
     // NOTE: These must be added before the
     //       Application object is created.
     //
-    NodeStore::addAvailableBackends ();
+    NodeStore::Database::addAvailableBackends ();
 
-    // VFALCO NOTE SqliteBackendFactory is here because it has
+    // VFALCO NOTE SqliteFactory is here because it has
     //             dependencies like SqliteDatabase and DatabaseCon
     //
-    NodeStore::addBackendFactory (SqliteBackendFactory::getInstance ());
+    NodeStore::Database::addFactory (SqliteFactory::getInstance ());
 
     if (! RandomNumbers::getInstance ().initialize ())
     {

src/ripple_app/node/SqliteFactory.cpp

@@ -30,10 +30,10 @@ static int s_nodeStoreDBCount = NUMBER (s_nodeStoreDBInit);
 
 //------------------------------------------------------------------------------
 
-class SqliteBackendFactory::Backend : public NodeStore::Backend
+class SqliteFactory::BackendImp : public NodeStore::Backend
 {
 public:
-    Backend (size_t keyBytes, std::string const& path, NodeStore::Scheduler& scheduler)
+    BackendImp (size_t keyBytes, std::string const& path, NodeStore::Scheduler& scheduler)
         : m_keyBytes (keyBytes)
         , m_name (path)
         , m_db (new DatabaseCon(path, s_nodeStoreDBInit, s_nodeStoreDBCount))
@@ -47,7 +47,7 @@ public:
         m_db->getDB()->executeSQL (s.toStdString ().c_str ());
     }
 
-    ~Backend()
+    ~BackendImp()
     {
     }
 
@@ -58,9 +58,9 @@ public:
 
     //--------------------------------------------------------------------------
 
-    Status fetch (void const* key, NodeObject::Ptr* pObject)
+    NodeStore::Status fetch (void const* key, NodeObject::Ptr* pObject)
     {
-        Status result = ok;
+        NodeStore::Status result = NodeStore::ok;
 
         pObject->reset ();
 
@@ -87,7 +87,7 @@ public:
             }
             else
             {
-                result = notFound;
+                result = NodeStore::notFound;
             }
 
             pSt.reset();
@@ -132,7 +132,7 @@ public:
         pStE.reset();
     }
 
-    void visitAll (VisitCallback& callback)
+    void visitAll (NodeStore::VisitCallback& callback)
     {
         // No lock needed as per the visitAll() API
 
@@ -165,11 +165,6 @@ public:
         return 0;
     }
 
-    void stopAsync ()
-    {
-        m_scheduler.scheduledTasksStopped ();
-    }
-
     //--------------------------------------------------------------------------
 
     void doBind (SqliteStatement& statement, NodeObject::ref object)
@@ -216,28 +211,28 @@ private:
 
 //------------------------------------------------------------------------------
 
-SqliteBackendFactory::SqliteBackendFactory ()
+SqliteFactory::SqliteFactory ()
 {
 }
 
-SqliteBackendFactory::~SqliteBackendFactory ()
+SqliteFactory::~SqliteFactory ()
 {
 }
 
-SqliteBackendFactory* SqliteBackendFactory::getInstance ()
+SqliteFactory* SqliteFactory::getInstance ()
 {
-    return new SqliteBackendFactory;
+    return new SqliteFactory;
 }
 
-String SqliteBackendFactory::getName () const
+String SqliteFactory::getName () const
 {
     return "Sqlite";
 }
 
-NodeStore::Backend* SqliteBackendFactory::createInstance (
+NodeStore::Backend* SqliteFactory::createInstance (
     size_t keyBytes,
-    StringPairArray const& keyValues,
+    NodeStore::Parameters const& keyValues,
     NodeStore::Scheduler& scheduler)
 {
-    return new Backend (keyBytes, keyValues ["path"].toStdString (), scheduler);
+    return new BackendImp (keyBytes, keyValues ["path"].toStdString (), scheduler);
 }

src/ripple_app/node/SqliteFactory.h

@@ -4,28 +4,28 @@
 */
 //==============================================================================
 
-#ifndef RIPPLE_SQLITEBACKENDFACTORY_H_INCLUDED
+#ifndef RIPPLE_APP_SQLITEFACTORY_H_INCLUDED
-#define RIPPLE_SQLITEBACKENDFACTORY_H_INCLUDED
+#define RIPPLE_APP_SQLITEFACTORY_H_INCLUDED
 
 /** Factory to produce SQLite backends for the NodeStore.
 
-    @see NodeStore
+    @see Database
 */
-class SqliteBackendFactory : public NodeStore::BackendFactory
+class SqliteFactory : public NodeStore::Factory
 {
 private:
-    class Backend;
+    SqliteFactory ();
-
+    ~SqliteFactory ();
-    SqliteBackendFactory ();
-    ~SqliteBackendFactory ();
 
 public:
-    static SqliteBackendFactory* getInstance ();
+    class BackendImp;
+
+    static SqliteFactory* getInstance ();
 
     String getName () const;
 
     NodeStore::Backend* createInstance (size_t keyBytes,
-                                        StringPairArray const& keyValues,
+                                        NodeStore::Parameters const& keyValues,
                                         NodeStore::Scheduler& scheduler);
 };
 

src/ripple_app/ripple_app.cpp

@@ -30,6 +30,9 @@ namespace ripple
 // Application
 //
 
+# include "main/NodeStoreSchedulerService.h"
+#include "main/NodeStoreSchedulerService.cpp"
+
 # include "main/IoServicePool.h"
 #include "main/IoServicePool.cpp"
 
@@ -66,8 +69,8 @@ namespace ripple
 // RippleMain
 //
 # include "main/RippleMain.h"
-# include "node/SqliteBackendFactory.h"
+# include "node/SqliteFactory.h"
-#include "node/SqliteBackendFactory.cpp"
+#include "node/SqliteFactory.cpp"
 #include "main/RippleMain.cpp"
 
 }

src/ripple_app/shamap/SHAMap.cpp

@@ -856,7 +856,7 @@ SHAMapTreeNode::pointer SHAMap::fetchNodeExternalNT (const SHAMapNode& id, uint2
 
     // These are for diagnosing a crash on exit
     Application& app (getApp ());
-    NodeStore& nodeStore (app.getNodeStore ());
+    NodeStore::Database& nodeStore (app.getNodeStore ());
     NodeObject::pointer obj (nodeStore.fetch (hash));
 
     if (!obj)

src/ripple_core/functional/Config.h

@@ -288,7 +288,7 @@ public:
         This is 1 or more strings of the form <key>=<value>
         The 'type' and 'path' keys are required, see rippled-example.cfg
 
-        @see NodeStore
+        @see Database
     */
     StringPairArray nodeDatabase;
 
@@ -300,7 +300,7 @@ public:
 
         The format is the same as that for @ref nodeDatabase
 
-        @see NodeStore
+        @see Database
     */
     StringPairArray ephemeralNodeDatabase;
 

src/ripple_core/node/KeyvaDBBackendFactory.h

@@ -1,32 +0,0 @@
-//------------------------------------------------------------------------------
-/*
-    Copyright (c) 2011-2013, OpenCoin, Inc.
-*/
-//==============================================================================
-
-#ifndef RIPPLE_CORE_NODE_KEYVABACKENDFACTORY_H_INCLUDED
-#define RIPPLE_CORE_NODE_KEYVABACKENDFACTORY_H_INCLUDED
-
-/** Factory to produce KeyvaDB backends for the NodeStore.
-
-    @see NodeStore
-*/
-class KeyvaDBBackendFactory : public NodeStore::BackendFactory
-{
-private:
-    class Backend;
-
-    KeyvaDBBackendFactory ();
-    ~KeyvaDBBackendFactory ();
-
-public:
-    static KeyvaDBBackendFactory* getInstance ();
-
-    String getName () const;
-
-    NodeStore::Backend* createInstance (size_t keyBytes,
-                                        StringPairArray const& keyValues,
-                                        NodeStore::Scheduler& scheduler);
-};
-
-#endif

src/ripple_core/node/LevelDBBackendFactory.h

@@ -1,35 +0,0 @@
-//------------------------------------------------------------------------------
-/*
-    Copyright (c) 2011-2013, OpenCoin, Inc.
-*/
-//==============================================================================
-
-#ifndef RIPPLE_CORE_NODE_LEVELDBBACKENDFACTORY_H_INCLUDED
-#define RIPPLE_CORE_NODE_LEVELDBBACKENDFACTORY_H_INCLUDED
-
-/** Factory to produce LevelDB backends for the NodeStore.
-
-    @see NodeStore
-*/
-class LevelDBBackendFactory : public NodeStore::BackendFactory
-{
-private:
-    class Backend;
-
-    LevelDBBackendFactory ();
-    ~LevelDBBackendFactory ();
-
-public:
-    static LevelDBBackendFactory* getInstance ();
-
-    String getName () const;
-
-    NodeStore::Backend* createInstance (size_t keyBytes,
-                                        StringPairArray const& keyValues,
-                                        NodeStore::Scheduler& scheduler);
-
-private:
-    void* m_lruCache;
-};
-
-#endif

src/ripple_core/node/MdbBackendFactory.h

@@ -1,38 +0,0 @@
-//------------------------------------------------------------------------------
-/*
-    Copyright (c) 2011-2013, OpenCoin, Inc.
-*/
-//==============================================================================
-
-#ifndef RIPPLE_CORE_NODE_MDBBACKENDFACTORY_H_INCLUDED
-#define RIPPLE_CORE_NODE_MDBBACKENDFACTORY_H_INCLUDED
-
-#if RIPPLE_MDB_AVAILABLE
-
-/** Factory to produce a backend using MDB.
-
-    @note MDB is not currently available for Win32
-
-    @see NodeStore
-*/
-class MdbBackendFactory : public NodeStore::BackendFactory
-{
-private:
-    class Backend;
-
-    MdbBackendFactory ();
-    ~MdbBackendFactory ();
-
-public:
-    static MdbBackendFactory* getInstance ();
-
-    String getName () const;
-
-    NodeStore::Backend* createInstance (size_t keyBytes,
-                                        StringPairArray const& keyValues,
-                                        NodeStore::Scheduler& scheduler);
-};
-
-#endif
-
-#endif

src/ripple_core/node/MemoryBackendFactory.h

@@ -1,32 +0,0 @@
-//------------------------------------------------------------------------------
-/*
-    Copyright (c) 2011-2013, OpenCoin, Inc.
-*/
-//==============================================================================
-
-#ifndef RIPPLE_CORE_NODE_MEMORYBACKENDFACTORY_H_INCLUDED
-#define RIPPLE_CORE_NODE_MEMORYBACKENDFACTORY_H_INCLUDED
-
-/** Factory to produce a RAM based backend for the NodeStore.
-
-    @see NodeStore
-*/
-class MemoryBackendFactory : public NodeStore::BackendFactory
-{
-private:
-    class Backend;
-
-    MemoryBackendFactory ();
-    ~MemoryBackendFactory ();
-
-public:
-    static MemoryBackendFactory* getInstance ();
-
-    String getName () const;
-
-    NodeStore::Backend* createInstance (size_t keyBytes,
-                                        StringPairArray const& keyValues,
-                                        NodeStore::Scheduler& scheduler);
-};
-
-#endif

src/ripple_core/node/NodeStore.h

@@ -1,448 +0,0 @@
-//------------------------------------------------------------------------------
-/*
-    Copyright (c) 2011-2013, OpenCoin, Inc.
-*/
-//==============================================================================
-
-#ifndef RIPPLE_CORE_NODE_NODESTORE_H_INCLUDED
-#define RIPPLE_CORE_NODE_NODESTORE_H_INCLUDED
-
-/** Persistency layer for NodeObject
-
-    A Node is a ledger object which is uniquely identified by a key, which is
-    the 256-bit hash of the body of the node. The payload is a variable length
-    block of serialized data.
-
-    All ledger data is stored as node objects and as such, needs to be persisted
-    between launches. Furthermore, since the set of node objects will in
-    general be larger than the amount of available memory, purged node objects
-    which are later accessed must be retrieved from the node store.
-
-    @see NodeObject
-*/
-class NodeStore : public Service
-{
-protected:
-    NodeStore (char const* name, Service& parent);
-
-public:
-    enum
-    {
-        // This is only used to pre-allocate the array for
-        // batch objects and does not affect the amount written.
-        //
-        batchWritePreallocationSize = 128
-    };
-
-    typedef std::vector <NodeObject::Ptr> Batch;
-
-    typedef StringPairArray Parameters;
-
-    //--------------------------------------------------------------------------
-
-    /** Parsed key/value blob into NodeObject components.
-
-        This will extract the information required to construct a NodeObject. It
-        also does consistency checking and returns the result, so it is possible
-        to determine if the data is corrupted without throwing an exception. Not
-        all forms of corruption are detected so further analysis will be needed
-        to eliminate false negatives.
-
-        @note This defines the database format of a NodeObject!
-    */
-    class DecodedBlob
-    {
-    public:
-        /** Construct the decoded blob from raw data. */
-        DecodedBlob (void const* key, void const* value, int valueBytes);
-
-        /** Determine if the decoding was successful. */
-        bool wasOk () const noexcept { return m_success; }
-
-        /** Create a NodeObject from this data. */
-        NodeObject::Ptr createObject ();
-
-    private:
-        bool m_success;
-
-        void const* m_key;
-        LedgerIndex m_ledgerIndex;
-        NodeObjectType m_objectType;
-        unsigned char const* m_objectData;
-        int m_dataBytes;
-    };
-
-    //--------------------------------------------------------------------------
-
-    /** Utility for producing flattened node objects.
-
-        These get recycled to prevent many small allocations.
-
-        @note This defines the database format of a NodeObject!
-    */
-    struct EncodedBlob
-    {
-        typedef RecycledObjectPool <EncodedBlob> Pool;
-
-        void prepare (NodeObject::Ptr const& object);
-
-        void const* getKey () const noexcept { return m_key; }
-
-        size_t getSize () const noexcept { return m_size; }
-
-        void const* getData () const noexcept { return m_data.getData (); }
-
-    private:
-        void const* m_key;
-        MemoryBlock m_data;
-        size_t m_size;
-    };
-
-    //--------------------------------------------------------------------------
-
-    /** Provides optional asynchronous scheduling for backends.
-
-        For improved performance, a backend has the option of performing writes
-        in batches. These writes can be scheduled using the provided scheduler
-        object.
-
-        @see BatchWriter
-    */
-    class Scheduler
-    {
-    public:
-        /** Derived classes perform scheduled tasks. */
-        struct Task
-        {
-            virtual ~Task () { }
-
-            /** Performs the task.
-                The call may take place on a foreign thread.
-            */
-            virtual void performScheduledTask () = 0;
-        };
-
-        /** Schedules a task.
-
-            Depending on the implementation, this could happen
-            immediately or get deferred.
-        */
-        virtual void scheduleTask (Task& task) = 0;
-
-        /** Notifies the scheduler that all tasks are complete. */
-        virtual void scheduledTasksStopped () = 0;
-    };
-
-    //--------------------------------------------------------------------------
-
-    /** Helps with batch writing.
-
-        The batch writes are performed with a scheduled task. Use of the
-        class it not required. A backend can implement its own write batching,
-        or skip write batching if doing so yields a performance benefit.
-
-        @see Scheduler
-    */
-    // VFALCO NOTE I'm not entirely happy having placed this here,
-    //             because whoever needs to use NodeStore certainly doesn't
-    //             need to see the implementation details of BatchWriter.
-    //
-    class BatchWriter : private Scheduler::Task
-    {
-    public:
-        /** This callback does the actual writing. */
-        struct Callback
-        {
-            virtual void writeBatch (Batch const& batch) = 0;
-
-            // Called after stopAsync when there is no more pending write
-            virtual void writeStopped () = 0;
-        };
-
-        /** Create a batch writer. */
-        BatchWriter (Callback& callback, Scheduler& scheduler);
-
-        /** Destroy a batch writer.
-
-            Anything pending in the batch is written out before this returns.
-        */
-        ~BatchWriter ();
-
-        /** Store the object.
-
-            This will add to the batch and initiate a scheduled task to
-            write the batch out.
-        */
-        void store (NodeObject::Ptr const& object);
-
-        /** Get an estimate of the amount of writing I/O pending. */
-        int getWriteLoad ();
-
-        /** Called to notify that the NodeStore wants to stop. */
-        void stopAsync ();
-
-    private:
-        void performScheduledTask ();
-        void writeBatch ();
-        void waitForWriting ();
-
-    private:
-        typedef boost::recursive_mutex LockType;
-        typedef boost::condition_variable_any CondvarType;
-
-        Callback& m_callback;
-        Scheduler& m_scheduler;
-        LockType mWriteMutex;
-        CondvarType mWriteCondition;
-        int mWriteGeneration;
-        int mWriteLoad;
-        bool mWritePending;
-        Batch mWriteSet;
-    };
-
-    //--------------------------------------------------------------------------
-
-    /** A backend used for the store.
-
-        The NodeStore uses a swappable backend so that other database systems
-        can be tried. Different databases may offer various features such
-        as improved performance, fault tolerant or distributed storage, or
-        all in-memory operation.
-
-        A given instance of a backend is fixed to a particular key size.
-    */
-    class Backend
-    {
-    public:
-        /** Return codes from operations. */
-        enum Status
-        {
-            ok,
-            notFound,
-            dataCorrupt,
-            unknown
-        };
-
-        /** Destroy the backend.
-
-            All open files are closed and flushed. If there are batched writes
-            or other tasks scheduled, they will be completed before this call
-            returns.
-        */
-        virtual ~Backend () { }
-
-        /** Get the human-readable name of this backend.
-
-            This is used for diagnostic output.
-        */
-        virtual std::string getName() = 0;
-
-        /** Fetch a single object.
-
-            If the object is not found or an error is encountered, the
-            result will indicate the condition.
-
-            @note This will be called concurrently.
-
-            @param key A pointer to the key data.
-            @param pObject [out] The created object if successful.
-
-            @return The result of the operation.
-        */
-        virtual Status fetch (void const* key, NodeObject::Ptr* pObject) = 0;
-
-        /** Store a single object.
-
-            Depending on the implementation this may happen immediately
-            or deferred using a scheduled task.
-
-            @note This will be called concurrently.
-
-            @param object The object to store.
-        */
-        virtual void store (NodeObject::Ptr const& object) = 0;
-
-        /** Store a group of objects.
-            
-            @note This function will not be called concurrently with
-                  itself or @ref store.
-        */
-        virtual void storeBatch (Batch const& batch) = 0;
-
-        /** Callback for iterating through objects.
-
-            @see visitAll
-        */
-        struct VisitCallback
-        {
-            virtual void visitObject (NodeObject::Ptr const& object) = 0;
-        };
-
-        /** Visit every object in the database
-            
-            This is usually called during import.
-
-            @note This routine will not be called concurrently with itself
-                  or other methods.
-
-            @see import, VisitCallback
-        */
-        virtual void visitAll (VisitCallback& callback) = 0;
-
-        /** Estimate the number of write operations pending. */
-        virtual int getWriteLoad () = 0;
-
-        /** Called to notify the Backend that the NodeStore wants to stop. */
-        virtual void stopAsync () = 0;
-    };
-
-    //--------------------------------------------------------------------------
-
-    /** Factory to produce backends.
-    */
-    class BackendFactory
-    {
-    public:
-        virtual ~BackendFactory () { }
-
-        /** Retrieve the name of this factory. */
-        virtual String getName () const = 0;
-
-        /** Create an instance of this factory's backend.
-            
-            @param keyBytes The fixed number of bytes per key.
-            @param keyValues A set of key/value configuration pairs.
-            @param scheduler The scheduler to use for running tasks.
-
-            @return A pointer to the Backend object.
-        */
-        virtual Backend* createInstance (size_t keyBytes,
-                                         Parameters const& parameters,
-                                         Scheduler& scheduler) = 0;
-    };
-
-    //--------------------------------------------------------------------------
-
-    /** Construct a node store.
-
-        The parameters are key value pairs passed to the backend. The
-        'type' key must exist, it defines the choice of backend. Most
-        backends also require a 'path' field.
-        
-        Some choices for 'type' are:
-            HyperLevelDB, LevelDB, SQLite, KeyvaDB, MDB
-
-        If the fastBackendParameter is omitted or empty, no ephemeral database
-        is used. If the scheduler parameter is omited or unspecified, a
-        synchronous scheduler is used which performs all tasks immediately on
-        the caller's thread.
-
-        @note If the database cannot be opened or created, an exception is thrown.
-
-        @param backendParameters The parameter string for the persistent backend.
-        @param fastBackendParameters [optional] The parameter string for the ephemeral backend.                        
-        @param scheduler [optional The scheduler to use for performing asynchronous tasks.
-
-        @return The opened database.
-    */
-    static NodeStore* New (char const* name,
-                           Service& parent,
-                           Parameters const& backendParameters,
-                           Parameters fastBackendParameters = Parameters (),
-                           Scheduler& scheduler = getSynchronousScheduler ());
-
-    /** Get the synchronous scheduler.
-
-        The synchronous scheduler performs all tasks immediately, before
-        returning to the caller, using the caller's thread.
-    */
-    static Scheduler& getSynchronousScheduler ();
-
-    /** Destroy the node store.
-
-        All pending operations are completed, pending writes flushed,
-        and files closed before this returns.
-    */
-    virtual ~NodeStore () { }
-
-    /** Retrieve the name associated with this backend.
-
-        This is used for diagnostics and may not reflect the actual path
-        or paths used by the underlying backend.
-    */
-    virtual String getName () const = 0;
-
-    /** Add the specified backend factory to the list of available factories.
-        The names of available factories are compared against the "type"
-        value in the parameter list on construction. Ownership of the object
-        is transferred. The object must be allocated using new.
-        @param factory The factory to add.
-    */
-    static void addBackendFactory (BackendFactory* factory);
-
-    /** Fetch an object.
-
-        If the object is known to be not in the database, isn't found in the
-        database during the fetch, or failed to load correctly during the fetch,
-        `nullptr` is returned.
-
-        @note This can be called concurrently.
-
-        @param hash The key of the object to retrieve.  
-
-        @return The object, or nullptr if it couldn't be retrieved.
-    */
-    virtual NodeObject::pointer fetch (uint256 const& hash) = 0;
-
-    /** Store the object.
-
-        The caller's Blob parameter is overwritten.
-
-        @param type The type of object.
-        @param ledgerIndex The ledger in which the object appears.
-        @param data The payload of the object. The caller's
-                    variable is overwritten.
-        @param hash The 256-bit hash of the payload data.
-
-        @return `true` if the object was stored?              
-    */
-    virtual void store (NodeObjectType type,
-                        uint32 ledgerIndex,
-                        Blob& data,
-                        uint256 const& hash) = 0;
-
-    /** Visit every object in the database
-            
-        This is usually called during import.
-
-        @note This routine will not be called concurrently with itself
-                or other methods.
-
-        @see import
-    */
-    virtual void visitAll (Backend::VisitCallback& callback) = 0;
-
-    /** Import objects from another database. */
-    virtual void import (NodeStore& sourceDatabase) = 0;
-
-    /** Retrieve the estimated number of pending write operations.
-        This is used for diagnostics.
-    */
-    virtual int getWriteLoad () = 0;
-
-    // VFALCO TODO Document this.
-    virtual float getCacheHitRate () = 0;
-
-    // VFALCO TODO Document this.
-    //        TODO Document the parameter meanings.
-    virtual void tune (int size, int age) = 0;
-
-    // VFALCO TODO Document this.
-    virtual void sweep () = 0;
-
-    /** Add the known Backend factories to the singleton.
-    */
-    static void addAvailableBackends ();
-};
-
-#endif

src/ripple_core/node/NullBackendFactory.cpp

@@ -1,84 +0,0 @@
-//------------------------------------------------------------------------------
-/*
-    Copyright (c) 2011-2013, OpenCoin, Inc.
-*/
-//==============================================================================
-
-class NullBackendFactory::Backend : public NodeStore::Backend
-{
-public:
-    explicit Backend (NodeStore::Scheduler& scheduler)
-        : m_scheduler (scheduler)
-    {
-    }
-
-    ~Backend ()
-    {
-    }
-
-    std::string getName()
-    {
-        return std::string ();
-    }
-
-    Status fetch (void const*, NodeObject::Ptr*)
-    {
-        return notFound;
-    }
-    
-    void store (NodeObject::ref object)
-    {
-    }
-    
-    void storeBatch (NodeStore::Batch const& batch)
-    {
-    }
-
-    void visitAll (VisitCallback& callback)
-    {
-    }
-
-    int getWriteLoad ()
-    {
-        return 0;
-    }
-
-    void stopAsync ()
-    {
-        m_scheduler.scheduledTasksStopped ();
-    }
-
-private:
-    NodeStore::Scheduler& m_scheduler;
-};
-
-//------------------------------------------------------------------------------
-
-NullBackendFactory::NullBackendFactory ()
-{
-}
-
-NullBackendFactory::~NullBackendFactory ()
-{
-}
-
-NullBackendFactory* NullBackendFactory::getInstance ()
-{
-    return new NullBackendFactory;
-}
-
-String NullBackendFactory::getName () const
-{
-    return "none";
-}
-
-NodeStore::Backend* NullBackendFactory::createInstance (
-    size_t,
-    StringPairArray const&,
-    NodeStore::Scheduler& scheduler)
-{
-    return new NullBackendFactory::Backend (scheduler);
-}
-
-//------------------------------------------------------------------------------
-

src/ripple_core/node/NullBackendFactory.h

@@ -1,34 +0,0 @@
-//------------------------------------------------------------------------------
-/*
-    Copyright (c) 2011-2013, OpenCoin, Inc.
-*/
-//==============================================================================
-
-#ifndef RIPPLE_CORE_NODE_NULLBACKENDFACTORY_H_INCLUDED
-#define RIPPLE_CORE_NODE_NULLBACKENDFACTORY_H_INCLUDED
-
-/** Factory to produce a null backend.
-
-    This is for standalone / testing mode.
-
-    @see NodeStore
-*/
-class NullBackendFactory : public NodeStore::BackendFactory
-{
-private:
-    class Backend;
-
-    NullBackendFactory ();
-    ~NullBackendFactory ();
-
-public:
-    static NullBackendFactory* getInstance ();
-
-    String getName () const;
-
-    NodeStore::Backend* createInstance (size_t keyBytes,
-                                        StringPairArray const& keyValues,
-                                        NodeStore::Scheduler& scheduler);
-};
-
-#endif

src/ripple_core/node/SophiaBackendFactory.h

@@ -1,32 +0,0 @@
-//------------------------------------------------------------------------------
-/*
-    Copyright (c) 2011-2013, OpenCoin, Inc.
-*/
-//==============================================================================
-
-#ifndef RIPPLE_CORE_SOPHIABACKENDFACTORY_H_INCLUDED
-#define RIPPLE_CORE_SOPHIABACKENDFACTORY_H_INCLUDED
-
-/** Factory to produce Sophia backends for the NodeStore.
-
-    @see NodeStore
-*/
-class SophiaBackendFactory : public NodeStore::BackendFactory
-{
-private:
-    class Backend;
-
-    SophiaBackendFactory ();
-    ~SophiaBackendFactory ();
-
-public:
-    static SophiaBackendFactory* getInstance ();
-
-    String getName () const;
-
-    NodeStore::Backend* createInstance (size_t keyBytes,
-                                        StringPairArray const& keyValues,
-                                        NodeStore::Scheduler& scheduler);
-};
-
-#endif

src/ripple_core/nodestore/NodeStore.cpp

@@ -0,0 +1,49 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+// backend support
+#include "beast/modules/beast_db/beast_db.h"
+#include "../ripple_hyperleveldb/ripple_hyperleveldb.h"
+#include "../ripple_leveldb/ripple_leveldb.h"
+#include "../ripple_mdb/ripple_mdb.h"
+#include "../ripple/sophia/ripple_sophia.h"
+
+namespace ripple
+{
+
+#  include "impl/DecodedBlob.h"
+#  include "impl/EncodedBlob.h"
+#  include "impl/BatchWriter.h"
+# include "backend/HyperDBFactory.h"
+#include "backend/HyperDBFactory.cpp"
+# include "backend/KeyvaDBFactory.h"
+#include "backend/KeyvaDBFactory.cpp"
+# include "backend/LevelDBFactory.h"
+#include "backend/LevelDBFactory.cpp"
+# include "backend/MemoryFactory.h"
+#include "backend/MemoryFactory.cpp"
+# include "backend/NullFactory.h"
+#include "backend/NullFactory.cpp"
+# include "backend/MdbFactory.h"
+#include "backend/MdbFactory.cpp"
+# include "backend/SophiaFactory.h"
+#include "backend/SophiaFactory.cpp"
+
+#include "impl/BatchWriter.cpp"
+# include "impl/Factories.h"
+# include "impl/DatabaseImp.h"
+#include "impl/DummyScheduler.cpp"
+#include "impl/DecodedBlob.cpp"
+#include "impl/EncodedBlob.cpp"
+#include "impl/NodeObject.cpp"
+
+# include "tests/TestBase.h"
+#include "tests/BackendTests.cpp"
+#include "tests/BasicTests.cpp"
+#include "tests/DatabaseTests.cpp"
+#include "tests/TimingTests.cpp"
+
+}

src/ripple_core/nodestore/NodeStore.h

@@ -0,0 +1,25 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_NODESTORE_H_INCLUDED
+#define RIPPLE_NODESTORE_H_INCLUDED
+
+namespace ripple
+{
+
+#include "api/NodeObject.h"
+#include "api/Types.h"
+#include "api/VisitCallback.h"
+#include "api/Backend.h"
+#  include "api/Task.h"
+# include "api/Scheduler.h"
+#include "api/DummyScheduler.h"
+#include "api/Factory.h"
+#include "api/Database.h"
+
+}
+
+#endif

src/ripple_core/nodestore/api/Backend.h

@@ -0,0 +1,88 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_NODESTORE_BACKEND_H_INCLUDED
+#define RIPPLE_NODESTORE_BACKEND_H_INCLUDED
+
+namespace NodeStore
+{
+
+/** A backend used for the store.
+
+    The NodeStore uses a swappable backend so that other database systems
+    can be tried. Different databases may offer various features such
+    as improved performance, fault tolerant or distributed storage, or
+    all in-memory operation.
+
+    A given instance of a backend is fixed to a particular key size.
+*/
+class Backend
+{
+public:
+    /** Destroy the backend.
+
+        All open files are closed and flushed. If there are batched writes
+        or other tasks scheduled, they will be completed before this call
+        returns.
+    */
+    virtual ~Backend () { }
+
+    /** Get the human-readable name of this backend.
+
+        This is used for diagnostic output.
+    */
+    virtual std::string getName() = 0;
+
+    /** Fetch a single object.
+
+        If the object is not found or an error is encountered, the
+        result will indicate the condition.
+
+        @note This will be called concurrently.
+
+        @param key A pointer to the key data.
+        @param pObject [out] The created object if successful.
+
+        @return The result of the operation.
+    */
+    virtual Status fetch (void const* key, NodeObject::Ptr* pObject) = 0;
+
+    /** Store a single object.
+
+        Depending on the implementation this may happen immediately
+        or deferred using a scheduled task.
+
+        @note This will be called concurrently.
+
+        @param object The object to store.
+    */
+    virtual void store (NodeObject::Ptr const& object) = 0;
+
+    /** Store a group of objects.
+            
+        @note This function will not be called concurrently with
+                itself or @ref store.
+    */
+    virtual void storeBatch (Batch const& batch) = 0;
+
+    /** Visit every object in the database
+            
+        This is usually called during import.
+
+        @note This routine will not be called concurrently with itself
+                or other methods.
+
+        @see import, VisitCallback
+    */
+    virtual void visitAll (VisitCallback& callback) = 0;
+
+    /** Estimate the number of write operations pending. */
+    virtual int getWriteLoad () = 0;
+};
+
+}
+
+#endif

src/ripple_core/nodestore/api/Database.h

@@ -0,0 +1,139 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_NODESTORE_DATABASE_H_INCLUDED
+#define RIPPLE_NODESTORE_DATABASE_H_INCLUDED
+
+namespace NodeStore
+{
+
+/** Persistency layer for NodeObject
+
+    A Node is a ledger object which is uniquely identified by a key, which is
+    the 256-bit hash of the body of the node. The payload is a variable length
+    block of serialized data.
+
+    All ledger data is stored as node objects and as such, needs to be persisted
+    between launches. Furthermore, since the set of node objects will in
+    general be larger than the amount of available memory, purged node objects
+    which are later accessed must be retrieved from the node store.
+
+    @see NodeObject
+*/
+class Database
+{
+public:
+    /** Construct a node store database.
+
+        The parameters are key value pairs passed to the backend. The
+        'type' key must exist, it defines the choice of backend. Most
+        backends also require a 'path' field.
+        
+        Some choices for 'type' are:
+            HyperLevelDB, LevelDBFactory, SQLite, KeyvaDB, MDB
+
+        If the fastBackendParameter is omitted or empty, no ephemeral database
+        is used. If the scheduler parameter is omited or unspecified, a
+        synchronous scheduler is used which performs all tasks immediately on
+        the caller's thread.
+
+        @note If the database cannot be opened or created, an exception is thrown.
+
+        @param name A diagnostic label for the database.
+        @param scheduler The scheduler to use for performing asynchronous tasks.
+        @param backendParameters The parameter string for the persistent backend.
+        @param fastBackendParameters [optional] The parameter string for the ephemeral backend.                        
+
+        @return The opened database.
+    */
+    static Database* New (char const* name,
+                          Scheduler& scheduler,
+                          Parameters const& backendParameters,
+                          Parameters fastBackendParameters = Parameters ());
+
+    /** Destroy the node store.
+        All pending operations are completed, pending writes flushed,
+        and files closed before this returns.
+    */
+    virtual ~Database () { }
+
+    /** Retrieve the name associated with this backend.
+        This is used for diagnostics and may not reflect the actual path
+        or paths used by the underlying backend.
+    */
+    virtual String getName () const = 0;
+
+    /** Add the specified backend factory to the list of available factories.
+        The names of available factories are compared against the "type"
+        value in the parameter list on construction. Ownership of the object
+        is transferred. The object must be allocated using new.
+        @param factory The factory to add.
+    */
+    static void addFactory (Factory* factory);
+
+    /** Fetch an object.
+        If the object is known to be not in the database, isn't found in the
+        database during the fetch, or failed to load correctly during the fetch,
+        `nullptr` is returned.
+
+        @note This can be called concurrently.
+        @param hash The key of the object to retrieve.  
+        @return The object, or nullptr if it couldn't be retrieved.
+    */
+    virtual NodeObject::pointer fetch (uint256 const& hash) = 0;
+
+    /** Store the object.
+
+        The caller's Blob parameter is overwritten.
+
+        @param type The type of object.
+        @param ledgerIndex The ledger in which the object appears.
+        @param data The payload of the object. The caller's
+                    variable is overwritten.
+        @param hash The 256-bit hash of the payload data.
+
+        @return `true` if the object was stored?              
+    */
+    virtual void store (NodeObjectType type,
+                        uint32 ledgerIndex,
+                        Blob& data,
+                        uint256 const& hash) = 0;
+
+    /** Visit every object in the database
+        This is usually called during import.
+
+        @note This routine will not be called concurrently with itself
+                or other methods.
+        @see import
+    */
+    virtual void visitAll (VisitCallback& callback) = 0;
+
+    /** Import objects from another database. */
+    virtual void import (Database& sourceDatabase) = 0;
+
+    /** Retrieve the estimated number of pending write operations.
+        This is used for diagnostics.
+    */
+    virtual int getWriteLoad () = 0;
+
+    // VFALCO TODO Document this.
+    virtual float getCacheHitRate () = 0;
+
+    // VFALCO TODO Document this.
+    //        TODO Document the parameter meanings.
+    virtual void tune (int size, int age) = 0;
+
+    // VFALCO TODO Document this.
+    virtual void sweep () = 0;
+
+    /** Add the known Backend factories to the singleton.
+    */
+    static void addAvailableBackends ();
+};
+
+}
+
+#endif

src/ripple_core/nodestore/api/DummyScheduler.h

@@ -0,0 +1,28 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_NODESTORE_DUMMYSCHEDULER_H_INCLUDED
+#define RIPPLE_NODESTORE_DUMMYSCHEDULER_H_INCLUDED
+
+namespace NodeStore
+{
+
+// Simple Scheduler that just peforms the tasks synchronously.
+class DummyScheduler : public Scheduler
+{
+public:
+    DummyScheduler ();
+
+    ~DummyScheduler ();
+
+    void scheduleTask (Task& task);
+
+    void scheduledTasksStopped ();
+};
+
+}
+
+#endif

src/ripple_core/nodestore/api/Factory.h

@@ -0,0 +1,37 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_NODESTORE_FACTORY_H_INCLUDED
+#define RIPPLE_NODESTORE_FACTORY_H_INCLUDED
+
+namespace NodeStore
+{
+
+/** Factory to produce backends. */
+class Factory
+{
+public:
+    virtual ~Factory () { }
+
+    /** Retrieve the name of this factory. */
+    virtual String getName () const = 0;
+
+    /** Create an instance of this factory's backend.
+            
+        @param keyBytes The fixed number of bytes per key.
+        @param keyValues A set of key/value configuration pairs.
+        @param scheduler The scheduler to use for running tasks.
+
+        @return A pointer to the Backend object.
+    */
+    virtual Backend* createInstance (size_t keyBytes,
+                                     Parameters const& parameters,
+                                     Scheduler& scheduler) = 0;
+};
+
+}
+
+#endif

src/ripple_core/nodestore/api/NodeObject.h

@@ -4,11 +4,14 @@
 */
 //==============================================================================
 
-#ifndef RIPPLE_CORE_NODE_NODEOBJECT_H_INCLUDED
+#ifndef RIPPLE_NODESTORE_NODEOBJECT_H_INCLUDED
-#define RIPPLE_CORE_NODE_NODEOBJECT_H_INCLUDED
+#define RIPPLE_NODESTORE_NODEOBJECT_H_INCLUDED
 
-/** The types of node objects.
+// VFALCO NOTE Intentionally not in the NodeStore namespace
-*/
+
+
+
+/** The types of node objects. */
 enum NodeObjectType
 {
     hotUNKNOWN = 0,

src/ripple_core/nodestore/api/Scheduler.h

@@ -0,0 +1,36 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_NODESTORE_SCHEDULER_H_INCLUDED
+#define RIPPLE_NODESTORE_SCHEDULER_H_INCLUDED
+
+namespace NodeStore
+{
+
+/** Scheduling for asynchronous backend activity
+    
+    For improved performance, a backend has the option of performing writes
+    in batches. These writes can be scheduled using the provided scheduler
+    object.
+
+    @see BatchWriter
+*/
+class Scheduler
+{
+public:
+    virtual ~Scheduler() { }
+
+    /** Schedules a task.
+        Depending on the implementation, the task may be invoked either on
+        the current thread of execution, or an unspecified implementation-defined
+        foreign thread.
+    */
+    virtual void scheduleTask (Task& task) = 0;
+};
+
+}
+
+#endif

src/ripple_core/nodestore/api/Task.h

@@ -0,0 +1,26 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_NODESTORE_TASK_H_INCLUDED
+#define RIPPLE_NODESTORE_TASK_H_INCLUDED
+
+namespace NodeStore
+{
+
+/** Derived classes perform scheduled tasks. */
+struct Task
+{
+    virtual ~Task () { }
+
+    /** Performs the task.
+        The call may take place on a foreign thread.
+    */
+    virtual void performScheduledTask () = 0;
+};
+
+}
+
+#endif

src/ripple_core/nodestore/api/Types.h

@@ -0,0 +1,38 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_NODESTORE_TYPES_H_INCLUDED
+#define RIPPLE_NODESTORE_TYPES_H_INCLUDED
+
+namespace NodeStore
+{
+
+enum
+{
+    // This is only used to pre-allocate the array for
+    // batch objects and does not affect the amount written.
+    //
+    batchWritePreallocationSize = 128
+};
+
+/** Return codes from Backend operations. */
+enum Status
+{
+    ok,
+    notFound,
+    dataCorrupt,
+    unknown
+};
+
+/** A batch of NodeObjects to write at once. */
+typedef std::vector <NodeObject::Ptr> Batch;
+
+/** A list of key/value parameter pairs passed to the backend. */
+typedef StringPairArray Parameters;
+
+}
+
+#endif

src/ripple_core/nodestore/api/VisitCallback.h

@@ -0,0 +1,24 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_NODESTORE_VISITCALLBACK_H_INCLUDED
+#define RIPPLE_NODESTORE_VISITCALLBACK_H_INCLUDED
+
+namespace NodeStore
+{
+
+/** Callback for iterating through objects.
+
+    @see visitAll
+*/
+struct VisitCallback
+{
+    virtual void visitObject (NodeObject::Ptr const& object) = 0;
+};
+
+}
+
+#endif

src/ripple_core/nodestore/backend/HyperDBFactory.cpp

@@ -6,24 +6,27 @@
 
 #if RIPPLE_HYPERLEVELDB_AVAILABLE
 
-class HyperLevelDBBackendFactory::Backend
+namespace NodeStore
-    : public NodeStore::Backend
+{
-    , public NodeStore::BatchWriter::Callback
+
-    , LeakChecked <HyperLevelDBBackendFactory::Backend>
+class HyperDBFactory::BackendImp
+    : public Backend
+    , public BatchWriter::Callback
+    , public LeakChecked <HyperDBFactory::BackendImp>
 {
 public:
     typedef RecycledObjectPool <std::string> StringPool;
 
-    Backend (size_t keyBytes,
+    BackendImp (size_t keyBytes,
-             StringPairArray const& keyValues,
+             Parameters const& keyValues,
-             NodeStore::Scheduler& scheduler)
+             Scheduler& scheduler)
         : m_keyBytes (keyBytes)
         , m_scheduler (scheduler)
         , m_batch (*this, scheduler)
         , m_name (keyValues ["path"].toStdString ())
     {
         if (m_name.empty ())
-            Throw (std::runtime_error ("Missing path in LevelDB backend"));
+            Throw (std::runtime_error ("Missing path in LevelDBFactory backend"));
 
         hyperleveldb::Options options;
         options.create_if_missing = true;
@@ -60,7 +63,7 @@ public:
         m_db = db;
     }
 
-    ~Backend ()
+    ~BackendImp ()
     {
     }
 
@@ -91,7 +94,7 @@ public:
 
             if (getStatus.ok ())
             {
-                NodeStore::DecodedBlob decoded (key, string.data (), string.size ());
+                DecodedBlob decoded (key, string.data (), string.size ());
 
                 if (decoded.wasOk ())
                 {
@@ -129,12 +132,12 @@ public:
         m_batch.store (object);
     }
 
-    void storeBatch (NodeStore::Batch const& batch)
+    void storeBatch (Batch const& batch)
     {
         hyperleveldb::WriteBatch wb;
 
         {
-            NodeStore::EncodedBlob::Pool::ScopedItem item (m_blobPool);
+            EncodedBlob::Pool::ScopedItem item (m_blobPool);
 
             BOOST_FOREACH (NodeObject::ref object, batch)
             {
@@ -163,7 +166,7 @@ public:
         {
             if (it->key ().size () == m_keyBytes)
             {
-                NodeStore::DecodedBlob decoded (it->key ().data (),
+                DecodedBlob decoded (it->key ().data (),
                                                 it->value ().data (),
                                                 it->value ().size ());
 
@@ -193,61 +196,51 @@ public:
         return m_batch.getWriteLoad ();
     }
 
-    void stopAsync ()
-    {
-        m_batch.stopAsync();
-    }
-
     //--------------------------------------------------------------------------
 
-    void writeBatch (NodeStore::Batch const& batch)
+    void writeBatch (Batch const& batch)
     {
         storeBatch (batch);
     }
 
-    void writeStopped ()
-    {
-        m_scheduler.scheduledTasksStopped ();
-    }
-
 private:
     size_t const m_keyBytes;
-    NodeStore::Scheduler& m_scheduler;
+    Scheduler& m_scheduler;
-    NodeStore::BatchWriter m_batch;
+    BatchWriter m_batch;
     StringPool m_stringPool;
-    NodeStore::EncodedBlob::Pool m_blobPool;
+    EncodedBlob::Pool m_blobPool;
     std::string m_name;
     ScopedPointer <hyperleveldb::DB> m_db;
 };
 
 //------------------------------------------------------------------------------
 
-HyperLevelDBBackendFactory::HyperLevelDBBackendFactory ()
+HyperDBFactory::HyperDBFactory ()
 {
 }
 
-HyperLevelDBBackendFactory::~HyperLevelDBBackendFactory ()
+HyperDBFactory::~HyperDBFactory ()
 {
 }
 
-HyperLevelDBBackendFactory* HyperLevelDBBackendFactory::getInstance ()
+HyperDBFactory* HyperDBFactory::getInstance ()
 {
-    return new HyperLevelDBBackendFactory;
+    return new HyperDBFactory;
 }
 
-String HyperLevelDBBackendFactory::getName () const
+String HyperDBFactory::getName () const
 {
     return "HyperLevelDB";
 }
 
-NodeStore::Backend* HyperLevelDBBackendFactory::createInstance (
+Backend* HyperDBFactory::createInstance (
     size_t keyBytes,
-    StringPairArray const& keyValues,
+    Parameters const& keyValues,
-    NodeStore::Scheduler& scheduler)
+    Scheduler& scheduler)
 {
-    return new HyperLevelDBBackendFactory::Backend (keyBytes, keyValues, scheduler);
+    return new HyperDBFactory::BackendImp (keyBytes, keyValues, scheduler);
 }
 
-//------------------------------------------------------------------------------
+}
 
 #endif

src/ripple_core/nodestore/backend/HyperDBFactory.h

@@ -4,33 +4,38 @@
 */
 //==============================================================================
 
-#ifndef RIPPLE_CORE_NODE_HYPERLEVELDBBACKENDFACTORY_H_INCLUDED
+#ifndef RIPPLE_NODESTORE_HYPERDBFACTORY_H_INCLUDED
-#define RIPPLE_CORE_NODE_HYPERLEVELDBBACKENDFACTORY_H_INCLUDED
+#define RIPPLE_NODESTORE_HYPERDBFACTORY_H_INCLUDED
 
 #if RIPPLE_HYPERLEVELDB_AVAILABLE
 
+namespace NodeStore
+{
+
 /** Factory to produce HyperLevelDB backends for the NodeStore.
 
-    @see NodeStore
+    @see Database
 */
-class HyperLevelDBBackendFactory : public NodeStore::BackendFactory
+class HyperDBFactory : public NodeStore::Factory
 {
 private:
-    class Backend;
+    HyperDBFactory ();
-
+    ~HyperDBFactory ();
-    HyperLevelDBBackendFactory ();
-    ~HyperLevelDBBackendFactory ();
 
 public:
-    static HyperLevelDBBackendFactory* getInstance ();
+    class BackendImp;
+
+    static HyperDBFactory* getInstance ();
 
     String getName () const;
 
     NodeStore::Backend* createInstance (size_t keyBytes,
-                                        StringPairArray const& keyValues,
+                                        Parameters const& keyValues,
                                         NodeStore::Scheduler& scheduler);
 };
 
+}
+
 #endif
 
 #endif

src/ripple_core/nodestore/backend/KeyvaDBFactory.cpp

@@ -4,16 +4,19 @@
 */
 //==============================================================================
 
-class KeyvaDBBackendFactory::Backend : public NodeStore::Backend
+namespace NodeStore
+{
+
+class KeyvaDBFactory::BackendImp : public Backend
 {
 private:
     typedef RecycledObjectPool <MemoryBlock> MemoryPool;
-    typedef RecycledObjectPool <NodeStore::EncodedBlob> EncodedBlobPool;
+    typedef RecycledObjectPool <EncodedBlob> EncodedBlobPool;
 
 public:
-    Backend (size_t keyBytes,
+    BackendImp (size_t keyBytes,
-             StringPairArray const& keyValues,
+             Parameters const& keyValues,
-             NodeStore::Scheduler& scheduler)
+             Scheduler& scheduler)
         : m_keyBytes (keyBytes)
         , m_scheduler (scheduler)
         , m_path (keyValues ["path"])
@@ -25,7 +28,7 @@ public:
     {
     }
 
-    ~Backend ()
+    ~BackendImp ()
     {
     }
 
@@ -83,7 +86,7 @@ public:
 
         if (found)
         {
-            NodeStore::DecodedBlob decoded (key, cb.getData (), cb.getSize ());
+            DecodedBlob decoded (key, cb.getData (), cb.getSize ());
 
             if (decoded.wasOk ())
             {
@@ -107,14 +110,14 @@ public:
     void store (NodeObject::ref object)
     {
         EncodedBlobPool::ScopedItem item (m_blobPool);
-        NodeStore::EncodedBlob& encoded (item.getObject ());
+        EncodedBlob& encoded (item.getObject ());
 
         encoded.prepare (object);
 
         m_db->put (encoded.getKey (), encoded.getData (), encoded.getSize ());
     }
 
-    void storeBatch (NodeStore::Batch const& batch)
+    void storeBatch (Batch const& batch)
     {
         for (std::size_t i = 0; i < batch.size (); ++i)
             store (batch [i]);
@@ -134,16 +137,11 @@ public:
         return 0;
     }
 
-    void stopAsync ()
-    {
-        m_scheduler.scheduledTasksStopped ();
-    }
-
     //--------------------------------------------------------------------------
 
 private:
     size_t const m_keyBytes;
-    NodeStore::Scheduler& m_scheduler;
+    Scheduler& m_scheduler;
     String m_path;
     ScopedPointer <KeyvaDB> m_db;
     MemoryPool m_memoryPool;
@@ -152,31 +150,30 @@ private:
 
 //------------------------------------------------------------------------------
 
-KeyvaDBBackendFactory::KeyvaDBBackendFactory ()
+KeyvaDBFactory::KeyvaDBFactory ()
 {
 }
 
-KeyvaDBBackendFactory::~KeyvaDBBackendFactory ()
+KeyvaDBFactory::~KeyvaDBFactory ()
 {
 }
 
-KeyvaDBBackendFactory* KeyvaDBBackendFactory::getInstance ()
+KeyvaDBFactory* KeyvaDBFactory::getInstance ()
 {
-    return new KeyvaDBBackendFactory;
+    return new KeyvaDBFactory;
 }
 
-String KeyvaDBBackendFactory::getName () const
+String KeyvaDBFactory::getName () const
 {
     return "KeyvaDB";
 }
 
-NodeStore::Backend* KeyvaDBBackendFactory::createInstance (
+Backend* KeyvaDBFactory::createInstance (
     size_t keyBytes,
-    StringPairArray const& keyValues,
+    Parameters const& keyValues,
-    NodeStore::Scheduler& scheduler)
+    Scheduler& scheduler)
 {
-    return new KeyvaDBBackendFactory::Backend (keyBytes, keyValues, scheduler);
+    return new KeyvaDBFactory::BackendImp (keyBytes, keyValues, scheduler);
 }
 
-//------------------------------------------------------------------------------
+}
-

src/ripple_core/nodestore/backend/KeyvaDBFactory.h

@@ -0,0 +1,37 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_NODESTORE_KEYVADBFACTORY_H_INCLUDED
+#define RIPPLE_NODESTORE_KEYVADBFACTORY_H_INCLUDED
+
+namespace NodeStore
+{
+
+/** Factory to produce KeyvaDB backends for the NodeStore.
+
+    @see Database
+*/
+class KeyvaDBFactory : public Factory
+{
+private:
+    KeyvaDBFactory ();
+    ~KeyvaDBFactory ();
+
+public:
+    class BackendImp;
+
+    static KeyvaDBFactory* getInstance ();
+
+    String getName () const;
+
+    Backend* createInstance (size_t keyBytes,
+                             Parameters const& keyValues,
+                             Scheduler& scheduler);
+};
+
+}
+
+#endif

src/ripple_core/nodestore/backend/LevelDBFactory.cpp

@@ -4,26 +4,29 @@
 */
 //==============================================================================
 
-class LevelDBBackendFactory::Backend
+namespace NodeStore
-    : public NodeStore::Backend
+{
-    , public NodeStore::BatchWriter::Callback
+
-    , LeakChecked <LevelDBBackendFactory::Backend>
+class LevelDBFactory::BackendImp
+    : public Backend
+    , public BatchWriter::Callback
+    , public LeakChecked <LevelDBFactory::BackendImp>
 {
 public:
     typedef RecycledObjectPool <std::string> StringPool;
 
     //--------------------------------------------------------------------------
 
-    Backend (int keyBytes,
+    BackendImp (int keyBytes,
-             StringPairArray const& keyValues,
+             Parameters const& keyValues,
-             NodeStore::Scheduler& scheduler)
+             Scheduler& scheduler)
         : m_keyBytes (keyBytes)
         , m_scheduler (scheduler)
         , m_batch (*this, scheduler)
         , m_name (keyValues ["path"].toStdString ())
     {
         if (m_name.empty())
-            Throw (std::runtime_error ("Missing path in LevelDB backend"));
+            Throw (std::runtime_error ("Missing path in LevelDBFactory backend"));
 
         leveldb::Options options;
         options.create_if_missing = true;
@@ -60,7 +63,7 @@ public:
         m_db = db;
     }
 
-    ~Backend ()
+    ~BackendImp ()
     {
     }
 
@@ -91,7 +94,7 @@ public:
 
             if (getStatus.ok ())
             {
-                NodeStore::DecodedBlob decoded (key, string.data (), string.size ());
+                DecodedBlob decoded (key, string.data (), string.size ());
 
                 if (decoded.wasOk ())
                 {
@@ -129,12 +132,12 @@ public:
         m_batch.store (object);
     }
 
-    void storeBatch (NodeStore::Batch const& batch)
+    void storeBatch (Batch const& batch)
     {
         leveldb::WriteBatch wb;
 
         {
-            NodeStore::EncodedBlob::Pool::ScopedItem item (m_blobPool);
+            EncodedBlob::Pool::ScopedItem item (m_blobPool);
 
             BOOST_FOREACH (NodeObject::ref object, batch)
             {
@@ -163,7 +166,7 @@ public:
         {
             if (it->key ().size () == m_keyBytes)
             {
-                NodeStore::DecodedBlob decoded (it->key ().data (),
+                DecodedBlob decoded (it->key ().data (),
                                                 it->value ().data (),
                                                 it->value ().size ());
 
@@ -193,36 +196,26 @@ public:
         return m_batch.getWriteLoad ();
     }
 
-    void stopAsync ()
-    {
-        m_batch.stopAsync ();
-    }
-
     //--------------------------------------------------------------------------
 
-    void writeBatch (NodeStore::Batch const& batch)
+    void writeBatch (Batch const& batch)
     {
         storeBatch (batch);
     }
 
-    void writeStopped ()
-    {
-        m_scheduler.scheduledTasksStopped ();
-    }
-
 private:
     size_t const m_keyBytes;
-    NodeStore::Scheduler& m_scheduler;
+    Scheduler& m_scheduler;
-    NodeStore::BatchWriter m_batch;
+    BatchWriter m_batch;
     StringPool m_stringPool;
-    NodeStore::EncodedBlob::Pool m_blobPool;
+    EncodedBlob::Pool m_blobPool;
     std::string m_name;
     ScopedPointer <leveldb::DB> m_db;
 };
 
 //------------------------------------------------------------------------------
 
-LevelDBBackendFactory::LevelDBBackendFactory ()
+LevelDBFactory::LevelDBFactory ()
     : m_lruCache (nullptr)
 {
     leveldb::Options options;
@@ -233,29 +226,28 @@ LevelDBBackendFactory::LevelDBBackendFactory ()
     m_lruCache = options.block_cache;
 }
 
-LevelDBBackendFactory::~LevelDBBackendFactory ()
+LevelDBFactory::~LevelDBFactory ()
 {
     leveldb::Cache* cache (reinterpret_cast <leveldb::Cache*> (m_lruCache));
     delete cache;
 }
 
-LevelDBBackendFactory* LevelDBBackendFactory::getInstance ()
+LevelDBFactory* LevelDBFactory::getInstance ()
 {
-    return new LevelDBBackendFactory;
+    return new LevelDBFactory;
 }
 
-String LevelDBBackendFactory::getName () const
+String LevelDBFactory::getName () const
 {
     return "LevelDB";
 }
 
-NodeStore::Backend* LevelDBBackendFactory::createInstance (
+Backend* LevelDBFactory::createInstance (
     size_t keyBytes,
-    StringPairArray const& keyValues,
+    Parameters const& keyValues,
-    NodeStore::Scheduler& scheduler)
+    Scheduler& scheduler)
 {
-    return new LevelDBBackendFactory::Backend (keyBytes, keyValues, scheduler);
+    return new LevelDBFactory::BackendImp (keyBytes, keyValues, scheduler);
 }
 
-//------------------------------------------------------------------------------
+}
-

src/ripple_core/nodestore/backend/LevelDBFactory.h

@@ -0,0 +1,40 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_NODESTORE_LEVELDBFACTORY_H_INCLUDED
+#define RIPPLE_NODESTORE_LEVELDBFACTORY_H_INCLUDED
+
+namespace NodeStore
+{
+
+/** Factory to produce LevelDBFactory backends for the NodeStore.
+
+    @see Database
+*/
+class LevelDBFactory : public Factory
+{
+private:
+    LevelDBFactory ();
+    ~LevelDBFactory ();
+
+public:
+    class BackendImp;
+
+    static LevelDBFactory* getInstance ();
+
+    String getName () const;
+
+    Backend* createInstance (size_t keyBytes,
+                             Parameters const& keyValues,
+                             Scheduler& scheduler);
+
+private:
+    void* m_lruCache;
+};
+
+}
+
+#endif

src/ripple_core/nodestore/backend/MdbFactory.cpp

@@ -6,19 +6,18 @@
 
 #if RIPPLE_MDB_AVAILABLE
 
-class MdbBackendFactory::Backend
+namespace NodeStore
-    : public NodeStore::Backend
+{
-    , public NodeStore::BatchWriter::Callback
+
-    , LeakChecked <MdbBackendFactory::Backend>
+class MdbFactory::BackendImp
+    : public Backend
+    , public BatchWriter::Callback
+    , public LeakChecked <MdbFactory::BackendImp>
 {
 public:
-    typedef NodeStore::Batch Batch;
+    explicit BackendImp (size_t keyBytes,
-    typedef NodeStore::EncodedBlob EncodedBlob;
+                      Parameters const& keyValues,
-    typedef NodeStore::DecodedBlob DecodedBlob;
+                      Scheduler& scheduler)
-
-    explicit Backend (size_t keyBytes,
-                      StringPairArray const& keyValues,
-                      NodeStore::Scheduler& scheduler)
         : m_keyBytes (keyBytes)
         , m_scheduler (scheduler)
         , m_batch (*this, scheduler)
@@ -78,7 +77,7 @@ public:
         }
     }
 
-    ~Backend ()
+    ~BackendImp ()
     {
         if (m_env != nullptr)
         {
@@ -229,11 +228,6 @@ public:
         return m_batch.getWriteLoad ();
     }
 
-    void stopAsync ()
-    {
-        m_batch.stopAsync();
-    }
-
     //--------------------------------------------------------------------------
 
     void writeBatch (Batch const& batch)
@@ -241,16 +235,11 @@ public:
         storeBatch (batch);
     }
 
-    void writeStopped ()
-    {
-        m_scheduler.scheduledTasksStopped ();
-    }
-
 private:
     size_t const m_keyBytes;
-    NodeStore::Scheduler& m_scheduler;
+    Scheduler& m_scheduler;
-    NodeStore::BatchWriter m_batch;
+    BatchWriter m_batch;
-    NodeStore::EncodedBlob::Pool m_blobPool;
+    EncodedBlob::Pool m_blobPool;
     std::string m_basePath;
     MDB_env*    m_env;
     MDB_dbi     m_dbi;
@@ -258,30 +247,32 @@ private:
 
 //------------------------------------------------------------------------------
 
-MdbBackendFactory::MdbBackendFactory ()
+MdbFactory::MdbFactory ()
 {
 }
 
-MdbBackendFactory::~MdbBackendFactory ()
+MdbFactory::~MdbFactory ()
 {
 }
 
-MdbBackendFactory* MdbBackendFactory::getInstance ()
+MdbFactory* MdbFactory::getInstance ()
 {
-    return new MdbBackendFactory;
+    return new MdbFactory;
 }
 
-String MdbBackendFactory::getName () const
+String MdbFactory::getName () const
 {
     return "mdb";
 }
 
-NodeStore::Backend* MdbBackendFactory::createInstance (
+Backend* MdbFactory::createInstance (
     size_t keyBytes,
-    StringPairArray const& keyValues,
+    Parameters const& keyValues,
-    NodeStore::Scheduler& scheduler)
+    Scheduler& scheduler)
 {
-    return new MdbBackendFactory::Backend (keyBytes, keyValues, scheduler);
+    return new MdbFactory::BackendImp (keyBytes, keyValues, scheduler);
+}
+
 }
 
 #endif

src/ripple_core/nodestore/backend/MdbFactory.h

@@ -0,0 +1,43 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_NODESTORE_MDBFACTORY_H_INCLUDED
+#define RIPPLE_NODESTORE_MDBFACTORY_H_INCLUDED
+
+#if RIPPLE_MDB_AVAILABLE
+
+namespace NodeStore
+{
+
+/** Factory to produce a backend using MDB.
+
+    @note MDB is not currently available for Win32
+
+    @see Database
+*/
+class MdbFactory : public Factory
+{
+private:
+    MdbFactory ();
+    ~MdbFactory ();
+
+public:
+    class BackendImp;
+
+    static MdbFactory* getInstance ();
+
+    String getName () const;
+
+    Backend* createInstance (size_t keyBytes,
+                             Parameters const& keyValues,
+                             Scheduler& scheduler);
+};
+
+}
+
+#endif
+
+#endif

src/ripple_core/nodestore/backend/MemoryFactory.cpp

@@ -4,20 +4,23 @@
 */
 //==============================================================================
 
-class MemoryBackendFactory::Backend : public NodeStore::Backend
+namespace NodeStore
+{
+
+class MemoryFactory::BackendImp : public Backend
 {
 private:
     typedef std::map <uint256 const, NodeObject::Ptr> Map;
 
 public:
-    Backend (size_t keyBytes, StringPairArray const& keyValues,
+    BackendImp (size_t keyBytes, Parameters const& keyValues,
-        NodeStore::Scheduler& scheduler)
+        Scheduler& scheduler)
         : m_keyBytes (keyBytes)
         , m_scheduler (scheduler)
     {
     }
 
-    ~Backend ()
+    ~BackendImp ()
     {
     }
 
@@ -56,7 +59,7 @@ public:
         }
     }
 
-    void storeBatch (NodeStore::Batch const& batch)
+    void storeBatch (Batch const& batch)
     {
         for (std::size_t i = 0; i < batch.size (); ++i)
             store (batch [i]);
@@ -73,47 +76,41 @@ public:
         return 0;
     }
 
-    void stopAsync ()
-    {
-        m_scheduler.scheduledTasksStopped ();
-    }
-
     //--------------------------------------------------------------------------
 
 private:
     size_t const m_keyBytes;
 
     Map m_map;
-    NodeStore::Scheduler& m_scheduler;
+    Scheduler& m_scheduler;
 };
 
 //------------------------------------------------------------------------------
 
-MemoryBackendFactory::MemoryBackendFactory ()
+MemoryFactory::MemoryFactory ()
 {
 }
 
-MemoryBackendFactory::~MemoryBackendFactory ()
+MemoryFactory::~MemoryFactory ()
 {
 }
 
-MemoryBackendFactory* MemoryBackendFactory::getInstance ()
+MemoryFactory* MemoryFactory::getInstance ()
 {
-    return new MemoryBackendFactory;
+    return new MemoryFactory;
 }
 
-String MemoryBackendFactory::getName () const
+String MemoryFactory::getName () const
 {
     return "Memory";
 }
 
-NodeStore::Backend* MemoryBackendFactory::createInstance (
+Backend* MemoryFactory::createInstance (
     size_t keyBytes,
-    StringPairArray const& keyValues,
+    Parameters const& keyValues,
-    NodeStore::Scheduler& scheduler)
+    Scheduler& scheduler)
 {
-    return new MemoryBackendFactory::Backend (keyBytes, keyValues, scheduler);
+    return new MemoryFactory::BackendImp (keyBytes, keyValues, scheduler);
 }
 
-//------------------------------------------------------------------------------
+}
-

src/ripple_core/nodestore/backend/MemoryFactory.h

@@ -0,0 +1,37 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_NODESTORE_MEMORYFACTORY_H_INCLUDED
+#define RIPPLE_NODESTORE_MEMORYFACTORY_H_INCLUDED
+
+namespace NodeStore
+{
+
+/** Factory to produce a RAM based backend for the NodeStore.
+
+    @see Database
+*/
+class MemoryFactory : public Factory
+{
+private:
+    MemoryFactory ();
+    ~MemoryFactory ();
+
+public:
+    class BackendImp;
+
+    static MemoryFactory* getInstance ();
+
+    String getName () const;
+
+    Backend* createInstance (size_t keyBytes,
+                             Parameters const& keyValues,
+                             Scheduler& scheduler);
+};
+
+}
+
+#endif

src/ripple_core/nodestore/backend/NullFactory.cpp

@@ -0,0 +1,81 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+namespace NodeStore
+{
+
+class NullFactory::BackendImp : public Backend
+{
+public:
+    explicit BackendImp (Scheduler& scheduler)
+        : m_scheduler (scheduler)
+    {
+    }
+
+    ~BackendImp ()
+    {
+    }
+
+    std::string getName()
+    {
+        return std::string ();
+    }
+
+    Status fetch (void const*, NodeObject::Ptr*)
+    {
+        return notFound;
+    }
+    
+    void store (NodeObject::ref object)
+    {
+    }
+    
+    void storeBatch (Batch const& batch)
+    {
+    }
+
+    void visitAll (VisitCallback& callback)
+    {
+    }
+
+    int getWriteLoad ()
+    {
+        return 0;
+    }
+
+private:
+    Scheduler& m_scheduler;
+};
+
+//------------------------------------------------------------------------------
+
+NullFactory::NullFactory ()
+{
+}
+
+NullFactory::~NullFactory ()
+{
+}
+
+NullFactory* NullFactory::getInstance ()
+{
+    return new NullFactory;
+}
+
+String NullFactory::getName () const
+{
+    return "none";
+}
+
+Backend* NullFactory::createInstance (
+    size_t,
+    Parameters const&,
+    Scheduler& scheduler)
+{
+    return new NullFactory::BackendImp (scheduler);
+}
+
+}

src/ripple_core/nodestore/backend/NullFactory.h

@@ -0,0 +1,40 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_NODESTORE_NULLFACTORY_H_INCLUDED
+#define RIPPLE_NODESTORE_NULLFACTORY_H_INCLUDED
+
+namespace NodeStore
+{
+
+/** Factory to produce a null backend.
+
+    This is for standalone / testing mode.
+
+    @see Database
+*/
+class NullFactory : public Factory
+{
+private:
+    NullFactory ();
+    ~NullFactory ();
+
+public:
+    class BackendImp;
+
+    static NullFactory* getInstance ();
+
+    String getName () const;
+
+    Backend* createInstance (size_t keyBytes,
+                             Parameters const& keyValues,
+                             Scheduler& scheduler);
+};
+
+}
+
+#endif
+

src/ripple_core/nodestore/backend/SophiaFactory.cpp

@@ -6,22 +6,22 @@
 
 #if RIPPLE_SOPHIA_AVAILABLE
 
-class SophiaBackendFactory::Backend
+namespace NodeStore
-    : public NodeStore::Backend
+{
-    , public NodeStore::BatchWriter::Callback
+
-    , public LeakChecked <SophiaBackendFactory::Backend>
+class SophiaFactory::BackendImp
+    : public Backend
+    , public BatchWriter::Callback
+    , public LeakChecked <SophiaFactory::BackendImp>
 {
 public:
     typedef RecycledObjectPool <std::string> StringPool;
-    typedef NodeStore::Batch Batch;
-    typedef NodeStore::EncodedBlob EncodedBlob;
-    typedef NodeStore::DecodedBlob DecodedBlob;
 
     //--------------------------------------------------------------------------
 
-    Backend (int keyBytes,
+    BackendImp (int keyBytes,
-             StringPairArray const& keyValues,
+             Parameters const& keyValues,
-             NodeStore::Scheduler& scheduler)
+             Scheduler& scheduler)
         : m_keyBytes (keyBytes)
         , m_scheduler (scheduler)
         , m_batch (*this, scheduler)
@@ -41,7 +41,7 @@ public:
         }
     }
 
-    ~Backend ()
+    ~BackendImp ()
     {
         if (m_db != nullptr)
             sp_destroy (m_db);
@@ -108,7 +108,7 @@ public:
     {
         EncodedBlob::Pool::ScopedItem item (m_blobPool);
 
-        for (NodeStore::Batch::const_iterator iter (batch.begin());
+        for (Batch::const_iterator iter (batch.begin());
             iter != batch.end(); ++iter)
         {
             EncodedBlob& encoded (item.getObject ());
@@ -136,29 +136,19 @@ public:
         return m_batch.getWriteLoad ();
     }
 
-    void stopAsync ()
-    {
-        m_batch.stopAsync ();
-    }
-
     //--------------------------------------------------------------------------
 
-    void writeBatch (NodeStore::Batch const& batch)
+    void writeBatch (Batch const& batch)
     {
         storeBatch (batch);
     }
 
-    void writeStopped ()
-    {
-        m_scheduler.scheduledTasksStopped ();
-    }
-
 private:
     size_t const m_keyBytes;
-    NodeStore::Scheduler& m_scheduler;
+    Scheduler& m_scheduler;
-    NodeStore::BatchWriter m_batch;
+    BatchWriter m_batch;
     StringPool m_stringPool;
-    NodeStore::EncodedBlob::Pool m_blobPool;
+    EncodedBlob::Pool m_blobPool;
     std::string m_name;
     void* m_env;
     void* m_db;
@@ -166,7 +156,7 @@ private:
 
 //------------------------------------------------------------------------------
 
-SophiaBackendFactory::SophiaBackendFactory ()
+SophiaFactory::SophiaFactory ()
 {
     leveldb::Options options;
     options.create_if_missing = true;
@@ -174,28 +164,28 @@ SophiaBackendFactory::SophiaBackendFactory ()
         siHashNodeDBCache) * 1024 * 1024);
 }
 
-SophiaBackendFactory::~SophiaBackendFactory ()
+SophiaFactory::~SophiaFactory ()
 {
 }
 
-SophiaBackendFactory* SophiaBackendFactory::getInstance ()
+SophiaFactory* SophiaFactory::getInstance ()
 {
-    return new SophiaBackendFactory;
+    return new SophiaFactory;
 }
 
-String SophiaBackendFactory::getName () const
+String SophiaFactory::getName () const
 {
     return "sophia";
 }
 
-NodeStore::Backend* SophiaBackendFactory::createInstance (
+Backend* SophiaFactory::createInstance (
     size_t keyBytes,
-    StringPairArray const& keyValues,
+    Parameters const& keyValues,
-    NodeStore::Scheduler& scheduler)
+    Scheduler& scheduler)
 {
-    return new SophiaBackendFactory::Backend (keyBytes, keyValues, scheduler);
+    return new SophiaFactory::BackendImp (keyBytes, keyValues, scheduler);
 }
 
-//------------------------------------------------------------------------------
+}
 
 #endif

src/ripple_core/nodestore/backend/SophiaFactory.h

@@ -0,0 +1,37 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_NODESTORE_SOPHIAFACTORY_H_INCLUDED
+#define RIPPLE_NODESTORE_SOPHIAFACTORY_H_INCLUDED
+
+namespace NodeStore
+{
+
+/** Factory to produce Sophia backends for the NodeStore.
+
+    @see Database
+*/
+class SophiaFactory : public Factory
+{
+private:
+    SophiaFactory ();
+    ~SophiaFactory ();
+
+public:
+    class BackendImp;
+
+    static SophiaFactory* getInstance ();
+
+    String getName () const;
+
+    NodeStore::Backend* createInstance (size_t keyBytes,
+                                        Parameters const& keyValues,
+                                        Scheduler& scheduler);
+};
+
+}
+
+#endif

src/ripple_core/nodestore/impl/BatchWriter.cpp

@@ -0,0 +1,96 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+namespace NodeStore
+{
+
+BatchWriter::BatchWriter (Callback& callback, Scheduler& scheduler)
+    : m_callback (callback)
+    , m_scheduler (scheduler)
+    , mWriteGeneration (0)
+    , mWriteLoad (0)
+    , mWritePending (false)
+{
+    mWriteSet.reserve (batchWritePreallocationSize);
+}
+
+BatchWriter::~BatchWriter ()
+{
+    waitForWriting ();
+}
+
+void BatchWriter::store (NodeObject::ref object)
+{
+    LockType::scoped_lock sl (mWriteMutex);
+
+    mWriteSet.push_back (object);
+
+    if (! mWritePending)
+    {
+        mWritePending = true;
+
+        m_scheduler.scheduleTask (*this);
+    }
+}
+
+int BatchWriter::getWriteLoad ()
+{
+    LockType::scoped_lock sl (mWriteMutex);
+
+    return std::max (mWriteLoad, static_cast<int> (mWriteSet.size ()));
+}
+
+void BatchWriter::performScheduledTask ()
+{
+    writeBatch ();
+}
+
+void BatchWriter::writeBatch ()
+{
+    int setSize = 0;
+
+    for (;;)
+    {
+        std::vector< boost::shared_ptr<NodeObject> > set;
+
+        set.reserve (batchWritePreallocationSize);
+
+        {
+            LockType::scoped_lock sl (mWriteMutex);
+
+            mWriteSet.swap (set);
+            assert (mWriteSet.empty ());
+            ++mWriteGeneration;
+            mWriteCondition.notify_all ();
+
+            if (set.empty ())
+            {
+                mWritePending = false;
+                mWriteLoad = 0;
+
+                // VFALCO NOTE Fix this function to not return from the middle
+                return;
+            }
+
+            int newSetSize = mWriteSet.size();
+            mWriteLoad = std::max (setSize, newSetSize);
+            newSetSize = set.size ();
+        }
+
+        m_callback.writeBatch (set);
+    }
+}
+
+void BatchWriter::waitForWriting ()
+{
+    LockType::scoped_lock sl (mWriteMutex);
+    int gen = mWriteGeneration;
+
+    while (mWritePending && (mWriteGeneration == gen))
+        mWriteCondition.wait (sl);
+}
+
+}

src/ripple_core/nodestore/impl/BatchWriter.h

@@ -0,0 +1,74 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_NODESTORE_BATCHWRITER_H_INCLUDED
+#define RIPPLE_NODESTORE_BATCHWRITER_H_INCLUDED
+
+namespace NodeStore
+{
+
+/** Batch-writing assist logic.
+
+    The batch writes are performed with a scheduled task. Use of the
+    class it not required. A backend can implement its own write batching,
+    or skip write batching if doing so yields a performance benefit.
+
+    @see Scheduler
+*/
+// VFALCO NOTE I'm not entirely happy having placed this here,
+//             because whoever needs to use NodeStore certainly doesn't
+//             need to see the implementation details of BatchWriter.
+//
+class BatchWriter : private Task
+{
+public:
+    /** This callback does the actual writing. */
+    struct Callback
+    {
+        virtual void writeBatch (Batch const& batch) = 0;
+    };
+
+    /** Create a batch writer. */
+    BatchWriter (Callback& callback, Scheduler& scheduler);
+
+    /** Destroy a batch writer.
+
+        Anything pending in the batch is written out before this returns.
+    */
+    ~BatchWriter ();
+
+    /** Store the object.
+
+        This will add to the batch and initiate a scheduled task to
+        write the batch out.
+    */
+    void store (NodeObject::Ptr const& object);
+
+    /** Get an estimate of the amount of writing I/O pending. */
+    int getWriteLoad ();
+
+private:
+    void performScheduledTask ();
+    void writeBatch ();
+    void waitForWriting ();
+
+private:
+    typedef boost::recursive_mutex LockType;
+    typedef boost::condition_variable_any CondvarType;
+
+    Callback& m_callback;
+    Scheduler& m_scheduler;
+    LockType mWriteMutex;
+    CondvarType mWriteCondition;
+    int mWriteGeneration;
+    int mWriteLoad;
+    bool mWritePending;
+    Batch mWriteSet;
+};
+
+}
+
+#endif

src/ripple_core/nodestore/impl/DatabaseImp.h

@@ -0,0 +1,352 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+namespace NodeStore
+{
+
+class DatabaseImp
+    : public Database
+    , LeakChecked <DatabaseImp>
+{
+public:
+    DatabaseImp (char const* name,
+                 Scheduler& scheduler,
+                 Parameters const& backendParameters,
+                 Parameters const& fastBackendParameters)
+        : m_scheduler (scheduler)
+        , m_backend (createBackend (backendParameters, scheduler))
+        , m_fastBackend ((fastBackendParameters.size () > 0)
+            ? createBackend (fastBackendParameters, scheduler) : nullptr)
+        , m_cache ("NodeStore", 16384, 300)
+        , m_negativeCache ("NoteStoreNegativeCache", 0, 120)
+    {
+    }
+
+    ~DatabaseImp ()
+    {
+    }
+
+    String getName () const
+    {
+        return m_backend->getName ();
+    }
+
+    //------------------------------------------------------------------------------
+
+    NodeObject::Ptr fetch (uint256 const& hash)
+    {
+        // See if the object already exists in the cache
+        //
+        NodeObject::Ptr obj = m_cache.fetch (hash);
+
+        if (obj == nullptr)
+        {
+            // It's not in the cache, see if we can skip checking the db.
+            //
+            if (! m_negativeCache.isPresent (hash))
+            {
+                // There's still a chance it could be in one of the databases.
+
+                bool foundInFastBackend = false;
+
+                // Check the fast backend database if we have one
+                //
+                if (m_fastBackend != nullptr)
+                {
+                    obj = fetchInternal (m_fastBackend, hash);
+
+                    // If we found the object, avoid storing it again later.
+                    if (obj != nullptr)
+                        foundInFastBackend = true;
+                }
+
+                // Are we still without an object?
+                //
+                if (obj == nullptr)
+                {
+                    // Yes so at last we will try the main database.
+                    //
+                    {
+                        // Monitor this operation's load since it is expensive.
+                        //
+                        // VFALCO TODO Why is this an autoptr? Why can't it just be a plain old object?
+                        //
+                        // VFALCO NOTE Commented this out because it breaks the unit test!
+                        //
+                        //LoadEvent::autoptr event (getApp().getJobQueue ().getLoadEventAP (jtHO_READ, "HOS::retrieve"));
+
+                        obj = fetchInternal (m_backend, hash);
+                    }
+
+                    // If it's not in the main database, remember that so we
+                    // can skip the lookup for the same object again later.
+                    //
+                    if (obj == nullptr)
+                        m_negativeCache.add (hash);
+                }
+
+                // Did we finally get something?
+                //
+                if (obj != nullptr)
+                {
+                    // Yes it so canonicalize. This solves the problem where
+                    // more than one thread has its own copy of the same object.
+                    //
+                    m_cache.canonicalize (hash, obj);
+
+                    if (! foundInFastBackend)
+                    {
+                        // If we have a fast back end, store it there for later.
+                        //
+                        if (m_fastBackend != nullptr)
+                            m_fastBackend->store (obj);
+
+                        // Since this was a 'hard' fetch, we will log it.
+                        //
+                        WriteLog (lsTRACE, NodeObject) << "HOS: " << hash << " fetch: in db";
+                    }
+                }
+            }
+            else
+            {
+                // hash is known not to be in the database
+            }
+        }
+        else
+        {
+            // found it!
+        }
+
+        return obj;
+    }
+
+    NodeObject::Ptr fetchInternal (Backend* backend, uint256 const& hash)
+    {
+        NodeObject::Ptr object;
+
+        Status const status = backend->fetch (hash.begin (), &object);
+
+        switch (status)
+        {
+        case ok:
+        case notFound:
+            break;
+
+        case dataCorrupt:
+            // VFALCO TODO Deal with encountering corrupt data!
+            //
+            WriteLog (lsFATAL, NodeObject) << "Corrupt NodeObject #" << hash;
+            break;
+
+        default:
+            WriteLog (lsWARNING, NodeObject) << "Unknown status=" << status;
+            break;
+        }
+
+        return object;
+    }
+
+    //------------------------------------------------------------------------------
+
+    void store (NodeObjectType type,
+                uint32 index,
+                Blob& data,
+                uint256 const& hash)
+    {
+        bool const keyFoundAndObjectCached = m_cache.refreshIfPresent (hash);
+
+        // VFALCO NOTE What happens if the key is found, but the object
+        //             fell out of the cache? We will end up passing it
+        //             to the backend anyway.
+        //
+        if (! keyFoundAndObjectCached)
+        {
+        #if RIPPLE_VERIFY_NODEOBJECT_KEYS
+            assert (hash == Serializer::getSHA512Half (data));
+        #endif
+
+            NodeObject::Ptr object = NodeObject::createObject (
+                type, index, data, hash);
+
+            if (!m_cache.canonicalize (hash, object))
+            {
+                m_backend->store (object);
+
+                if (m_fastBackend)
+                    m_fastBackend->store (object);
+            }
+
+            m_negativeCache.del (hash);
+        }
+    }
+
+    //------------------------------------------------------------------------------
+
+    float getCacheHitRate ()
+    {
+        return m_cache.getHitRate ();
+    }
+
+    void tune (int size, int age)
+    {
+        m_cache.setTargetSize (size);
+        m_cache.setTargetAge (age);
+    }
+
+    void sweep ()
+    {
+        m_cache.sweep ();
+        m_negativeCache.sweep ();
+    }
+
+    int getWriteLoad ()
+    {
+        return m_backend->getWriteLoad ();
+    }
+
+    //------------------------------------------------------------------------------
+
+    void visitAll (VisitCallback& callback)
+    {
+        m_backend->visitAll (callback);
+    }
+
+    void import (Database& sourceDatabase)
+    {
+        class ImportVisitCallback : public VisitCallback
+        {
+        public:
+            explicit ImportVisitCallback (Backend& backend)
+                : m_backend (backend)
+            {
+                m_objects.reserve (batchWritePreallocationSize);
+            }
+
+            ~ImportVisitCallback ()
+            {
+                if (! m_objects.empty ())
+                    m_backend.storeBatch (m_objects);
+            }
+
+            void visitObject (NodeObject::Ptr const& object)
+            {
+                if (m_objects.size () >= batchWritePreallocationSize)
+                {
+                    m_backend.storeBatch (m_objects);
+
+                    m_objects.clear ();
+                    m_objects.reserve (batchWritePreallocationSize);
+                }
+
+                m_objects.push_back (object);
+            }
+
+        private:
+            Backend& m_backend;
+            Batch m_objects;
+        };
+
+        //--------------------------------------------------------------------------
+
+        ImportVisitCallback callback (*m_backend);
+
+        sourceDatabase.visitAll (callback);
+    }
+
+    //------------------------------------------------------------------------------
+
+    static void missing_backend ()
+    {
+        fatal_error ("Your rippled.cfg is missing a [node_db] entry, please see the rippled-example.cfg file!");
+    }
+
+    static Backend* createBackend (Parameters const& parameters, Scheduler& scheduler)
+    {
+        Backend* backend = nullptr;
+
+        String const& type = parameters ["type"];
+
+        if (type.isNotEmpty ())
+        {
+            Factory* factory (Factories::get().find (type));
+
+            if (factory != nullptr)
+            {
+                backend = factory->createInstance (NodeObject::keyBytes, parameters, scheduler);
+            }
+            else
+            {
+                missing_backend ();
+            }
+        }
+        else
+        {
+            missing_backend ();
+        }
+
+        return backend;
+    }
+
+    //------------------------------------------------------------------------------
+
+private:
+    Scheduler& m_scheduler;
+
+    // Persistent key/value storage.
+    ScopedPointer <Backend> m_backend;
+
+    // Larger key/value storage, but not necessarily persistent.
+    ScopedPointer <Backend> m_fastBackend;
+
+    // VFALCO NOTE What are these things for? We need comments.
+    TaggedCacheType <uint256, NodeObject, UptimeTimerAdapter> m_cache;
+    KeyCache <uint256, UptimeTimerAdapter> m_negativeCache;
+};
+
+//------------------------------------------------------------------------------
+
+void Database::addFactory (Factory* factory)
+{
+    Factories::get().add (factory);
+}
+
+void Database::addAvailableBackends ()
+{
+    // This is part of the ripple_app module since it has dependencies
+    //addFactory (SqliteFactory::getInstance ());
+
+    addFactory (LevelDBFactory::getInstance ());
+
+    addFactory (MemoryFactory::getInstance ());
+    addFactory (NullFactory::getInstance ());
+
+#if RIPPLE_HYPERLEVELDB_AVAILABLE
+    addFactory (HyperDBFactory::getInstance ());
+#endif
+
+#if RIPPLE_MDB_AVAILABLE
+    addFactory (MdbFactory::getInstance ());
+#endif
+
+#if RIPPLE_SOPHIA_AVAILABLE
+    addFactory (SophiaFactory::getInstance ());
+#endif
+
+    addFactory (KeyvaDBFactory::getInstance ());
+}
+
+//------------------------------------------------------------------------------
+
+Database* Database::New (char const* name,
+                         Scheduler& scheduler,
+                         Parameters const& backendParameters,
+                         Parameters fastBackendParameters)
+{
+    return new DatabaseImp (name,
+        scheduler, backendParameters, fastBackendParameters);
+}
+
+}

src/ripple_core/nodestore/impl/DecodedBlob.cpp

@@ -0,0 +1,83 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+namespace NodeStore
+{
+
+DecodedBlob::DecodedBlob (void const* key, void const* value, int valueBytes)
+{
+    /*  Data format:
+
+        Bytes
+
+        0...3       LedgerIndex     32-bit big endian integer
+        4...7       Unused?         An unused copy of the LedgerIndex
+        8           char            One of NodeObjectType
+        9...end                     The body of the object data
+    */
+
+    m_success = false;
+    m_key = key;
+    // VFALCO NOTE Ledger indexes should have started at 1
+    m_ledgerIndex = LedgerIndex (-1);
+    m_objectType = hotUNKNOWN;
+    m_objectData = nullptr;
+    m_dataBytes = bmax (0, valueBytes - 9);
+
+    if (valueBytes > 4)
+    {
+        LedgerIndex const* index = static_cast <LedgerIndex const*> (value);
+        m_ledgerIndex = ByteOrder::swapIfLittleEndian (*index);
+    }
+
+    // VFALCO NOTE What about bytes 4 through 7 inclusive?
+
+    if (valueBytes > 8)
+    {
+        unsigned char const* byte = static_cast <unsigned char const*> (value);
+        m_objectType = static_cast <NodeObjectType> (byte [8]);
+    }
+
+    if (valueBytes > 9)
+    {
+        m_objectData = static_cast <unsigned char const*> (value) + 9;
+
+        switch (m_objectType)
+        {
+        case hotUNKNOWN:
+        default:
+            break;
+
+        case hotLEDGER:
+        case hotTRANSACTION:
+        case hotACCOUNT_NODE:
+        case hotTRANSACTION_NODE:
+            m_success = true;
+            break;
+        }
+    }
+}
+
+NodeObject::Ptr DecodedBlob::createObject ()
+{
+    bassert (m_success);
+
+    NodeObject::Ptr object;
+
+    if (m_success)
+    {
+        Blob data (m_dataBytes);
+
+        memcpy (data.data (), m_objectData, m_dataBytes);
+
+        object = NodeObject::createObject (
+            m_objectType, m_ledgerIndex, data, uint256::fromVoid (m_key));
+    }
+
+    return object;
+}
+
+}

src/ripple_core/nodestore/impl/DecodedBlob.h

@@ -0,0 +1,47 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_NODESTORE_DECODEDBLOB_H_INCLUDED
+#define RIPPLE_NODESTORE_DECODEDBLOB_H_INCLUDED
+
+namespace NodeStore
+{
+
+/** Parsed key/value blob into NodeObject components.
+
+    This will extract the information required to construct a NodeObject. It
+    also does consistency checking and returns the result, so it is possible
+    to determine if the data is corrupted without throwing an exception. Not
+    all forms of corruption are detected so further analysis will be needed
+    to eliminate false negatives.
+
+    @note This defines the database format of a NodeObject!
+*/
+class DecodedBlob
+{
+public:
+    /** Construct the decoded blob from raw data. */
+    DecodedBlob (void const* key, void const* value, int valueBytes);
+
+    /** Determine if the decoding was successful. */
+    bool wasOk () const noexcept { return m_success; }
+
+    /** Create a NodeObject from this data. */
+    NodeObject::Ptr createObject ();
+
+private:
+    bool m_success;
+
+    void const* m_key;
+    LedgerIndex m_ledgerIndex;
+    NodeObjectType m_objectType;
+    unsigned char const* m_objectData;
+    int m_dataBytes;
+};
+
+}
+
+#endif

src/ripple_core/nodestore/impl/DummyScheduler.cpp

@@ -0,0 +1,29 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+namespace NodeStore
+{
+
+DummyScheduler::DummyScheduler ()
+{
+}
+
+DummyScheduler::~DummyScheduler ()
+{
+}
+
+void DummyScheduler::scheduleTask (Task& task)
+{
+    // Invoke the task synchronously.
+    task.performScheduledTask();
+}
+
+void DummyScheduler::scheduledTasksStopped ()
+{
+}
+
+}
+

src/ripple_core/nodestore/impl/EncodedBlob.cpp

@@ -0,0 +1,38 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+namespace NodeStore
+{
+
+void EncodedBlob::prepare (NodeObject::Ptr const& object)
+{
+    m_key = object->getHash ().begin ();
+
+    // This is how many bytes we need in the flat data
+    m_size = object->getData ().size () + 9;
+
+    m_data.ensureSize (m_size);
+
+    // These sizes must be the same!
+    static_bassert (sizeof (uint32) == sizeof (object->getIndex ()));
+
+    {
+        uint32* buf = static_cast <uint32*> (m_data.getData ());
+
+        buf [0] = ByteOrder::swapIfLittleEndian (object->getIndex ());
+        buf [1] = ByteOrder::swapIfLittleEndian (object->getIndex ());
+    }
+
+    {
+        unsigned char* buf = static_cast <unsigned char*> (m_data.getData ());
+
+        buf [8] = static_cast <unsigned char> (object->getType ());
+
+        memcpy (&buf [9], object->getData ().data (), object->getData ().size ());
+    }
+}
+
+}

src/ripple_core/nodestore/impl/EncodedBlob.h

@@ -0,0 +1,40 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_NODESTORE_ENCODEDBLOB_H_INCLUDED
+#define RIPPLE_NODESTORE_ENCODEDBLOB_H_INCLUDED
+
+namespace NodeStore
+{
+
+/** Utility for producing flattened node objects.
+
+    These get recycled to prevent many small allocations.
+
+    @note This defines the database format of a NodeObject!
+*/
+struct EncodedBlob
+{
+public:
+    typedef RecycledObjectPool <EncodedBlob> Pool;
+
+    void prepare (NodeObject::Ptr const& object);
+
+    void const* getKey () const noexcept { return m_key; }
+
+    size_t getSize () const noexcept { return m_size; }
+
+    void const* getData () const noexcept { return m_data.getData (); }
+
+private:
+    void const* m_key;
+    MemoryBlock m_data;
+    size_t m_size;
+};
+
+}
+
+#endif

src/ripple_core/nodestore/impl/Factories.h

@@ -0,0 +1,50 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_NODESTORE_FACTORIES_H_INCLUDED
+#define RIPPLE_NODESTORE_FACTORIES_H_INCLUDED
+
+namespace NodeStore
+{
+
+// Holds the list of Backend factories
+class Factories
+{
+public:
+    Factories ()
+    {
+    }
+
+    ~Factories ()
+    {
+    }
+
+    void add (Factory* factory)
+    {
+        m_list.add (factory);
+    }
+
+    Factory* find (String name) const
+    {
+        for (int i = 0; i < m_list.size(); ++i)
+            if (m_list [i]->getName ().compareIgnoreCase (name) == 0)
+                return m_list [i];
+        return nullptr;
+    }
+
+    static Factories& get ()
+    {
+        return *SharedSingleton <Factories>::get (
+            SingletonLifetime::persistAfterCreation);
+    }
+
+private:
+    OwnedArray <Factory> m_list;
+};
+
+}
+
+#endif

src/ripple_core/nodestore/impl/NodeObject.cpp

@@ -73,19 +73,3 @@ bool NodeObject::isCloneOf (NodeObject::Ptr const& other) const
 
 //------------------------------------------------------------------------------
 
-class NodeObjectTests : public UnitTest
-{
-public:
-
-    NodeObjectTests () : UnitTest ("NodeObject", "ripple")
-    {
-    }
-
-
-    void runTest ()
-    {
-    }
-};
-
-static NodeObjectTests nodeObjectTests;
-

src/ripple_core/nodestore/tests/BackendTests.cpp

@@ -0,0 +1,97 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+namespace NodeStore
+{
+
+// Tests the Backend interface
+//
+class BackendTests : public TestBase
+{
+public:
+    void testBackend (String type, int64 const seedValue, int numObjectsToTest = 2000)
+    {
+        DummyScheduler scheduler;
+
+        beginTestCase (String ("Backend type=") + type);
+
+        StringPairArray params;
+        File const path (File::createTempFile ("node_db"));
+        params.set ("type", type);
+        params.set ("path", path.getFullPathName ());
+
+        // Create a batch
+        Batch batch;
+        createPredictableBatch (batch, 0, numObjectsToTest, seedValue);
+
+        {
+            // Open the backend
+            ScopedPointer <Backend> backend (DatabaseImp::createBackend (params, scheduler));
+
+            // Write the batch
+            storeBatch (*backend, batch);
+
+            {
+                // Read it back in
+                Batch copy;
+                fetchCopyOfBatch (*backend, &copy, batch);
+                expect (areBatchesEqual (batch, copy), "Should be equal");
+            }
+
+            {
+                // Reorder and read the copy again
+                Batch copy;
+                UnitTestUtilities::repeatableShuffle (batch.size (), batch, seedValue);
+                fetchCopyOfBatch (*backend, &copy, batch);
+                expect (areBatchesEqual (batch, copy), "Should be equal");
+            }
+        }
+
+        {
+            // Re-open the backend
+            ScopedPointer <Backend> backend (DatabaseImp::createBackend (params, scheduler));
+
+            // Read it back in
+            Batch copy;
+            fetchCopyOfBatch (*backend, &copy, batch);
+            // Canonicalize the source and destination batches
+            std::sort (batch.begin (), batch.end (), NodeObject::LessThan ());
+            std::sort (copy.begin (), copy.end (), NodeObject::LessThan ());
+            expect (areBatchesEqual (batch, copy), "Should be equal");
+        }
+    }
+
+    //--------------------------------------------------------------------------
+
+    void runTest ()
+    {
+        int const seedValue = 50;
+
+        testBackend ("leveldb", seedValue);
+
+        testBackend ("sqlite", seedValue);
+
+        #if RIPPLE_HYPERLEVELDB_AVAILABLE
+        testBackend ("hyperleveldb", seedValue);
+        #endif
+
+        #if RIPPLE_MDB_AVAILABLE
+        testBackend ("mdb", seedValue, 200);
+        #endif
+
+        #if RIPPLE_SOPHIA_AVAILABLE
+        testBackend ("sophia", seedValue);
+        #endif
+    }
+
+    BackendTests () : TestBase ("NodeStoreBackend")
+    {
+    }
+};
+
+static BackendTests backendTests;
+
+}

src/ripple_core/nodestore/tests/BasicTests.cpp

@@ -0,0 +1,76 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+namespace NodeStore
+{
+
+// Tests predictable batches, and NodeObject blob encoding
+//
+class BasicTests : public TestBase
+{
+public:
+    BasicTests () : TestBase ("NodeStoreBasics")
+    {
+    }
+
+    // Make sure predictable object generation works!
+    void testBatches (int64 const seedValue)
+    {
+        beginTestCase ("batch");
+
+        Batch batch1;
+        createPredictableBatch (batch1, 0, numObjectsToTest, seedValue);
+
+        Batch batch2;
+        createPredictableBatch (batch2, 0, numObjectsToTest, seedValue);
+
+        expect (areBatchesEqual (batch1, batch2), "Should be equal");
+
+        Batch batch3;
+        createPredictableBatch (batch3, 1, numObjectsToTest, seedValue);
+
+        expect (! areBatchesEqual (batch1, batch3), "Should not be equal");
+    }
+
+    // Checks encoding/decoding blobs
+    void testBlobs (int64 const seedValue)
+    {
+        beginTestCase ("encoding");
+
+        Batch batch;
+        createPredictableBatch (batch, 0, numObjectsToTest, seedValue);
+
+        EncodedBlob encoded;
+        for (int i = 0; i < batch.size (); ++i)
+        {
+            encoded.prepare (batch [i]);
+
+            DecodedBlob decoded (encoded.getKey (), encoded.getData (), encoded.getSize ());
+
+            expect (decoded.wasOk (), "Should be ok");
+
+            if (decoded.wasOk ())
+            {
+                NodeObject::Ptr const object (decoded.createObject ());
+
+                expect (batch [i]->isCloneOf (object), "Should be clones");
+            }
+        }
+    }
+
+    void runTest ()
+    {
+        int64 const seedValue = 50;
+
+        testBatches (seedValue);
+
+        testBlobs (seedValue);
+    }
+};
+
+static BasicTests basicTests;
+
+}

src/ripple_core/nodestore/tests/DatabaseTests.cpp

@@ -0,0 +1,228 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+namespace NodeStore
+{
+
+class DatabaseTests : public TestBase
+{
+public:
+    DatabaseTests ()
+        : TestBase ("NodeStore")
+    {
+    }
+
+    ~DatabaseTests ()
+    {
+    }
+
+    //--------------------------------------------------------------------------
+
+    void testImport (String destBackendType, String srcBackendType, int64 seedValue)
+    {
+        DummyScheduler scheduler;
+
+        File const node_db (File::createTempFile ("node_db"));
+        StringPairArray srcParams;
+        srcParams.set ("type", srcBackendType);
+        srcParams.set ("path", node_db.getFullPathName ());
+
+        // Create a batch
+        Batch batch;
+        createPredictableBatch (batch, 0, numObjectsToTest, seedValue);
+
+        // Write to source db
+        {
+            ScopedPointer <Database> src (Database::New ("test", scheduler, srcParams));
+            storeBatch (*src, batch);
+        }
+
+        Batch copy;
+
+        {
+            // Re-open the db
+            ScopedPointer <Database> src (Database::New ("test", scheduler, srcParams));
+
+            // Set up the destination database
+            File const dest_db (File::createTempFile ("dest_db"));
+            StringPairArray destParams;
+            destParams.set ("type", destBackendType);
+            destParams.set ("path", dest_db.getFullPathName ());
+
+            ScopedPointer <Database> dest (Database::New ("test", scheduler, destParams));
+
+            beginTestCase (String ("import into '") + destBackendType + "' from '" + srcBackendType + "'");
+
+            // Do the import
+            dest->import (*src);
+
+            // Get the results of the import
+            fetchCopyOfBatch (*dest, &copy, batch);
+        }
+
+        // Canonicalize the source and destination batches
+        std::sort (batch.begin (), batch.end (), NodeObject::LessThan ());
+        std::sort (copy.begin (), copy.end (), NodeObject::LessThan ());
+        expect (areBatchesEqual (batch, copy), "Should be equal");
+
+    }
+
+    //--------------------------------------------------------------------------
+
+    void testNodeStore (String type,
+                        bool const useEphemeralDatabase,
+                        bool const testPersistence,
+                        int64 const seedValue,
+                        int numObjectsToTest = 2000)
+    {
+        DummyScheduler scheduler;
+
+        String s;
+        s << String ("NodeStore backend '") + type + "'";
+        if (useEphemeralDatabase)
+            s << " (with ephemeral database)";
+
+        beginTestCase (s);
+
+        File const node_db (File::createTempFile ("node_db"));
+        StringPairArray nodeParams;
+        nodeParams.set ("type", type);
+        nodeParams.set ("path", node_db.getFullPathName ());
+
+        File const temp_db  (File::createTempFile ("temp_db"));
+        StringPairArray tempParams;
+        if (useEphemeralDatabase)
+        {
+            tempParams.set ("type", type);
+            tempParams.set ("path", temp_db.getFullPathName ());
+        }
+
+        // Create a batch
+        Batch batch;
+        createPredictableBatch (batch, 0, numObjectsToTest, seedValue);
+
+        {
+            // Open the database
+            ScopedPointer <Database> db (Database::New ("test", scheduler, nodeParams, tempParams));
+
+            // Write the batch
+            storeBatch (*db, batch);
+
+            {
+                // Read it back in
+                Batch copy;
+                fetchCopyOfBatch (*db, &copy, batch);
+                expect (areBatchesEqual (batch, copy), "Should be equal");
+            }
+
+            {
+                // Reorder and read the copy again
+                Batch copy;
+                UnitTestUtilities::repeatableShuffle (batch.size (), batch, seedValue);
+                fetchCopyOfBatch (*db, &copy, batch);
+                expect (areBatchesEqual (batch, copy), "Should be equal");
+            }
+        }
+
+        if (testPersistence)
+        {
+            {
+                // Re-open the database without the ephemeral DB
+                ScopedPointer <Database> db (Database::New ("test", scheduler, nodeParams));
+
+                // Read it back in
+                Batch copy;
+                fetchCopyOfBatch (*db, &copy, batch);
+
+                // Canonicalize the source and destination batches
+                std::sort (batch.begin (), batch.end (), NodeObject::LessThan ());
+                std::sort (copy.begin (), copy.end (), NodeObject::LessThan ());
+                expect (areBatchesEqual (batch, copy), "Should be equal");
+            }
+
+            if (useEphemeralDatabase)
+            {
+                // Verify the ephemeral db
+                ScopedPointer <Database> db (Database::New ("test",
+                    scheduler, tempParams, StringPairArray ()));
+
+                // Read it back in
+                Batch copy;
+                fetchCopyOfBatch (*db, &copy, batch);
+
+                // Canonicalize the source and destination batches
+                std::sort (batch.begin (), batch.end (), NodeObject::LessThan ());
+                std::sort (copy.begin (), copy.end (), NodeObject::LessThan ());
+                expect (areBatchesEqual (batch, copy), "Should be equal");
+            }
+        }
+    }
+
+    //--------------------------------------------------------------------------
+
+    void runBackendTests (bool useEphemeralDatabase, int64 const seedValue)
+    {
+        testNodeStore ("leveldb", useEphemeralDatabase, true, seedValue);
+
+    #if RIPPLE_HYPERLEVELDB_AVAILABLE
+        testNodeStore ("hyperleveldb", useEphemeralDatabase, true, seedValue);
+    #endif
+
+    #if RIPPLE_MDB_AVAILABLE
+        testNodeStore ("mdb", useEphemeralDatabase, true, seedValue, 200);
+    #endif
+
+    #if RIPPLE_SOPHIA_AVAILABLE
+        testNodeStore ("sophia", useEphemeralDatabase, true, seedValue);
+    #endif
+
+        testNodeStore ("sqlite", useEphemeralDatabase, true, seedValue);
+    }
+
+    //--------------------------------------------------------------------------
+
+    void runImportTests (int64 const seedValue)
+    {
+        testImport ("leveldb", "leveldb", seedValue);
+
+    #if RIPPLE_HYPERLEVELDB_AVAILABLE
+        testImport ("hyperleveldb", "hyperleveldb", seedValue);
+    #endif
+
+    /*
+    #if RIPPLE_MDB_AVAILABLE
+        testImport ("mdb", "mdb", seedValue);
+    #endif
+    */
+
+    /*
+    #if RIPPLE_SOPHIA_AVAILABLE
+        testImport ("sophia", "sophia", seedValue);
+    #endif
+    */
+
+        testImport ("sqlite", "sqlite", seedValue);
+    }
+
+    //--------------------------------------------------------------------------
+
+    void runTest ()
+    {
+        int64 const seedValue = 50;
+
+        testNodeStore ("memory", false, false, seedValue);
+
+        runBackendTests (false, seedValue);
+
+        runBackendTests (true, seedValue);
+
+        runImportTests (seedValue);
+    }
+};
+
+static DatabaseTests databaseTests;
+
+}

src/ripple_core/nodestore/tests/TestBase.h

@@ -0,0 +1,179 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_NODESTORE_TESTBASE_H_INCLUDED
+#define RIPPLE_NODESTORE_TESTBASE_H_INCLUDED
+
+namespace NodeStore
+{
+
+// Some common code for the unit tests
+//
+class TestBase : public UnitTest
+{
+public:
+    // Tunable parameters
+    //
+    enum
+    {
+        maxPayloadBytes = 2000,
+        numObjectsToTest = 2000
+    };
+
+    // Creates predictable objects
+    class PredictableObjectFactory
+    {
+    public:
+        explicit PredictableObjectFactory (int64 seedValue)
+            : m_seedValue (seedValue)
+        {
+        }
+
+        NodeObject::Ptr createObject (int index)
+        {
+            Random r (m_seedValue + index);
+
+            NodeObjectType type;
+            switch (r.nextInt (4))
+            {
+            case 0: type = hotLEDGER; break;
+            case 1: type = hotTRANSACTION; break;
+            case 2: type = hotACCOUNT_NODE; break;
+            case 3: type = hotTRANSACTION_NODE; break;
+            default:
+                type = hotUNKNOWN;
+                break;
+            };
+
+            LedgerIndex ledgerIndex = 1 + r.nextInt (1024 * 1024);
+
+            uint256 hash;
+            r.fillBitsRandomly (hash.begin (), hash.size ());
+
+            int const payloadBytes = 1 + r.nextInt (maxPayloadBytes);
+
+            Blob data (payloadBytes);
+
+            r.fillBitsRandomly (data.data (), payloadBytes);
+
+            return NodeObject::createObject (type, ledgerIndex, data, hash);
+        }
+
+    private:
+        int64 const m_seedValue;
+    };
+
+public:
+    // Create a predictable batch of objects
+    static void createPredictableBatch (Batch& batch, int startingIndex, int numObjects, int64 seedValue)
+    {
+        batch.reserve (numObjects);
+
+        PredictableObjectFactory factory (seedValue);
+
+        for (int i = 0; i < numObjects; ++i)
+            batch.push_back (factory.createObject (startingIndex + i));
+    }
+
+    // Compare two batches for equality
+    static bool areBatchesEqual (Batch const& lhs, Batch const& rhs)
+    {
+        bool result = true;
+
+        if (lhs.size () == rhs.size ())
+        {
+            for (int i = 0; i < lhs.size (); ++i)
+            {
+                if (! lhs [i]->isCloneOf (rhs [i]))
+                {
+                    result = false;
+                    break;
+                }
+            }
+        }
+        else
+        {
+            result = false;
+        }
+
+        return result;
+    }
+
+    // Store a batch in a backend
+    void storeBatch (Backend& backend, Batch const& batch)
+    {
+        for (int i = 0; i < batch.size (); ++i)
+        {
+            backend.store (batch [i]);
+        }
+    }
+
+    // Get a copy of a batch in a backend
+    void fetchCopyOfBatch (Backend& backend, Batch* pCopy, Batch const& batch)
+    {
+        pCopy->clear ();
+        pCopy->reserve (batch.size ());
+
+        for (int i = 0; i < batch.size (); ++i)
+        {
+            NodeObject::Ptr object;
+
+            Status const status = backend.fetch (
+                batch [i]->getHash ().cbegin (), &object);
+
+            expect (status == ok, "Should be ok");
+
+            if (status == ok)
+            {
+                expect (object != nullptr, "Should not be null");
+
+                pCopy->push_back (object);
+            }
+        }
+    }
+
+    // Store all objects in a batch
+    static void storeBatch (Database& db, Batch const& batch)
+    {
+        for (int i = 0; i < batch.size (); ++i)
+        {
+            NodeObject::Ptr const object (batch [i]);
+
+            Blob data (object->getData ());
+
+            db.store (object->getType (),
+                      object->getIndex (),
+                      data,
+                      object->getHash ());
+        }
+    }
+
+    // Fetch all the hashes in one batch, into another batch.
+    static void fetchCopyOfBatch (Database& db,
+                                  Batch* pCopy,
+                                  Batch const& batch)
+    {
+        pCopy->clear ();
+        pCopy->reserve (batch.size ());
+
+        for (int i = 0; i < batch.size (); ++i)
+        {
+            NodeObject::Ptr object = db.fetch (batch [i]->getHash ());
+
+            if (object != nullptr)
+                pCopy->push_back (object);
+        }
+    }
+
+    TestBase (String name, UnitTest::When when = UnitTest::runNormal)
+        : UnitTest (name, "ripple", when)
+    {
+    }
+};
+
+}
+
+#endif

src/ripple_core/nodestore/tests/TimingTests.cpp

@@ -0,0 +1,126 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+namespace NodeStore
+{
+
+class TimingTests : public TestBase
+{
+public:
+    enum
+    {
+        numObjectsToTest     = 10000
+    };
+
+    TimingTests ()
+        : TestBase ("NodeStoreTiming", UnitTest::runManual)
+    {
+    }
+
+    class Stopwatch
+    {
+    public:
+        Stopwatch ()
+        {
+        }
+
+        void start ()
+        {
+            m_startTime = Time::getHighResolutionTicks ();
+        }
+
+        double getElapsed ()
+        {
+            int64 const now = Time::getHighResolutionTicks();
+
+            return Time::highResolutionTicksToSeconds (now - m_startTime);
+        }
+
+    private:
+        int64 m_startTime;
+    };
+
+    //--------------------------------------------------------------------------
+
+    void testBackend (String type, int64 const seedValue)
+    {
+        DummyScheduler scheduler;
+
+        String s;
+        s << "Testing backend '" << type << "' performance";
+        beginTestCase (s);
+
+        StringPairArray params;
+        File const path (File::createTempFile ("node_db"));
+        params.set ("type", type);
+        params.set ("path", path.getFullPathName ());
+
+        // Create batches
+        NodeStore::Batch batch1;
+        createPredictableBatch (batch1, 0, numObjectsToTest, seedValue);
+        NodeStore::Batch batch2;
+        createPredictableBatch (batch2, 0, numObjectsToTest, seedValue);
+
+        // Open the backend
+        ScopedPointer <Backend> backend (DatabaseImp::createBackend (params, scheduler));
+
+        Stopwatch t;
+
+        // Individual write batch test
+        t.start ();
+        storeBatch (*backend, batch1);
+        s = "";
+        s << "  Single write: " << String (t.getElapsed (), 2) << " seconds";
+        logMessage (s);
+
+        // Bulk write batch test
+        t.start ();
+        backend->storeBatch (batch2);
+        s = "";
+        s << "  Batch write:  " << String (t.getElapsed (), 2) << " seconds";
+        logMessage (s);
+
+        // Read test
+        Batch copy;
+        t.start ();
+        fetchCopyOfBatch (*backend, &copy, batch1);
+        fetchCopyOfBatch (*backend, &copy, batch2);
+        s = "";
+        s << "  Batch read:   " << String (t.getElapsed (), 2) << " seconds";
+        logMessage (s);
+    }
+
+    //--------------------------------------------------------------------------
+
+    void runTest ()
+    {
+        int const seedValue = 50;
+
+        testBackend ("leveldb", seedValue);
+
+    #if RIPPLE_HYPERLEVELDB_AVAILABLE
+        testBackend ("hyperleveldb", seedValue);
+    #endif
+
+    /*
+    #if RIPPLE_MDB_AVAILABLE
+        testBackend ("mdb", seedValue);
+    #endif
+    */
+
+    #if RIPPLE_SOPHIA_AVAILABLE
+        testBackend ("sophia", seedValue);
+    #endif
+
+    /*
+        testBackend ("sqlite", seedValue);
+    */
+    }
+};
+
+static TimingTests timingTests;
+
+}

src/ripple_core/ripple_core.cpp

@@ -19,12 +19,9 @@
 #include <boost/unordered_map.hpp>
 #include <boost/unordered_set.hpp>
 
-// For NodeStore backends
+
-#include "beast/modules/beast_db/beast_db.h"
+
-#include "../ripple_hyperleveldb/ripple_hyperleveldb.h"
+#include "nodestore/NodeStore.cpp"
-#include "../ripple_leveldb/ripple_leveldb.h"
-#include "../ripple_mdb/ripple_mdb.h"
-#include "../ripple/sophia/ripple_sophia.h"
 
 namespace ripple
 {
@@ -37,23 +34,6 @@ namespace ripple
 #include "functional/LoadEvent.cpp"
 #include "functional/LoadMonitor.cpp"
 
-#  include "node/HyperLevelDBBackendFactory.h"
-# include "node/HyperLevelDBBackendFactory.cpp"
-#  include "node/KeyvaDBBackendFactory.h"
-# include "node/KeyvaDBBackendFactory.cpp"
-#  include "node/LevelDBBackendFactory.h"
-# include "node/LevelDBBackendFactory.cpp"
-#  include "node/MemoryBackendFactory.h"
-# include "node/MemoryBackendFactory.cpp"
-#  include "node/NullBackendFactory.h"
-# include "node/NullBackendFactory.cpp"
-#  include "node/MdbBackendFactory.h"
-# include "node/MdbBackendFactory.cpp"
-#  include "node/SophiaBackendFactory.h"
-# include "node/SophiaBackendFactory.cpp"
-#include "node/NodeStore.cpp"
-#include "node/NodeObject.cpp"
-
 #include "peerfinder/PeerFinder.cpp"
 
 }

src/ripple_core/ripple_core.h

@@ -14,6 +14,10 @@
 #include "../ripple_basics/ripple_basics.h"
 #include "../ripple_data/ripple_data.h"
 
+
+
+#include "nodestore/NodeStore.h"
+
 namespace ripple
 {
 
@@ -29,9 +33,6 @@ namespace ripple
 # include "functional/LoadType.h"
 #include "functional/LoadSource.h"
 
-#include "node/NodeObject.h"
-#include "node/NodeStore.h"
-
 #include "peerfinder/PeerFinder.h"
 
 }