Complete NodeStore documentation

* Complete README.md
* Add missing doxygen
* Fix getHitRate bug

Builds/VisualStudio2013/RippleD.vcxproj.filters

@@ -283,15 +283,6 @@
     <Filter Include="[1] Ripple\common\impl">
       <UniqueIdentifier>{6f3085f6-dbe3-4622-a680-682787d0708c}</UniqueIdentifier>
     </Filter>
-    <Filter Include="[1] Ripple\radmap">
-      <UniqueIdentifier>{1b34e7e8-8260-488c-8d09-bdd3b474f9e3}</UniqueIdentifier>
-    </Filter>
-    <Filter Include="[1] Ripple\radmap\api">
-      <UniqueIdentifier>{19de3695-4341-49db-9da2-b220bc9e0149}</UniqueIdentifier>
-    </Filter>
-    <Filter Include="[1] Ripple\radmap\impl">
-      <UniqueIdentifier>{67371f65-f9be-45b1-81e8-c83ef3336e5c}</UniqueIdentifier>
-    </Filter>
     <Filter Include="[1] Ripple\peerfinder\sim">
       <UniqueIdentifier>{c429638b-4572-44e4-a48a-c18fdd094ae1}</UniqueIdentifier>
     </Filter>
@@ -337,6 +328,15 @@
     <Filter Include="[1] Ripple\websocket">
       <UniqueIdentifier>{6967f835-a1b0-47e3-9bd9-6fc8bd0f3df7}</UniqueIdentifier>
     </Filter>
+    <Filter Include="[1] Ripple\proto\radmap">
+      <UniqueIdentifier>{1b34e7e8-8260-488c-8d09-bdd3b474f9e3}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="[1] Ripple\proto\radmap\api">
+      <UniqueIdentifier>{19de3695-4341-49db-9da2-b220bc9e0149}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="[1] Ripple\proto\radmap\impl">
+      <UniqueIdentifier>{67371f65-f9be-45b1-81e8-c83ef3336e5c}</UniqueIdentifier>
+    </Filter>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\..\src\ripple_basics\containers\RangeSet.cpp">
@@ -1330,10 +1330,10 @@
       <Filter>[2] Old Ripple\ripple_app\ledger</Filter>
     </ClCompile>
     <ClCompile Include="..\..\src\ripple\radmap\ripple_radmap.cpp">
-      <Filter>[1] Ripple\radmap</Filter>
+      <Filter>[1] Ripple\proto\radmap</Filter>
     </ClCompile>
     <ClCompile Include="..\..\src\ripple\radmap\impl\BasicFullBelowCache.cpp">
-      <Filter>[1] Ripple\radmap\impl</Filter>
+      <Filter>[1] Ripple\proto\radmap\impl</Filter>
     </ClCompile>
     <ClCompile Include="..\..\src\ripple\common\impl\KeyCache.cpp">
       <Filter>[1] Ripple\common\impl</Filter>
@@ -3057,13 +3057,13 @@
       <Filter>[2] Old Ripple\ripple_basics\containers</Filter>
     </ClInclude>
     <ClInclude Include="..\..\src\ripple\radmap\ripple_radmap.h">
-      <Filter>[1] Ripple\radmap</Filter>
+      <Filter>[1] Ripple\proto\radmap</Filter>
     </ClInclude>
     <ClInclude Include="..\..\src\ripple\radmap\api\BasicFullBelowCache.h">
-      <Filter>[1] Ripple\radmap\api</Filter>
+      <Filter>[1] Ripple\proto\radmap\api</Filter>
     </ClInclude>
     <ClInclude Include="..\..\src\ripple\radmap\api\Tuning.h">
-      <Filter>[1] Ripple\radmap\api</Filter>
+      <Filter>[1] Ripple\proto\radmap\api</Filter>
     </ClInclude>
     <ClInclude Include="..\..\src\ripple\common\KeyCache.h">
       <Filter>[1] Ripple\common</Filter>
@@ -3507,10 +3507,10 @@
       <Filter>[2] Old Ripple\ripple_app\ledger</Filter>
     </None>
     <None Include="..\..\src\ripple\radmap\README.md">
-      <Filter>[1] Ripple\radmap</Filter>
+      <Filter>[1] Ripple\proto\radmap</Filter>
     </None>
     <None Include="..\..\src\ripple\radmap\TODO.md">
-      <Filter>[1] Ripple\radmap</Filter>
+      <Filter>[1] Ripple\proto\radmap</Filter>
     </None>
     <None Include="..\..\src\ripple_rpc\README.md">
       <Filter>[2] Old Ripple\ripple_rpc</Filter>

src/ripple/common/TaggedCache.h

@@ -145,7 +145,8 @@ public:
     float getHitRate ()
     {
         lock_guard lock (m_mutex);
-        return (static_cast<float> (m_hits) * 100) / (1.0f + m_hits + m_misses);
+        auto const total = static_cast<float> (m_hits + m_misses);
+        return m_hits * (100.0f / std::max (1.0f, total));
     }
 
     void clearStats ()

src/ripple_app/node/SqliteFactory.cpp

@@ -194,7 +194,7 @@ public:
 
         statement.bind(1, to_string (object->getHash()));
         statement.bind(2, type);
-        statement.bind(3, object->getIndex());
+        statement.bind(3, object->getLedgerIndex());
         statement.bindStatic(4, object->getData());
     }
 

src/ripple_core/nodestore/README.md

@@ -1,25 +1,102 @@
-
 # NodeStore
 
 ## Introduction
 
-The NodeStore provides an interface that stores, in a persistent database, the collection of
-NodeObject that rippled uses as its primary representation of ledger items.
+A `NodeObject` is a simple object that the Ledger uses to store entries. It is 
+comprised of a type, a hash, a ledger index and a blob. It can be uniquely 
+identified by the hash, which is a 256 bit hash of the blob. The blob is a 
+variable length block of serialized data. The type identifies what the blob 
+contains. The fields are as follows: 
     
-## Module directory structure
+* `mType`
 
-nodestore
-|-api // Public Interface 
-|
-|-backend // Factory classes for various databases 
-|
-|-impl // Private Implementation
-|
-|-test // Unit tests
+ An enumeration that determines what the blob holds. There are four 
+ different types of objects stored. 
 
-The NodeStore class is a simple object that the Ledger uses to store entries. It has a enumeration type, a hash, a ledger index and a Blob which stores arbritary data.
+ * **ledger**	
     	
+   A ledger header.
 
-# Document WIP notes
+ * **transaction**
       
-If the MemoryFactory backend database is used, do we loose persistance?
+   A signed transaction.
+      
+ * **account node**
+        
+   A node in a ledger's account state tree.
+        
+ * **transaction node**
+        
+   A node in a ledger's transaction tree.
+    
+* `mHash`
+
+ A 256-bit hash of the blob.
+
+* `mLedgerIndex`
+      
+ An unsigned integer that uniquely identifies the ledger in which this 
+   NodeObject appears.
+
+* `mData`
+      
+ A blob containing the payload. Stored in the following format.
+ 
+|Byte   |                     |                          |
+|:------|:--------------------|:-------------------------|
+|0...3  |ledger index         |32-bit big endian integer |
+|4...7  |reserved ledger index|32-bit big endian integer |
+|8      |type                 |NodeObjectType enumeration|
+|9...end|data                 |body of the object data   |
+---    
+The `NodeStore` provides an interface that stores, in a persistent database, a 
+collection of NodeObjects that rippled uses as its primary representation of 
+ledger entries. All ledger entries are stored as NodeObjects and as such, need 
+to be persisted between launches. If a NodeObject is accessed and is not in 
+memory, it will be retrieved from the database.
+
+## Backend
+
+The `NodeStore` implementation provides the `Backend` abstract interface, 
+which lets different key/value databases to be chosen at run-time. This allows 
+experimentation with different engines. Improvements in the performance of the 
+NodeStore are a constant area of research. The database can be specified in 
+the configuration file [node_db] section as follows.
+
+One or more lines of key / value pairs
+
+Example:
+```
+type=RocksDB
+path=rocksdb
+compression=1
+```
+Choices for 'type' (not case-sensitive)
+   
+* **HyperLevelDB**
+  
+ An improved version of LevelDB (preferred).
+
+* **LevelDB**
+
+ Google's LevelDB database (deprecated).
+
+* **none**
+
+ Use no backend.
+
+* **RocksDB**
+
+ Facebook's RocksDB database, builds on LevelDB. 
+
+* **SQLite**
+
+ Use SQLite.
+
+'path' speficies where the backend will store its data files.
+
+Choices for 'compression'
+
+* **0** off
+
+* **1** on (default)

src/ripple_core/nodestore/api/Backend.h

@@ -23,7 +23,7 @@
 namespace ripple {
 namespace NodeStore {
 
-/** A backend used for the store.
+/** A backend used for the NodeStore.
 
     The NodeStore uses a swappable backend so that other database systems
     can be tried. Different databases may offer various features such

src/ripple_core/nodestore/api/Database.h

@@ -118,14 +118,17 @@ public:
     */
     virtual int getWriteLoad () = 0;
 
-    // VFALCO TODO Document this.
+    /** Get the positive cache hits to total attempts ratio. */
     virtual float getCacheHitRate () = 0;
 
-    // VFALCO TODO Document this.
-    //        TODO Document the parameter meanings.
+    /** Set the maximum number of entries and maximum cache age for both caches.
+        
+        @param size Number of cache entries (0 = ignore)
+        @param age Maximum cache age in seconds
+    */
     virtual void tune (int size, int age) = 0;
 
-    // VFALCO TODO Document this.
+    /** Remove expired entries from the positive and negative caches. */
     virtual void sweep () = 0;
 };
 

src/ripple_core/nodestore/api/Manager.h

@@ -23,7 +23,17 @@
 namespace ripple {
 namespace NodeStore {
 
-/** Singleton for managing NodeStore factories and back ends. */
+/** Singleton for managing NodeStore factories and back ends.
+    Create a Manager like this:
+    @code
+    NodeStore::DummyScheduler scheduler;
+    std::unique_ptr <NodeStore::Manager> m (NodeStore::make_Manager());
+
+    std::unique_ptr <NodeStore::Database> nodeStore (m->make_Database ("nodeStore",
+        scheduler, LogPartition::getJournal <NodeObject> (), 4,
+        getConfig ().nodeDatabase));
+    @endcode
+*/
 class Manager
 {
 public:
@@ -45,7 +55,7 @@ public:
     virtual std::unique_ptr <Backend> make_Backend (Parameters const& parameters,
         Scheduler& scheduler, beast::Journal journal) = 0;
 
-    /** Construct a node store database.
+    /** Construct a NodeStore database.
 
         The parameters are key value pairs passed to the backend. The
         'type' key must exist, it defines the choice of backend. Most

src/ripple_core/nodestore/api/NodeObject.h

@@ -34,13 +34,11 @@ enum NodeObjectType
     hotTRANSACTION_NODE = 4
 };
 
-/** A blob of data with associated metadata, referenced by hash.
-
-    The metadata includes the following:
-
-    - Type of the blob
-    - The ledger index in which it appears
-    - The SHA 256 hash
+/** A simple object that the Ledger uses to store entries. 
+    NodeObjects are comprised of a type, a hash, a ledger index and a blob. 
+    They can be uniquely identified by the hash, which is a SHA 256 of the 
+    blob. The blob is a variable length block of serialized data. The type 
+    identifies what the blob contains.
 
     @note No checking is performed to make sure the hash matches the data.
     @see SHAMap
@@ -107,8 +105,7 @@ public:
 
     /** Retrieve the ledger index in which this object appears.
     */
-    // VFALCO TODO rename to getLedgerIndex or getLedgerId
-    LedgerIndex getIndex () const;
+    LedgerIndex getLedgerIndex() const;
 
     /** Retrieve the binary data.
     */

src/ripple_core/nodestore/impl/BatchWriter.h

@@ -34,10 +34,6 @@ namespace NodeStore {
 
     @see Scheduler
 */
-// VFALCO NOTE I'm not entirely happy having placed this here,
-//             because whoever needs to use NodeStore certainly doesn't
-//             need to see the implementation details of BatchWriter.
-//
 class BatchWriter : private Task
 {
 public:

src/ripple_core/nodestore/impl/EncodedBlob.cpp

@@ -31,13 +31,13 @@ EncodedBlob::prepare (NodeObject::Ptr const& object)
     m_data.ensureSize (m_size);
 
     // These sizes must be the same!
-    static_bassert (sizeof (std::uint32_t) == sizeof (object->getIndex ()));
+    static_bassert (sizeof (std::uint32_t) == sizeof (object->getLedgerIndex ()));
 
     {
         std::uint32_t* buf = static_cast <std::uint32_t*> (m_data.getData ());
 
-        buf [0] = beast::ByteOrder::swapIfLittleEndian (object->getIndex ());
-        buf [1] = beast::ByteOrder::swapIfLittleEndian (object->getIndex ());
+        buf [0] = beast::ByteOrder::swapIfLittleEndian (object->getLedgerIndex ());
+        buf [1] = beast::ByteOrder::swapIfLittleEndian (object->getLedgerIndex ());
     }
 
     {

src/ripple_core/nodestore/impl/Manager.cpp

@@ -26,11 +26,9 @@ public:
     typedef std::vector <std::unique_ptr <Factory>> List;
     List m_list;
 
-    explicit ManagerImp (std::vector <std::unique_ptr <Factory>> factories)
+    explicit ManagerImp (std::vector <std::unique_ptr <Factory>>&& factories)
         : m_list (std::move (factories))
     {
-        m_list.reserve (20);
-
         add_known_factories ();
     }
 

src/ripple_core/nodestore/impl/NodeObject.cpp

@@ -59,7 +59,7 @@ NodeObject::getHash () const
 }
 
 LedgerIndex
-NodeObject::getIndex () const
+NodeObject::getLedgerIndex () const
 {
     return mLedgerIndex;
 }

src/ripple_core/nodestore/tests/TestBase.h

@@ -162,7 +162,7 @@ public:
             Blob data (object->getData ());
 
             db.store (object->getType (),
-                      object->getIndex (),
+                      object->getLedgerIndex (),
                       std::move (data),
                       object->getHash ());
         }

src/ripple_overlay/impl/PeerImp.h

@@ -1744,8 +1744,8 @@ private:
                         if (obj.has_nodeid ())
                             newObj.set_index (obj.nodeid ());
 
-                        if (!reply.has_seq () && (hObj->getIndex () != 0))
-                            reply.set_seq (hObj->getIndex ());
+                        if (!reply.has_seq () && (hObj->getLedgerIndex () != 0))
+                            reply.set_seq (hObj->getLedgerIndex ());
                     }
                 }
             }