Add FetchPack unit test

Builds/VisualStudio2012/RippleD.vcxproj

@@ -597,12 +597,6 @@
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
     </ClCompile>
-    <ClCompile Include="..\..\src\ripple\rpc\impl\ErrorCodes.cpp">
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
-    </ClCompile>
     <ClCompile Include="..\..\src\ripple\rpc\impl\Handler.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
@@ -1207,6 +1201,12 @@
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
     </ClCompile>
+    <ClCompile Include="..\..\src\ripple_app\shamap\FetchPackTests.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
+    </ClCompile>
     <ClCompile Include="..\..\src\ripple_app\shamap\SHAMap.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
@@ -1249,6 +1249,12 @@
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
     </ClCompile>
+    <ClCompile Include="..\..\src\ripple_app\shamap\RadixMapTest.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
+    </ClCompile>
     <ClCompile Include="..\..\src\ripple_app\shamap\SHAMapTreeNode.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
@@ -2265,7 +2271,6 @@
     <ClInclude Include="..\..\src\ripple\rocksdb\rocksdb\util\string_util.h" />
     <ClInclude Include="..\..\src\ripple\rocksdb\rocksdb\util\testharness.h" />
     <ClInclude Include="..\..\src\ripple\rocksdb\rocksdb\util\testutil.h" />
-    <ClInclude Include="..\..\src\ripple\rpc\api\ErrorCodes.h" />
     <ClInclude Include="..\..\src\ripple\rpc\api\Handler.h" />
     <ClInclude Include="..\..\src\ripple\rpc\api\Manager.h" />
     <ClInclude Include="..\..\src\ripple\rpc\api\Service.h" />
@@ -2411,6 +2416,7 @@
     <ClInclude Include="..\..\src\ripple_app\shamap\SHAMapNode.h" />
     <ClInclude Include="..\..\src\ripple_app\shamap\SHAMapSyncFilter.h" />
     <ClInclude Include="..\..\src\ripple_app\shamap\SHAMapSyncFilters.h" />
+    <ClInclude Include="..\..\src\ripple_app\shamap\RadixMapTest.h" />
     <ClInclude Include="..\..\src\ripple_app\shamap\SHAMapTreeNode.h" />
     <ClInclude Include="..\..\src\ripple_app\tx\AccountSetTransactor.h" />
     <ClInclude Include="..\..\src\ripple_app\tx\ChangeTransactor.h" />

Builds/VisualStudio2012/RippleD.vcxproj.filters

@@ -1386,15 +1386,18 @@
     <ClCompile Include="..\..\src\ripple_app\ledger\LedgerCleaner.cpp">
       <Filter>[2] Old Ripple\ripple_app\ledger</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\src\ripple\rpc\impl\ErrorCodes.cpp">
-      <Filter>[1] Ripple\rpc\impl</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\src\ripple_app\main\CollectorManager.cpp">
       <Filter>[2] Old Ripple\ripple_app\main</Filter>
     </ClCompile>
     <ClCompile Include="..\..\src\ripple_app\peers\NameResolver.cpp">
       <Filter>[2] Old Ripple\ripple_app\peers</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\ripple_app\shamap\FetchPackTests.cpp">
+      <Filter>[2] Old Ripple\ripple_app\shamap</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple_app\shamap\RadixMapTest.cpp">
+      <Filter>[2] Old Ripple\ripple_app\shamap</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\src\ripple_basics\containers\KeyCache.h">
@@ -2859,15 +2862,15 @@
     <ClInclude Include="..\..\src\ripple\algorithm\api\DecayingSample.h">
       <Filter>[1] Ripple\algorithm\api</Filter>
     </ClInclude>
-    <ClInclude Include="..\..\src\ripple\rpc\api\ErrorCodes.h">
-      <Filter>[1] Ripple\rpc\api</Filter>
-    </ClInclude>
     <ClInclude Include="..\..\src\ripple_app\main\CollectorManager.h">
       <Filter>[2] Old Ripple\ripple_app\main</Filter>
     </ClInclude>
     <ClInclude Include="..\..\src\ripple_app\peers\NameResolver.h">
       <Filter>[2] Old Ripple\ripple_app\peers</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\src\ripple_app\shamap\RadixMapTest.h">
+      <Filter>[2] Old Ripple\ripple_app\shamap</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <CustomBuild Include="..\..\src\ripple_data\protocol\ripple.proto">

src/ripple_app/ripple_app_pt2.cpp

@@ -43,3 +43,7 @@ namespace ripple {
 #include "data/DBInit.cpp"
 
 }
+
+# include "shamap/RadixMapTest.h"
+#include "shamap/RadixMapTest.cpp"
+#include "shamap/FetchPackTests.cpp"

src/ripple_app/shamap/FetchPackTests.cpp

@@ -0,0 +1,123 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2012, 2013 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+namespace ripple {
+
+class FetchPackTests : public UnitTest
+{
+public:
+    enum
+    {
+        tableItems = 100,
+        tableItemsExtra = 20
+    };
+
+    typedef boost::unordered_map <uint256, Blob> Map;
+
+    struct TestFilter : SHAMapSyncFilter
+    {
+        TestFilter (Map& map, Journal journal) : mMap (map), mJournal (journal)
+        {
+        }
+
+        void gotNode (bool fromFilter,
+            SHAMapNode const& id, uint256 const& nodeHash,
+                Blob& nodeData, SHAMapTreeNode::TNType type)
+        {
+        }
+
+        bool haveNode (SHAMapNode const& id,
+            uint256 const& nodeHash, Blob& nodeData)
+        {
+            Map::iterator it = mMap.find (nodeHash);
+            if (it == mMap.end ())
+            {
+                mJournal.fatal << "Test filter missing node";
+                return false;
+            }
+            nodeData = it->second;
+            return true;
+        }
+
+        Map& mMap;
+        Journal mJournal;
+    };
+
+
+    void on_fetch (Map& map, uint256 const& hash, Blob const& blob)
+    {
+        Serializer s (blob);
+        expect (s.getSHA512Half() == hash, "Hash mismatch");
+        map.emplace (hash, blob);
+    }
+
+    void runTest ()
+    {
+        using namespace RadixMap;
+
+        beginTestCase ("Build");
+
+        shared_ptr <Table> t1 (beast::make_shared <Table> (smtFREE));
+        add_random_items (tableItems, *t1, random());
+        shared_ptr <Table> t2 (t1->snapShot (true));
+        
+        add_random_items (tableItemsExtra, *t1, random ());
+        add_random_items (tableItemsExtra, *t2, random ());
+
+        // turn t1 into t2
+        Map map;
+        t2->getFetchPack (t1.get(), true, 1000000, boost::bind (
+            &FetchPackTests::on_fetch, this, boost::ref (map), _1, _2));
+        t1->getFetchPack (nullptr, true, 1000000, boost::bind (
+            &FetchPackTests::on_fetch, this, boost::ref (map), _1, _2));
+
+        // try to rebuild t2 from the fetch pack
+        shared_ptr <Table> t3;
+        try
+        {
+            TestFilter filter (map, journal ());
+
+            t3 = beast::make_shared <Table> (smtFREE, t2->getHash () );
+
+            expect (t3->fetchRoot (t2->getHash (), &filter), "unable to get root");
+
+            // everything should be in the pack, no hashes should be needed
+            std::vector<uint256> hashes = t3->getNeededHashes(1, &filter);
+            expect (hashes.empty(), "missing hashes");
+        }
+        catch (...)
+        {
+            journal().fatal << "exception";
+            fail ();
+        }
+
+        expect (t3->getHash () == t2->getHash (), "root hashes do not match");
+        expect (t3->deepCompare (*t2), "failed compare");
+
+        pass ();
+    }
+
+    FetchPackTests () : UnitTest ("FetchPack", "ripple")
+    {
+    }
+};
+
+static FetchPackTests fetchPackTests;
+
+}

src/ripple_app/shamap/RadixMapTest.cpp

@@ -0,0 +1,46 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2012, 2013 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+namespace ripple {
+namespace RadixMap {
+
+shared_ptr <Item> make_random_item (Random& r)
+{
+    Serializer s;
+    for (int d = 0; d < 3; ++d)
+        s.add32 (r.nextInt ());
+    return beast::make_shared <Item> (
+        s.getRIPEMD160().to256(), s.peekData ());
+}
+
+//------------------------------------------------------------------------------
+
+void add_random_items (std::size_t n, Table& t, Random& r)
+{
+    while (n--)
+    {
+        shared_ptr <SHAMapItem> item (
+            make_random_item (r));
+        meets_postcondition (
+            t.addItem (*item, false, false));
+    }
+}
+
+}
+}

src/ripple_app/shamap/RadixMapTest.h

@@ -0,0 +1,39 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2012, 2013 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+namespace ripple {
+namespace RadixMap {
+
+typedef SHAMap Table;
+typedef SHAMapItem Item;
+
+// Utility functions for RadixMap::Table (a.k.a. SHAMap) unit tests
+
+/** Returns a pseudo random Table item. */
+shared_ptr <Item> make_random_item (Random& r);
+
+/** Adds a set of random items to the Table.
+    @param n The number of items to add.
+    @param t The table to add the items to.
+    @param r A pseudo random number generator.
+*/
+void add_random_items (std::size_t n, Table& t, Random& r);
+
+}
+}

src/ripple_app/shamap/SHAMap.cpp

@@ -53,7 +53,6 @@ SHAMap::SHAMap (SHAMapType t, uint256 const& hash,
     , mType (t)
     , m_missing_node_handler (missing_node_handler)
 {
-    // FIXME: Need to acquire root node
     if (t == smtSTATE)
         mTNByID.rehash (STATE_MAP_BUCKETS);
 

src/ripple_app/shamap/SHAMapSync.cpp

@@ -428,7 +428,7 @@ bool SHAMap::deepCompare (SHAMap& other)
         }
         else if (otherNode->getNodeHash () != node->getNodeHash ())
         {
-            WriteLog (lsWARNING, SHAMap) << "node hash mismatch";
+            WriteLog (lsWARNING, SHAMap) << "node hash mismatch " << *node;
             return false;
         }
 
@@ -657,7 +657,7 @@ public:
         return boost::make_shared<SHAMapItem> (s.getRIPEMD160 ().to256 (), s.peekData ());
     }
 
-    static bool confuseMap (SHAMap& map, int count)
+    bool confuseMap (SHAMap& map, int count)
     {
         // add a bunch of random states to a map, then remove them
         // map should be the same
@@ -672,7 +672,8 @@ public:
 
             if (!map.addItem (*item, false, false))
             {
-                WriteLog (lsFATAL, SHAMap) << "Unable to add item to map";
+                journal().fatal <<
+                    "Unable to add item to map";
                 return false;
             }
         }
@@ -681,14 +682,16 @@ public:
         {
             if (!map.delItem (*it))
             {
-                WriteLog (lsFATAL, SHAMap) << "Unable to remove item from map";
+                journal().fatal <<
+                    "Unable to remove item from map";
                 return false;
             }
         }
 
         if (beforeHash != map.getHash ())
         {
-            WriteLog (lsFATAL, SHAMap) << "Hashes do not match";
+            journal().fatal <<
+                "Hashes do not match";
             return false;
         }