Add support for deterministic database shards (#2688):

This commit, if merged, adds support to allow multiple indepedent nodes to
produce a binary identical shard for a given range of ledgers. The advantage
is that servers can use content-addressable storage, and can more efficiently
retrieve shards by downloading from multiple peers at once and then verifying
the integrity of a shard by cross-checking its checksum with the checksum
other servers report.

src/ripple/nodestore/impl/Shard.cpp

@@ -40,6 +40,7 @@ Shard::Shard(
     std::uint32_t index,
     beast::Journal j)
     : app_(app)
+    , db_(db)
     , index_(index)
     , firstSeq_(db.firstLedgerSeq(index))
     , lastSeq_(std::max(firstSeq_, db.lastLedgerSeq(index)))
@@ -397,7 +398,8 @@ Shard::isLegacy() const
 bool
 Shard::finalize(
     bool const writeSQLite,
-    boost::optional<uint256> const& expectedHash)
+    boost::optional<uint256> const& expectedHash,
+    const bool writeDeterministicShard)
 {
     assert(backend_);
 
@@ -511,6 +513,17 @@ Shard::finalize(
     std::shared_ptr<Ledger const> next;
     auto const lastLedgerHash{hash};
 
+    std::shared_ptr<DeterministicShard> dsh;
+    if (writeDeterministicShard)
+    {
+        dsh = std::make_shared<DeterministicShard>(
+            app_, db_, index_, lastLedgerHash, j_);
+        if (!dsh->init())
+        {
+            return fail("can't create deterministic shard");
+        }
+    }
+
     // Start with the last ledger in the shard and walk backwards from
     // child to parent until we reach the first ledger
     seq = lastSeq_;
@@ -547,8 +560,11 @@ Shard::finalize(
             return fail("missing root TXN node");
         }
 
-        if (!valLedger(ledger, next))
-            return fail("failed to validate ledger");
+        if (dsh)
+            dsh->store(nObj);
+
+        if (!verifyLedger(ledger, next, dsh))
+            return fail("verification check failed");
 
         if (writeSQLite)
         {
@@ -609,6 +625,12 @@ Shard::finalize(
     {
         backend_->store(nObj);
 
+        if (dsh)
+        {
+            dsh->store(nObj);
+            dsh->flush();
+        }
+
         std::lock_guard lock(mutex_);
         final_ = true;
 
@@ -628,6 +650,23 @@ Shard::finalize(
             std::string("exception ") + e.what() + " in function " + __func__);
     }
 
+    if (dsh)
+    {
+        /* Close non-deterministic shard database. */
+        backend_->close();
+        /* Replace non-deterministic shard by deterministic one. */
+        dsh->close();
+        /* Re-open deterministic shard database. */
+        backend_->open(false);
+        /** The finalize() function verifies the shard and, if third parameter
+         *  is true, then replaces the shard by deterministic copy of the shard.
+         *  After deterministic shard is created it verifies again,
+         *  the finalize() function called here to verify deterministic shard,
+         *  third parameter is false.
+         */
+        return finalize(false, expectedHash, false);
+    }
+
     return true;
 }
 
@@ -922,9 +961,10 @@ Shard::setFileStats(std::lock_guard<std::recursive_mutex> const&)
 }
 
 bool
-Shard::valLedger(
+Shard::verifyLedger(
     std::shared_ptr<Ledger const> const& ledger,
-    std::shared_ptr<Ledger const> const& next) const
+    std::shared_ptr<Ledger const> const& next,
+    std::shared_ptr<DeterministicShard> dsh) const
 {
     auto fail = [j = j_, index = index_, &ledger](std::string const& msg) {
         JLOG(j.fatal()) << "shard " << index << ". " << msg
@@ -943,11 +983,14 @@ Shard::valLedger(
         return fail("Invalid ledger account hash");
 
     bool error{false};
-    auto visit = [this, &error](SHAMapAbstractNode& node) {
+    auto visit = [this, &error, dsh](SHAMapAbstractNode& node) {
         if (stop_)
             return false;
-        if (!valFetch(node.getNodeHash().as_uint256()))
+        auto nObj = valFetch(node.getNodeHash().as_uint256());
+        if (!nObj)
             error = true;
+        else if (dsh)
+            dsh->store(nObj);
         return !error;
     };