more debugging, getting closer

src/ripple/rpc/handlers/Catalogue.cpp

@@ -112,25 +112,26 @@ private:
 
             auto const& info = ledger->info();
 
-            uint64_t closeTime = info.closeTime.time_since_epoch().count();
-            uint64_t parentCloseTime =
-                info.parentCloseTime.time_since_epoch().count();
+            uint64_t closeTime =
+                0xCAFED00DCAFEBABEULL;  // info.closeTime.time_since_epoch().count();
+            uint64_t parentCloseTime = 0xDEADBEEFC001D00D;
+            //                info.parentCloseTime.time_since_epoch().count();
             uint32_t closeTimeResolution = info.closeTimeResolution.count();
 
+            uint64_t drops = info.drops.drops();
+
             // Write ledger header information
             if (!writeToFile(&info.seq, sizeof(info.seq)) ||
-                !writeToFile(&parentCloseTime, sizeof(parentCloseTime)) ||
                 !writeToFile(info.hash.data(), 32) ||
                 !writeToFile(info.txHash.data(), 32) ||
                 !writeToFile(info.accountHash.data(), 32) ||
                 !writeToFile(info.parentHash.data(), 32) ||
-                !writeToFile(&info.drops, sizeof(info.drops)) ||
-                !writeToFile(&info.validated, sizeof(info.validated)) ||
-                !writeToFile(&info.accepted, sizeof(info.accepted)) ||
+                !writeToFile(&drops, sizeof(drops)) ||
                 !writeToFile(&info.closeFlags, sizeof(info.closeFlags)) ||
                 !writeToFile(
                     &closeTimeResolution, sizeof(closeTimeResolution)) ||
-                !writeToFile(&closeTime, sizeof(closeTime)))
+                !writeToFile(&closeTime, sizeof(closeTime)) ||
+                !writeToFile(&parentCloseTime, sizeof(parentCloseTime)))
             {
                 return false;
             }
@@ -356,24 +357,16 @@ doCatalogueLoad(RPC::JsonContext& context)
         uint64_t closeTime = -1;
         uint64_t parentCloseTime = -1;
         uint32_t closeTimeResolution = -1;
+        uint64_t drops = -1;
+
         if (!infile.read(
                 reinterpret_cast<char*>(&info.seq), sizeof(info.seq)) ||
-            !infile.read(
-                reinterpret_cast<char*>(&parentCloseTime),
-                sizeof(parentCloseTime)) ||
             !infile.read(reinterpret_cast<char*>(info.hash.data()), 32) ||
             !infile.read(reinterpret_cast<char*>(info.txHash.data()), 32) ||
             !infile.read(
                 reinterpret_cast<char*>(info.accountHash.data()), 32) ||
             !infile.read(reinterpret_cast<char*>(info.parentHash.data()), 32) ||
-            !infile.read(
-                reinterpret_cast<char*>(&info.drops), sizeof(info.drops)) ||
-            !infile.read(
-                reinterpret_cast<char*>(&info.validated),
-                sizeof(info.validated)) ||
-            !infile.read(
-                reinterpret_cast<char*>(&info.accepted),
-                sizeof(info.accepted)) ||
+            !infile.read(reinterpret_cast<char*>(&drops), sizeof(drops)) ||
             !infile.read(
                 reinterpret_cast<char*>(&info.closeFlags),
                 sizeof(info.closeFlags)) ||
@@ -381,7 +374,10 @@ doCatalogueLoad(RPC::JsonContext& context)
                 reinterpret_cast<char*>(&closeTimeResolution),
                 sizeof(closeTimeResolution)) ||
             !infile.read(
-                reinterpret_cast<char*>(&closeTime), sizeof(closeTime)))
+                reinterpret_cast<char*>(&closeTime), sizeof(closeTime)) ||
+            !infile.read(
+                reinterpret_cast<char*>(&parentCloseTime),
+                sizeof(parentCloseTime)))
         {
             JLOG(context.j.warn()) << "Catalogue load expected but could not "
                                       "read the next ledger header.";
@@ -391,6 +387,8 @@ doCatalogueLoad(RPC::JsonContext& context)
         info.parentCloseTime = time_point{duration{parentCloseTime}};
         info.closeTimeResolution = duration{closeTimeResolution};
 
+        info.drops = drops;
+
         JLOG(context.j.info()) << "Found ledger " << info.seq << "...";
 
         if (info.seq != expected_seq++)
@@ -463,14 +461,13 @@ doCatalogueLoad(RPC::JsonContext& context)
         ledger->stateMap().flushDirty(hotACCOUNT_NODE);
         ledger->txMap().flushDirty(hotTRANSACTION_NODE);
 
-        // Set the ledger as validated
-        ledger->setValidated();
-
         ledger->setAccepted(
             info.closeTime,
             info.closeTimeResolution,
             info.closeFlags & sLCF_NoConsensusTime);
 
+        ledger->setValidated();
+
         // Set the proper close flags
         ledger->setCloseFlags(info.closeFlags);
 

src/ripple/shamap/SHAMapTreeNode.h

@@ -48,6 +48,7 @@ enum SHAMapNodeType : uint8_t {
     tnTRANSACTION_NM = 2,  // transaction, no metadata
     tnTRANSACTION_MD = 3,  // transaction, with metadata
     tnACCOUNT_STATE = 4,
+    tnREMOVE = 254,   // special type to mark deleted nodes in serialization
     tnTERMINAL = 255  // special type to mark the end of a serialization stream
 };
 

src/ripple/shamap/impl/SHAMap.cpp

@@ -1275,15 +1275,13 @@ SHAMap::serializeToStream(
     };
 
     auto serializeRemovedLeaf = [&stream](uint256 const& key) -> bool {
-        // to indicate a node is removed it is written with size 0
+        // to indicate a node is removed it is written with a removal type
+        auto t = SHAMapNodeType::tnREMOVE;
+        stream.write(reinterpret_cast<char const*>(&t), 1);
 
         // write the key
         stream.write(reinterpret_cast<char const*>(key.data()), 32);
 
-        // write the data size 0
-        uint32_t size{0};
-        stream.write(reinterpret_cast<char const*>(&size), 4);
-
         return !stream.fail();
     };
 
@@ -1329,6 +1327,10 @@ SHAMap::serializeToStream(
                     }
                 }
 
+                // write a terminal symbol to indicate the map stream has ended
+                auto t = SHAMapNodeType::tnTERMINAL;
+                stream.write(reinterpret_cast<char const*>(&t), 1);
+
                 return nodeCount;
             }
         }
@@ -1439,29 +1441,35 @@ SHAMap::deserializeFromStream(
                 return false;
             }
 
-            s.read(reinterpret_cast<char*>(&size), 4);
-
-            if (size == 0)
+            if (nodeType == SHAMapNodeType::tnREMOVE)
             {
                 // deletion
                 if (!hasItem(key))
                 {
                     std::cout << "item already missing at delete request "
                               << to_string(key) << "\n";
-                    return false;
+                    // return false;
+                }
+                else
+                {
+                    std::cout << "Successfully deleted " << to_string(key)
+                              << "\n";
+                    delItem(key);
                 }
-
-                std::cout << "Successfully deleted " << to_string(key) << "\n";
-                delItem(key);
                 return true;
             }
 
-            if (hasItem(key))
-            {
-                std::cout << "Removing modified item ahead of replacement "
-                          << to_string(key) << "\n";
-                delItem(key);
-            }
+            s.read(reinterpret_cast<char*>(&size), 4);
+
+            /*
+                        if (hasItem(key))
+                        {
+                            std::cout << "Removing modified item ahead of
+               replacement "
+                                      << to_string(key) << "\n";
+                            delItem(key);
+                        }
+            */
 
             if (s.fail())
             {
@@ -1489,6 +1497,9 @@ SHAMap::deserializeFromStream(
             }
 
             auto item = make_shamapitem(key, makeSlice(data));
+            if (hasItem(key))
+                return updateGiveItem(nodeType, std::move(item));
+
             return addGiveItem(nodeType, std::move(item));
         };