Pause for lagging validators.

src/test/consensus/Consensus_test.cpp

@@ -996,6 +996,125 @@ public:
         }
     }
 
+    // Helper collector for testPauseForLaggards
+    // This will remove the ledgerAccept delay used to
+    // initially create the slow vs. fast validator groups.
+    struct UndoDelay
+    {
+        csf::PeerGroup& g;
+
+        UndoDelay(csf::PeerGroup& a) : g(a)
+        {
+        }
+
+        template <class E>
+        void
+        on(csf::PeerID, csf::SimTime, E const&)
+        {
+        }
+
+        void
+        on(csf::PeerID who, csf::SimTime, csf::AcceptLedger const& e)
+        {
+            for (csf::Peer* p : g)
+            {
+                if (p->id == who)
+                    p->delays.ledgerAccept = std::chrono::seconds{0};
+            }
+        }
+    };
+
+    void
+    testPauseForLaggards()
+    {
+        using namespace csf;
+        using namespace std::chrono;
+
+        // Test that validators that jump ahead of the network slow
+        // down.
+
+        // We engineer the following validated ledger history scenario:
+        //
+        //  / --> B1 --> C1 --> ... -> G1  "ahead"
+        // A
+        //  \ --> B2 --> C2 "behind"
+        //
+        // After validating a common ledger A, a set of "behind" validators
+        // briefly run slower and validate the lower chain of ledgers.
+        // The "ahead" validators run normal speed and run ahead validating the
+        // upper chain of ledgers.
+        //
+        // Due to the uncommited support definition of the preferred branch
+        // protocol, even if the "behind" validators are a majority, the "ahead"
+        // validators cannot jump to the proper branch until the "behind"
+        // validators catch up to the same sequence number. For this test to
+        // succeed, the ahead validators need to briefly slow down consensus.
+
+        ConsensusParms const parms{};
+        Sim sim;
+        SimDuration delay =
+            date::round<milliseconds>(0.2 * parms.ledgerGRANULARITY);
+
+        PeerGroup behind = sim.createGroup(3);
+        PeerGroup ahead = sim.createGroup(2);
+        PeerGroup network = ahead + behind;
+
+        hash_set<Peer::NodeKey_t> trustedKeys;
+        for (Peer* p : network)
+            trustedKeys.insert(p->key);
+        for (Peer* p : network)
+            p->trustedKeys = trustedKeys;
+
+        network.trustAndConnect(network, delay);
+
+        // Initial seed round to set prior state
+        sim.run(1);
+
+        // Have the "behind" group initially take a really long time to
+        // accept a ledger after ending deliberation
+        for (Peer* p : behind)
+            p->delays.ledgerAccept = 20s;
+
+        // Use the collector to revert the delay after the single
+        // slow ledger is generated
+        UndoDelay undoDelay{behind};
+        sim.collectors.add(undoDelay);
+
+#if 0
+        // Have all beast::journal output printed to stdout
+        for (Peer* p : network)
+            p->sink.threshold(beast::severities::kAll);
+
+        // Print ledger accept and fully validated events to stdout
+        StreamCollector sc{std::cout};
+        sim.collectors.add(sc);
+#endif
+        // Run the simulation for 100 seconds of simulation time with
+        std::chrono::nanoseconds const simDuration = 100s;
+
+        // Simulate clients submitting 1 tx every 5 seconds to a random
+        // validator
+        Rate const rate{1, 5s};
+        auto peerSelector = makeSelector(
+            network.begin(),
+            network.end(),
+            std::vector<double>(network.size(), 1.),
+            sim.rng);
+        auto txSubmitter = makeSubmitter(
+            ConstantDistribution{rate.inv()},
+            sim.scheduler.now(),
+            sim.scheduler.now() + simDuration,
+            peerSelector,
+            sim.scheduler,
+            sim.rng);
+
+        // Run simulation
+        sim.run(simDuration);
+
+        // Verify that the network recovered
+        BEAST_EXPECT(sim.synchronized());
+    }
+
     void
     run() override
     {
@@ -1011,6 +1130,7 @@ public:
         testFork();
         testHubNetwork();
         testPreferredByBranch();
+        testPauseForLaggards();
     }
 };
 

src/test/csf/Peer.h

@@ -22,6 +22,7 @@
 #include <ripple/beast/utility/WrappedSink.h>
 #include <ripple/consensus/Consensus.h>
 #include <ripple/consensus/Validations.h>
+#include <ripple/protocol/PublicKey.h>
 #include <boost/container/flat_map.hpp>
 #include <boost/container/flat_set.hpp>
 #include <algorithm>
@@ -165,9 +166,11 @@ struct Peer
     //! Type definitions for generic consensus
     using Ledger_t = Ledger;
     using NodeID_t = PeerID;
+    using NodeKey_t = PeerKey;
     using TxSet_t = TxSet;
     using PeerPosition_t = Position;
     using Result = ConsensusResult<Peer>;
+    using NodeKey = Validation::NodeKey;
 
     //! Logging support that prefixes messages with the peer ID
     beast::WrappedSink sink;
@@ -239,10 +242,7 @@ struct Peer
     //! Whether to simulate running as validator or a tracking node
     bool runAsValidator = true;
 
-    //! Enforce invariants on validation sequence numbers
-    SeqEnforcer<Ledger::Seq> seqEnforcer;
-
-    //TODO: Consider removing these two, they are only a convenience for tests
+    // TODO: Consider removing these two, they are only a convenience for tests
     // Number of proposers in the prior round
     std::size_t prevProposers = 0;
     // Duration of prior round
@@ -252,6 +252,8 @@ struct Peer
     // TODO: Use the logic in ValidatorList to set this dynamically
     std::size_t quorum = 0;
 
+    hash_set<NodeKey_t> trustedKeys;
+
     // Simulation parameters
     ConsensusParms consensusParms;
 
@@ -574,8 +576,7 @@ struct Peer
 
             // Can only send one validated ledger per seq
             if (runAsValidator && isCompatible && !consensusFail &&
-                seqEnforcer(
-                    scheduler.now(), newLedger.seq(), validations.parms()))
+                validations.canValidateSeq(newLedger.seq()))
             {
                 bool isFull = proposing;
 
@@ -837,6 +838,39 @@ struct Peer
         return addTrustedValidation(v);
     }
 
+    bool
+    haveValidated() const
+    {
+        return fullyValidatedLedger.seq() > Ledger::Seq{0};
+    }
+
+    Ledger::Seq
+    getValidLedgerIndex() const
+    {
+        return earliestAllowedSeq();
+    }
+
+    std::pair<std::size_t, hash_set<NodeKey_t>>
+    getQuorumKeys()
+    {
+        hash_set<NodeKey_t > keys;
+        for (auto const& p : trustGraph.trustedPeers(this))
+            keys.insert(p->key);
+        return {quorum, keys};
+    }
+
+    std::size_t
+    laggards(Ledger::Seq const seq, hash_set<NodeKey_t>& trustedKeys)
+    {
+        return validations.laggards(seq, trustedKeys);
+    }
+
+    bool
+    validator() const
+    {
+        return runAsValidator;
+    }
+
     //--------------------------------------------------------------------------
     //  A locally submitted transaction
     void