diff --git a/Builds/levelization/results/ordering.txt b/Builds/levelization/results/ordering.txt index 37fdd648e2..eca7fc6dc2 100644 --- a/Builds/levelization/results/ordering.txt +++ b/Builds/levelization/results/ordering.txt @@ -43,6 +43,7 @@ test.consensus > xrpl.basics test.consensus > xrpld.app test.consensus > xrpld.consensus test.consensus > xrpld.ledger +test.consensus > xrpl.json test.core > test.jtx test.core > test.toplevel test.core > test.unit_test diff --git a/docs/consensus.md b/docs/consensus.md index 1b0063663a..4ee5aa70dc 100644 --- a/docs/consensus.md +++ b/docs/consensus.md @@ -558,7 +558,7 @@ struct ConsensusResult ConsensusTimer roundTime; // Indicates state in which consensus ended. Once in the accept phase - // will be either Yes or MovedOn + // will be either Yes or MovedOn or Expired ConsensusState state = ConsensusState::No; }; diff --git a/include/xrpl/protocol/TER.h b/include/xrpl/protocol/TER.h index 0b8b4e618c..06a1b34bd0 100644 --- a/include/xrpl/protocol/TER.h +++ b/include/xrpl/protocol/TER.h @@ -42,7 +42,7 @@ using TERUnderlyingType = int; enum TELcodes : TERUnderlyingType { // Note: Range is stable. // Exact numbers are used in ripple-binary-codec: - // https://github.com/ripple/ripple-binary-codec/blob/master/src/enums/definitions.json + // https://github.com/XRPLF/xrpl.js/blob/main/packages/ripple-binary-codec/src/enums/definitions.json // Use tokens. // -399 .. -300: L Local error (transaction fee inadequate, exceeds local @@ -73,7 +73,7 @@ enum TELcodes : TERUnderlyingType { enum TEMcodes : TERUnderlyingType { // Note: Range is stable. // Exact numbers are used in ripple-binary-codec: - // https://github.com/ripple/ripple-binary-codec/blob/master/src/enums/definitions.json + // https://github.com/XRPLF/xrpl.js/blob/main/packages/ripple-binary-codec/src/enums/definitions.json // Use tokens. // -299 .. -200: M Malformed (bad signature) @@ -148,7 +148,7 @@ enum TEMcodes : TERUnderlyingType { enum TEFcodes : TERUnderlyingType { // Note: Range is stable. // Exact numbers are used in ripple-binary-codec: - // https://github.com/ripple/ripple-binary-codec/blob/master/src/enums/definitions.json + // https://github.com/XRPLF/xrpl.js/blob/main/packages/ripple-binary-codec/src/enums/definitions.json // Use tokens. // -199 .. -100: F @@ -192,7 +192,7 @@ enum TEFcodes : TERUnderlyingType { enum TERcodes : TERUnderlyingType { // Note: Range is stable. // Exact numbers are used in ripple-binary-codec: - // https://github.com/ripple/ripple-binary-codec/blob/master/src/enums/definitions.json + // https://github.com/XRPLF/xrpl.js/blob/main/packages/ripple-binary-codec/src/enums/definitions.json // Use tokens. // -99 .. -1: R Retry diff --git a/src/test/consensus/Consensus_test.cpp b/src/test/consensus/Consensus_test.cpp index 50da44d410..db56ab58c6 100644 --- a/src/test/consensus/Consensus_test.cpp +++ b/src/test/consensus/Consensus_test.cpp @@ -23,6 +23,7 @@ #include #include +#include namespace ripple { namespace test { @@ -40,6 +41,7 @@ public: testShouldCloseLedger() { using namespace std::chrono_literals; + testcase("should close ledger"); // Use default parameters ConsensusParms const p{}; @@ -78,46 +80,102 @@ public: testCheckConsensus() { using namespace std::chrono_literals; + testcase("check consensus"); // Use default parameterss ConsensusParms const p{}; + /////////////// + // Disputes still in doubt + // // Not enough time has elapsed BEAST_EXPECT( ConsensusState::No == - checkConsensus(10, 2, 2, 0, 3s, 2s, p, true, journal_)); + checkConsensus(10, 2, 2, 0, 3s, 2s, false, p, true, journal_)); // If not enough peers have propsed, ensure // more time for proposals BEAST_EXPECT( ConsensusState::No == - checkConsensus(10, 2, 2, 0, 3s, 4s, p, true, journal_)); + checkConsensus(10, 2, 2, 0, 3s, 4s, false, p, true, journal_)); // Enough time has elapsed and we all agree BEAST_EXPECT( ConsensusState::Yes == - checkConsensus(10, 2, 2, 0, 3s, 10s, p, true, journal_)); + checkConsensus(10, 2, 2, 0, 3s, 10s, false, p, true, journal_)); // Enough time has elapsed and we don't yet agree BEAST_EXPECT( ConsensusState::No == - checkConsensus(10, 2, 1, 0, 3s, 10s, p, true, journal_)); + checkConsensus(10, 2, 1, 0, 3s, 10s, false, p, true, journal_)); // Our peers have moved on // Enough time has elapsed and we all agree BEAST_EXPECT( ConsensusState::MovedOn == - checkConsensus(10, 2, 1, 8, 3s, 10s, p, true, journal_)); + checkConsensus(10, 2, 1, 8, 3s, 10s, false, p, true, journal_)); // If no peers, don't agree until time has passed. BEAST_EXPECT( ConsensusState::No == - checkConsensus(0, 0, 0, 0, 3s, 10s, p, true, journal_)); + checkConsensus(0, 0, 0, 0, 3s, 10s, false, p, true, journal_)); // Agree if no peers and enough time has passed. BEAST_EXPECT( ConsensusState::Yes == - checkConsensus(0, 0, 0, 0, 3s, 16s, p, true, journal_)); + checkConsensus(0, 0, 0, 0, 3s, 16s, false, p, true, journal_)); + + // Expire if too much time has passed without agreement + BEAST_EXPECT( + ConsensusState::Expired == + checkConsensus(10, 8, 1, 0, 1s, 19s, false, p, true, journal_)); + + /////////////// + // Stalled + // + // Not enough time has elapsed + BEAST_EXPECT( + ConsensusState::No == + checkConsensus(10, 2, 2, 0, 3s, 2s, true, p, true, journal_)); + + // If not enough peers have propsed, ensure + // more time for proposals + BEAST_EXPECT( + ConsensusState::No == + checkConsensus(10, 2, 2, 0, 3s, 4s, true, p, true, journal_)); + + // Enough time has elapsed and we all agree + BEAST_EXPECT( + ConsensusState::Yes == + checkConsensus(10, 2, 2, 0, 3s, 10s, true, p, true, journal_)); + + // Enough time has elapsed and we don't yet agree, but there's nothing + // left to dispute + BEAST_EXPECT( + ConsensusState::Yes == + checkConsensus(10, 2, 1, 0, 3s, 10s, true, p, true, journal_)); + + // Our peers have moved on + // Enough time has elapsed and we all agree, nothing left to dispute + BEAST_EXPECT( + ConsensusState::Yes == + checkConsensus(10, 2, 1, 8, 3s, 10s, true, p, true, journal_)); + + // If no peers, don't agree until time has passed. + BEAST_EXPECT( + ConsensusState::No == + checkConsensus(0, 0, 0, 0, 3s, 10s, true, p, true, journal_)); + + // Agree if no peers and enough time has passed. + BEAST_EXPECT( + ConsensusState::Yes == + checkConsensus(0, 0, 0, 0, 3s, 16s, true, p, true, journal_)); + + // We are done if there's nothing left to dispute, no matter how much + // time has passed + BEAST_EXPECT( + ConsensusState::Yes == + checkConsensus(10, 8, 1, 0, 1s, 19s, true, p, true, journal_)); } void @@ -125,6 +183,7 @@ public: { using namespace std::chrono_literals; using namespace csf; + testcase("standalone"); Sim s; PeerGroup peers = s.createGroup(1); @@ -149,6 +208,7 @@ public: { using namespace csf; using namespace std::chrono; + testcase("peers agree"); ConsensusParms const parms{}; Sim sim; @@ -186,6 +246,7 @@ public: { using namespace csf; using namespace std::chrono; + testcase("slow peers"); // Several tests of a complete trust graph with a subset of peers // that have significantly longer network delays to the rest of the @@ -351,6 +412,7 @@ public: { using namespace csf; using namespace std::chrono; + testcase("close time disagree"); // This is a very specialized test to get ledgers to disagree on // the close time. It unfortunately assumes knowledge about current @@ -417,6 +479,8 @@ public: { using namespace csf; using namespace std::chrono; + testcase("wrong LCL"); + // Specialized test to exercise a temporary fork in which some peers // are working on an incorrect prior ledger. @@ -589,6 +653,7 @@ public: { using namespace csf; using namespace std::chrono; + testcase("consensus close time rounding"); // This is a specialized test engineered to yield ledgers with different // close times even though the peers believe they had close time @@ -604,9 +669,6 @@ public: PeerGroup fast = sim.createGroup(4); PeerGroup network = fast + slow; - for (Peer* peer : network) - peer->consensusParms = parms; - // Connected trust graph network.trust(network); @@ -692,6 +754,7 @@ public: { using namespace csf; using namespace std::chrono; + testcase("fork"); std::uint32_t numPeers = 10; // Vary overlap between two UNLs @@ -748,6 +811,7 @@ public: { using namespace csf; using namespace std::chrono; + testcase("hub network"); // Simulate a set of 5 validators that aren't directly connected but // rely on a single hub node for communication @@ -835,6 +899,7 @@ public: { using namespace csf; using namespace std::chrono; + testcase("preferred by branch"); // Simulate network splits that are prevented from forking when using // preferred ledger by trie. This is a contrived example that involves @@ -967,6 +1032,7 @@ public: { using namespace csf; using namespace std::chrono; + testcase("pause for laggards"); // Test that validators that jump ahead of the network slow // down. @@ -1052,6 +1118,302 @@ public: BEAST_EXPECT(sim.synchronized()); } + void + testDisputes() + { + testcase("disputes"); + + using namespace csf; + + // Test dispute objects directly + using Dispute = DisputedTx; + + Tx const txTrue{99}; + Tx const txFalse{98}; + Tx const txFollowingTrue{97}; + Tx const txFollowingFalse{96}; + int const numPeers = 100; + ConsensusParms p; + std::size_t peersUnchanged = 0; + + // Three cases: + // 1 proposing, initial vote yes + // 2 proposing, initial vote no + // 3 not proposing, initial vote doesn't matter after the first update, + // use yes + { + Dispute proposingTrue{txTrue.id(), true, numPeers, journal_}; + Dispute proposingFalse{txFalse.id(), false, numPeers, journal_}; + Dispute followingTrue{ + txFollowingTrue.id(), true, numPeers, journal_}; + Dispute followingFalse{ + txFollowingFalse.id(), false, numPeers, journal_}; + BEAST_EXPECT(proposingTrue.ID() == 99); + BEAST_EXPECT(proposingFalse.ID() == 98); + BEAST_EXPECT(followingTrue.ID() == 97); + BEAST_EXPECT(followingFalse.ID() == 96); + + // Create an even split in the peer votes + for (int i = 0; i < numPeers; ++i) + { + BEAST_EXPECT(proposingTrue.setVote(PeerID(i), i < 50)); + BEAST_EXPECT(proposingFalse.setVote(PeerID(i), i < 50)); + BEAST_EXPECT(followingTrue.setVote(PeerID(i), i < 50)); + BEAST_EXPECT(followingFalse.setVote(PeerID(i), i < 50)); + } + // Switch the middle vote to match mine + BEAST_EXPECT(proposingTrue.setVote(PeerID(50), true)); + BEAST_EXPECT(proposingFalse.setVote(PeerID(49), false)); + BEAST_EXPECT(followingTrue.setVote(PeerID(50), true)); + BEAST_EXPECT(followingFalse.setVote(PeerID(49), false)); + + // no changes yet + BEAST_EXPECT(proposingTrue.getOurVote() == true); + BEAST_EXPECT(proposingFalse.getOurVote() == false); + BEAST_EXPECT(followingTrue.getOurVote() == true); + BEAST_EXPECT(followingFalse.getOurVote() == false); + BEAST_EXPECT(!proposingTrue.stalled(p, true, peersUnchanged)); + BEAST_EXPECT(!proposingFalse.stalled(p, true, peersUnchanged)); + BEAST_EXPECT(!followingTrue.stalled(p, false, peersUnchanged)); + BEAST_EXPECT(!followingFalse.stalled(p, false, peersUnchanged)); + + // I'm in the majority, my vote should not change + BEAST_EXPECT(!proposingTrue.updateVote(5, true, p)); + BEAST_EXPECT(!proposingFalse.updateVote(5, true, p)); + BEAST_EXPECT(!followingTrue.updateVote(5, false, p)); + BEAST_EXPECT(!followingFalse.updateVote(5, false, p)); + + BEAST_EXPECT(!proposingTrue.updateVote(10, true, p)); + BEAST_EXPECT(!proposingFalse.updateVote(10, true, p)); + BEAST_EXPECT(!followingTrue.updateVote(10, false, p)); + BEAST_EXPECT(!followingFalse.updateVote(10, false, p)); + + peersUnchanged = 2; + BEAST_EXPECT(!proposingTrue.stalled(p, true, peersUnchanged)); + BEAST_EXPECT(!proposingFalse.stalled(p, true, peersUnchanged)); + BEAST_EXPECT(!followingTrue.stalled(p, false, peersUnchanged)); + BEAST_EXPECT(!followingFalse.stalled(p, false, peersUnchanged)); + + // Right now, the vote is 51%. The requirement is about to jump to + // 65% + BEAST_EXPECT(proposingTrue.updateVote(55, true, p)); + BEAST_EXPECT(!proposingFalse.updateVote(55, true, p)); + BEAST_EXPECT(!followingTrue.updateVote(55, false, p)); + BEAST_EXPECT(!followingFalse.updateVote(55, false, p)); + + BEAST_EXPECT(proposingTrue.getOurVote() == false); + BEAST_EXPECT(proposingFalse.getOurVote() == false); + BEAST_EXPECT(followingTrue.getOurVote() == true); + BEAST_EXPECT(followingFalse.getOurVote() == false); + // 16 validators change their vote to match my original vote + for (int i = 0; i < 16; ++i) + { + auto pTrue = PeerID(numPeers - i - 1); + auto pFalse = PeerID(i); + BEAST_EXPECT(proposingTrue.setVote(pTrue, true)); + BEAST_EXPECT(proposingFalse.setVote(pFalse, false)); + BEAST_EXPECT(followingTrue.setVote(pTrue, true)); + BEAST_EXPECT(followingFalse.setVote(pFalse, false)); + } + // The vote should now be 66%, threshold is 65% + BEAST_EXPECT(proposingTrue.updateVote(60, true, p)); + BEAST_EXPECT(!proposingFalse.updateVote(60, true, p)); + BEAST_EXPECT(!followingTrue.updateVote(60, false, p)); + BEAST_EXPECT(!followingFalse.updateVote(60, false, p)); + + BEAST_EXPECT(proposingTrue.getOurVote() == true); + BEAST_EXPECT(proposingFalse.getOurVote() == false); + BEAST_EXPECT(followingTrue.getOurVote() == true); + BEAST_EXPECT(followingFalse.getOurVote() == false); + + // Threshold jumps to 70% + BEAST_EXPECT(proposingTrue.updateVote(86, true, p)); + BEAST_EXPECT(!proposingFalse.updateVote(86, true, p)); + BEAST_EXPECT(!followingTrue.updateVote(86, false, p)); + BEAST_EXPECT(!followingFalse.updateVote(86, false, p)); + + BEAST_EXPECT(proposingTrue.getOurVote() == false); + BEAST_EXPECT(proposingFalse.getOurVote() == false); + BEAST_EXPECT(followingTrue.getOurVote() == true); + BEAST_EXPECT(followingFalse.getOurVote() == false); + + // 5 more validators change their vote to match my original vote + for (int i = 16; i < 21; ++i) + { + auto pTrue = PeerID(numPeers - i - 1); + auto pFalse = PeerID(i); + BEAST_EXPECT(proposingTrue.setVote(pTrue, true)); + BEAST_EXPECT(proposingFalse.setVote(pFalse, false)); + BEAST_EXPECT(followingTrue.setVote(pTrue, true)); + BEAST_EXPECT(followingFalse.setVote(pFalse, false)); + } + + // The vote should now be 71%, threshold is 70% + BEAST_EXPECT(proposingTrue.updateVote(90, true, p)); + BEAST_EXPECT(!proposingFalse.updateVote(90, true, p)); + BEAST_EXPECT(!followingTrue.updateVote(90, false, p)); + BEAST_EXPECT(!followingFalse.updateVote(90, false, p)); + + BEAST_EXPECT(proposingTrue.getOurVote() == true); + BEAST_EXPECT(proposingFalse.getOurVote() == false); + BEAST_EXPECT(followingTrue.getOurVote() == true); + BEAST_EXPECT(followingFalse.getOurVote() == false); + + // The vote should now be 71%, threshold is 70% + BEAST_EXPECT(!proposingTrue.updateVote(150, true, p)); + BEAST_EXPECT(!proposingFalse.updateVote(150, true, p)); + BEAST_EXPECT(!followingTrue.updateVote(150, false, p)); + BEAST_EXPECT(!followingFalse.updateVote(150, false, p)); + + BEAST_EXPECT(proposingTrue.getOurVote() == true); + BEAST_EXPECT(proposingFalse.getOurVote() == false); + BEAST_EXPECT(followingTrue.getOurVote() == true); + BEAST_EXPECT(followingFalse.getOurVote() == false); + + // The vote should now be 71%, threshold is 70% + BEAST_EXPECT(!proposingTrue.updateVote(190, true, p)); + BEAST_EXPECT(!proposingFalse.updateVote(190, true, p)); + BEAST_EXPECT(!followingTrue.updateVote(190, false, p)); + BEAST_EXPECT(!followingFalse.updateVote(190, false, p)); + + BEAST_EXPECT(proposingTrue.getOurVote() == true); + BEAST_EXPECT(proposingFalse.getOurVote() == false); + BEAST_EXPECT(followingTrue.getOurVote() == true); + BEAST_EXPECT(followingFalse.getOurVote() == false); + + peersUnchanged = 3; + BEAST_EXPECT(!proposingTrue.stalled(p, true, peersUnchanged)); + BEAST_EXPECT(!proposingFalse.stalled(p, true, peersUnchanged)); + BEAST_EXPECT(!followingTrue.stalled(p, false, peersUnchanged)); + BEAST_EXPECT(!followingFalse.stalled(p, false, peersUnchanged)); + + // Threshold jumps to 95% + BEAST_EXPECT(proposingTrue.updateVote(220, true, p)); + BEAST_EXPECT(!proposingFalse.updateVote(220, true, p)); + BEAST_EXPECT(!followingTrue.updateVote(220, false, p)); + BEAST_EXPECT(!followingFalse.updateVote(220, false, p)); + + BEAST_EXPECT(proposingTrue.getOurVote() == false); + BEAST_EXPECT(proposingFalse.getOurVote() == false); + BEAST_EXPECT(followingTrue.getOurVote() == true); + BEAST_EXPECT(followingFalse.getOurVote() == false); + + // 25 more validators change their vote to match my original vote + for (int i = 21; i < 46; ++i) + { + auto pTrue = PeerID(numPeers - i - 1); + auto pFalse = PeerID(i); + BEAST_EXPECT(proposingTrue.setVote(pTrue, true)); + BEAST_EXPECT(proposingFalse.setVote(pFalse, false)); + BEAST_EXPECT(followingTrue.setVote(pTrue, true)); + BEAST_EXPECT(followingFalse.setVote(pFalse, false)); + } + + // The vote should now be 96%, threshold is 95% + BEAST_EXPECT(proposingTrue.updateVote(250, true, p)); + BEAST_EXPECT(!proposingFalse.updateVote(250, true, p)); + BEAST_EXPECT(!followingTrue.updateVote(250, false, p)); + BEAST_EXPECT(!followingFalse.updateVote(250, false, p)); + + BEAST_EXPECT(proposingTrue.getOurVote() == true); + BEAST_EXPECT(proposingFalse.getOurVote() == false); + BEAST_EXPECT(followingTrue.getOurVote() == true); + BEAST_EXPECT(followingFalse.getOurVote() == false); + + for (peersUnchanged = 0; peersUnchanged < 6; ++peersUnchanged) + { + BEAST_EXPECT(!proposingTrue.stalled(p, true, peersUnchanged)); + BEAST_EXPECT(!proposingFalse.stalled(p, true, peersUnchanged)); + BEAST_EXPECT(!followingTrue.stalled(p, false, peersUnchanged)); + BEAST_EXPECT(!followingFalse.stalled(p, false, peersUnchanged)); + } + + for (int i = 0; i < 1; ++i) + { + BEAST_EXPECT(!proposingTrue.updateVote(250 + 10 * i, true, p)); + BEAST_EXPECT(!proposingFalse.updateVote(250 + 10 * i, true, p)); + BEAST_EXPECT(!followingTrue.updateVote(250 + 10 * i, false, p)); + BEAST_EXPECT( + !followingFalse.updateVote(250 + 10 * i, false, p)); + + BEAST_EXPECT(proposingTrue.getOurVote() == true); + BEAST_EXPECT(proposingFalse.getOurVote() == false); + BEAST_EXPECT(followingTrue.getOurVote() == true); + BEAST_EXPECT(followingFalse.getOurVote() == false); + + // true vote has changed recently, so not stalled + BEAST_EXPECT(!proposingTrue.stalled(p, true, 0)); + // remaining votes have been unchanged in so long that we only + // need to hit the second round at 95% to be stalled, regardless + // of peers + BEAST_EXPECT(proposingFalse.stalled(p, true, 0)); + BEAST_EXPECT(followingTrue.stalled(p, false, 0)); + BEAST_EXPECT(followingFalse.stalled(p, false, 0)); + + // true vote has changed recently, so not stalled + BEAST_EXPECT(!proposingTrue.stalled(p, true, peersUnchanged)); + // remaining votes have been unchanged in so long that we only + // need to hit the second round at 95% to be stalled, regardless + // of peers + BEAST_EXPECT(proposingFalse.stalled(p, true, peersUnchanged)); + BEAST_EXPECT(followingTrue.stalled(p, false, peersUnchanged)); + BEAST_EXPECT(followingFalse.stalled(p, false, peersUnchanged)); + } + for (int i = 1; i < 3; ++i) + { + BEAST_EXPECT(!proposingTrue.updateVote(250 + 10 * i, true, p)); + BEAST_EXPECT(!proposingFalse.updateVote(250 + 10 * i, true, p)); + BEAST_EXPECT(!followingTrue.updateVote(250 + 10 * i, false, p)); + BEAST_EXPECT( + !followingFalse.updateVote(250 + 10 * i, false, p)); + + BEAST_EXPECT(proposingTrue.getOurVote() == true); + BEAST_EXPECT(proposingFalse.getOurVote() == false); + BEAST_EXPECT(followingTrue.getOurVote() == true); + BEAST_EXPECT(followingFalse.getOurVote() == false); + + // true vote changed 2 rounds ago, and peers are changing, so + // not stalled + BEAST_EXPECT(!proposingTrue.stalled(p, true, 0)); + // still stalled + BEAST_EXPECT(proposingFalse.stalled(p, true, 0)); + BEAST_EXPECT(followingTrue.stalled(p, false, 0)); + BEAST_EXPECT(followingFalse.stalled(p, false, 0)); + + // true vote changed 2 rounds ago, and peers are NOT changing, + // so stalled + BEAST_EXPECT(proposingTrue.stalled(p, true, peersUnchanged)); + // still stalled + BEAST_EXPECT(proposingFalse.stalled(p, true, peersUnchanged)); + BEAST_EXPECT(followingTrue.stalled(p, false, peersUnchanged)); + BEAST_EXPECT(followingFalse.stalled(p, false, peersUnchanged)); + } + for (int i = 3; i < 5; ++i) + { + BEAST_EXPECT(!proposingTrue.updateVote(250 + 10 * i, true, p)); + BEAST_EXPECT(!proposingFalse.updateVote(250 + 10 * i, true, p)); + BEAST_EXPECT(!followingTrue.updateVote(250 + 10 * i, false, p)); + BEAST_EXPECT( + !followingFalse.updateVote(250 + 10 * i, false, p)); + + BEAST_EXPECT(proposingTrue.getOurVote() == true); + BEAST_EXPECT(proposingFalse.getOurVote() == false); + BEAST_EXPECT(followingTrue.getOurVote() == true); + BEAST_EXPECT(followingFalse.getOurVote() == false); + + BEAST_EXPECT(proposingTrue.stalled(p, true, 0)); + BEAST_EXPECT(proposingFalse.stalled(p, true, 0)); + BEAST_EXPECT(followingTrue.stalled(p, false, 0)); + BEAST_EXPECT(followingFalse.stalled(p, false, 0)); + + BEAST_EXPECT(proposingTrue.stalled(p, true, peersUnchanged)); + BEAST_EXPECT(proposingFalse.stalled(p, true, peersUnchanged)); + BEAST_EXPECT(followingTrue.stalled(p, false, peersUnchanged)); + BEAST_EXPECT(followingFalse.stalled(p, false, peersUnchanged)); + } + } + } + void run() override { @@ -1068,6 +1430,7 @@ public: testHubNetwork(); testPreferredByBranch(); testPauseForLaggards(); + testDisputes(); } }; diff --git a/src/test/csf/Tx.h b/src/test/csf/Tx.h index 6c8dc1496f..066aee2268 100644 --- a/src/test/csf/Tx.h +++ b/src/test/csf/Tx.h @@ -52,7 +52,7 @@ public: { } - ID + ID const& id() const { return id_; diff --git a/src/xrpld/app/misc/NetworkOPs.cpp b/src/xrpld/app/misc/NetworkOPs.cpp index 00029263ce..ce7c956850 100644 --- a/src/xrpld/app/misc/NetworkOPs.cpp +++ b/src/xrpld/app/misc/NetworkOPs.cpp @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -249,6 +250,12 @@ public: beast::get_abstract_clock(), validatorKeys, app_.logs().journal("LedgerConsensus")) + , validatorPK_( + validatorKeys.keys ? validatorKeys.keys->publicKey + : decltype(validatorPK_){}) + , validatorMasterPK_( + validatorKeys.keys ? validatorKeys.keys->masterPublicKey + : decltype(validatorMasterPK_){}) , m_ledgerMaster(ledgerMaster) , m_job_queue(job_queue) , m_standalone(standalone) @@ -732,6 +739,9 @@ private: RCLConsensus mConsensus; + std::optional const validatorPK_; + std::optional const validatorMasterPK_; + ConsensusPhase mLastConsensusPhase; LedgerMaster& m_ledgerMaster; @@ -1917,6 +1927,23 @@ NetworkOPsImp::beginConsensus( bool NetworkOPsImp::processTrustedProposal(RCLCxPeerPos peerPos) { + auto const& peerKey = peerPos.publicKey(); + if (validatorPK_ == peerKey || validatorMasterPK_ == peerKey) + { + // Could indicate a operator misconfiguration where two nodes are + // running with the same validator key configured, so this isn't fatal, + // and it doesn't necessarily indicate peer misbehavior. But since this + // is a trusted message, it could be a very big deal. Either way, we + // don't want to relay the proposal. Note that the byzantine behavior + // detection in handleNewValidation will notify other peers. + UNREACHABLE( + "ripple::NetworkOPsImp::processTrustedProposal : received own " + "proposal"); + JLOG(m_journal.error()) + << "Received a TRUSTED proposal signed with my key from a peer"; + return false; + } + return mConsensus.peerProposal(app_.timeKeeper().closeTime(), peerPos); } diff --git a/src/xrpld/consensus/Consensus.cpp b/src/xrpld/consensus/Consensus.cpp index 0071e1d614..fb57687df0 100644 --- a/src/xrpld/consensus/Consensus.cpp +++ b/src/xrpld/consensus/Consensus.cpp @@ -109,6 +109,7 @@ checkConsensusReached( bool count_self, std::size_t minConsensusPct, bool reachedMax, + bool stalled, std::unique_ptr const& clog) { CLOG(clog) << "checkConsensusReached params: agreeing: " << agreeing @@ -138,6 +139,17 @@ checkConsensusReached( return false; } + // We only get stalled when every disputed transaction unequivocally has 80% + // (minConsensusPct) agreement, either for or against. That is: either under + // 20% or over 80% consensus (repectively "nay" or "yay"). This prevents + // manipulation by a minority of byzantine peers of which transactions make + // the cut to get into the ledger. + if (stalled) + { + CLOG(clog) << "consensus stalled. "; + return true; + } + if (count_self) { ++agreeing; @@ -147,6 +159,7 @@ checkConsensusReached( } std::size_t currentPercentage = (agreeing * 100) / total; + CLOG(clog) << "currentPercentage: " << currentPercentage; bool const ret = currentPercentage >= minConsensusPct; if (ret) @@ -168,6 +181,7 @@ checkConsensus( std::size_t currentFinished, std::chrono::milliseconds previousAgreeTime, std::chrono::milliseconds currentAgreeTime, + bool stalled, ConsensusParms const& parms, bool proposing, beast::Journal j, @@ -181,7 +195,7 @@ checkConsensus( << " minimum duration to reach consensus: " << parms.ledgerMIN_CONSENSUS.count() << "ms" << " max consensus time " << parms.ledgerMAX_CONSENSUS.count() - << "s" + << "ms" << " minimum consensus percentage: " << parms.minCONSENSUS_PCT << ". "; @@ -211,10 +225,12 @@ checkConsensus( proposing, parms.minCONSENSUS_PCT, currentAgreeTime > parms.ledgerMAX_CONSENSUS, + stalled, clog)) { - JLOG(j.debug()) << "normal consensus"; - CLOG(clog) << "reached. "; + JLOG((stalled ? j.warn() : j.debug())) + << "normal consensus" << (stalled ? ", but stalled" : ""); + CLOG(clog) << "reached" << (stalled ? ", but stalled." : "."); return ConsensusState::Yes; } @@ -226,6 +242,7 @@ checkConsensus( false, parms.minCONSENSUS_PCT, currentAgreeTime > parms.ledgerMAX_CONSENSUS, + false, clog)) { JLOG(j.warn()) << "We see no consensus, but 80% of nodes have moved on"; @@ -233,6 +250,19 @@ checkConsensus( return ConsensusState::MovedOn; } + std::chrono::milliseconds const maxAgreeTime = + previousAgreeTime * parms.ledgerABANDON_CONSENSUS_FACTOR; + if (currentAgreeTime > std::clamp( + maxAgreeTime, + parms.ledgerMAX_CONSENSUS, + parms.ledgerABANDON_CONSENSUS)) + { + JLOG(j.warn()) << "consensus taken too long"; + CLOG(clog) << "Consensus taken too long. "; + // Note the Expired result may be overridden by the caller. + return ConsensusState::Expired; + } + // no consensus yet JLOG(j.trace()) << "no consensus"; CLOG(clog) << "No consensus. "; diff --git a/src/xrpld/consensus/Consensus.h b/src/xrpld/consensus/Consensus.h index ac595c1d1d..948c00a8b2 100644 --- a/src/xrpld/consensus/Consensus.h +++ b/src/xrpld/consensus/Consensus.h @@ -31,6 +31,7 @@ #include #include +#include #include #include #include @@ -81,6 +82,10 @@ shouldCloseLedger( last ledger @param currentAgreeTime how long, in milliseconds, we've been trying to agree + @param stalled the network appears to be stalled, where + neither we nor our peers have changed their vote on any disputes in a + while. This is undesirable, and will cause us to end consensus + without 80% agreement. @param parms Consensus constant parameters @param proposing whether we should count ourselves @param j journal for logging @@ -94,6 +99,7 @@ checkConsensus( std::size_t currentFinished, std::chrono::milliseconds previousAgreeTime, std::chrono::milliseconds currentAgreeTime, + bool stalled, ConsensusParms const& parms, bool proposing, beast::Journal j, @@ -574,6 +580,9 @@ private: NetClock::duration closeResolution_ = ledgerDefaultTimeResolution; + ConsensusParms::AvalancheState closeTimeAvalancheState_ = + ConsensusParms::init; + // Time it took for the last consensus round to converge std::chrono::milliseconds prevRoundTime_; @@ -599,6 +608,13 @@ private: std::optional result_; ConsensusCloseTimes rawCloseTimes_; + // The number of calls to phaseEstablish where none of our peers + // have changed any votes on disputed transactions. + std::size_t peerUnchangedCounter_ = 0; + + // The total number of times we have called phaseEstablish + std::size_t establishCounter_ = 0; + //------------------------------------------------------------------------- // Peer related consensus data @@ -696,6 +712,7 @@ Consensus::startRoundInternal( previousLedger_ = prevLedger; result_.reset(); convergePercent_ = 0; + closeTimeAvalancheState_ = ConsensusParms::init; haveCloseTimeConsensus_ = false; openTime_.reset(clock_.now()); currPeerPositions_.clear(); @@ -1351,6 +1368,9 @@ Consensus::phaseEstablish( // can only establish consensus if we already took a stance XRPL_ASSERT(result_, "ripple::Consensus::phaseEstablish : result is set"); + ++peerUnchangedCounter_; + ++establishCounter_; + using namespace std::chrono; ConsensusParms const& parms = adaptor_.parms(); @@ -1417,6 +1437,8 @@ Consensus::closeLedger(std::unique_ptr const& clog) phase_ = ConsensusPhase::establish; JLOG(j_.debug()) << "transitioned to ConsensusPhase::establish"; rawCloseTimes_.self = now_; + peerUnchangedCounter_ = 0; + establishCounter_ = 0; result_.emplace(adaptor_.onClose(previousLedger_, now_, mode_.get())); result_->roundTime.reset(clock_.now()); @@ -1550,16 +1572,11 @@ Consensus::updateOurPositions( } else { - int neededWeight; - - if (convergePercent_ < parms.avMID_CONSENSUS_TIME) - neededWeight = parms.avINIT_CONSENSUS_PCT; - else if (convergePercent_ < parms.avLATE_CONSENSUS_TIME) - neededWeight = parms.avMID_CONSENSUS_PCT; - else if (convergePercent_ < parms.avSTUCK_CONSENSUS_TIME) - neededWeight = parms.avLATE_CONSENSUS_PCT; - else - neededWeight = parms.avSTUCK_CONSENSUS_PCT; + // We don't track rounds for close time, so just pass 0s + auto const [neededWeight, newState] = getNeededWeight( + parms, closeTimeAvalancheState_, convergePercent_, 0, 0); + if (newState) + closeTimeAvalancheState_ = *newState; CLOG(clog) << "neededWeight " << neededWeight << ". "; int participants = currPeerPositions_.size(); @@ -1681,7 +1698,8 @@ Consensus::haveConsensus( } else { - JLOG(j_.debug()) << nodeId << " has " << peerProp.position(); + JLOG(j_.debug()) << "Proposal disagreement: Peer " << nodeId + << " has " << peerProp.position(); ++disagree; } } @@ -1691,6 +1709,17 @@ Consensus::haveConsensus( JLOG(j_.debug()) << "Checking for TX consensus: agree=" << agree << ", disagree=" << disagree; + ConsensusParms const& parms = adaptor_.parms(); + // Stalling is BAD + bool const stalled = haveCloseTimeConsensus_ && + std::ranges::all_of(result_->disputes, + [this, &parms](auto const& dispute) { + return dispute.second.stalled( + parms, + mode_.get() == ConsensusMode::proposing, + peerUnchangedCounter_); + }); + // Determine if we actually have consensus or not result_->state = checkConsensus( prevProposers_, @@ -1699,7 +1728,8 @@ Consensus::haveConsensus( currentFinished, prevRoundTime_, result_->roundTime.read(), - adaptor_.parms(), + stalled, + parms, mode_.get() == ConsensusMode::proposing, j_, clog); @@ -1710,6 +1740,33 @@ Consensus::haveConsensus( return false; } + // Consensus has taken far too long. Drop out of the round. + if (result_->state == ConsensusState::Expired) + { + static auto const minimumCounter = + parms.avalancheCutoffs.size() * parms.avMIN_ROUNDS; + std::stringstream ss; + if (establishCounter_ < minimumCounter) + { + // If each round of phaseEstablish takes a very long time, we may + // "expire" before we've given consensus enough time at each + // avalanche level to actually come to a consensus. In that case, + // keep trying. This should only happen if there are an extremely + // large number of disputes such that each round takes an inordinate + // amount of time. + + ss << "Consensus time has expired in round " << establishCounter_ + << "; continue until round " << minimumCounter << ". " + << Json::Compact{getJson(false)}; + JLOG(j_.error()) << ss.str(); + CLOG(clog) << ss.str() << ". "; + return false; + } + ss << "Consensus expired. " << Json::Compact{getJson(true)}; + JLOG(j_.error()) << ss.str(); + CLOG(clog) << ss.str() << ". "; + leaveConsensus(clog); + } // There is consensus, but we need to track if the network moved on // without us. if (result_->state == ConsensusState::MovedOn) @@ -1802,8 +1859,9 @@ Consensus::createDisputes( { Proposal_t const& peerProp = peerPos.proposal(); auto const cit = acquired_.find(peerProp.position()); - if (cit != acquired_.end()) - dtx.setVote(nodeId, cit->second.exists(txID)); + if (cit != acquired_.end() && + dtx.setVote(nodeId, cit->second.exists(txID))) + peerUnchangedCounter_ = 0; } adaptor_.share(dtx.tx()); @@ -1828,7 +1886,8 @@ Consensus::updateDisputes(NodeID_t const& node, TxSet_t const& other) for (auto& it : result_->disputes) { auto& d = it.second; - d.setVote(node, other.exists(d.tx().id())); + if (d.setVote(node, other.exists(d.tx().id()))) + peerUnchangedCounter_ = 0; } } diff --git a/src/xrpld/consensus/ConsensusParms.h b/src/xrpld/consensus/ConsensusParms.h index a0b6c6be8d..9bfacfb8d6 100644 --- a/src/xrpld/consensus/ConsensusParms.h +++ b/src/xrpld/consensus/ConsensusParms.h @@ -20,8 +20,13 @@ #ifndef RIPPLE_CONSENSUS_CONSENSUS_PARMS_H_INCLUDED #define RIPPLE_CONSENSUS_CONSENSUS_PARMS_H_INCLUDED +#include + #include #include +#include +#include +#include namespace ripple { @@ -43,7 +48,7 @@ struct ConsensusParms This is a safety to protect against very old validations and the time it takes to adjust the close time accuracy window. */ - std::chrono::seconds validationVALID_WALL = std::chrono::minutes{5}; + std::chrono::seconds const validationVALID_WALL = std::chrono::minutes{5}; /** Duration a validation remains current after first observed. @@ -51,33 +56,34 @@ struct ConsensusParms first saw it. This provides faster recovery in very rare cases where the number of validations produced by the network is lower than normal */ - std::chrono::seconds validationVALID_LOCAL = std::chrono::minutes{3}; + std::chrono::seconds const validationVALID_LOCAL = std::chrono::minutes{3}; /** Duration pre-close in which validations are acceptable. The number of seconds before a close time that we consider a validation acceptable. This protects against extreme clock errors */ - std::chrono::seconds validationVALID_EARLY = std::chrono::minutes{3}; + std::chrono::seconds const validationVALID_EARLY = std::chrono::minutes{3}; //! How long we consider a proposal fresh - std::chrono::seconds proposeFRESHNESS = std::chrono::seconds{20}; + std::chrono::seconds const proposeFRESHNESS = std::chrono::seconds{20}; //! How often we force generating a new proposal to keep ours fresh - std::chrono::seconds proposeINTERVAL = std::chrono::seconds{12}; + std::chrono::seconds const proposeINTERVAL = std::chrono::seconds{12}; //------------------------------------------------------------------------- // Consensus durations are relative to the internal Consensus clock and use // millisecond resolution. //! The percentage threshold above which we can declare consensus. - std::size_t minCONSENSUS_PCT = 80; + std::size_t const minCONSENSUS_PCT = 80; //! The duration a ledger may remain idle before closing - std::chrono::milliseconds ledgerIDLE_INTERVAL = std::chrono::seconds{15}; + std::chrono::milliseconds const ledgerIDLE_INTERVAL = + std::chrono::seconds{15}; //! The number of seconds we wait minimum to ensure participation - std::chrono::milliseconds ledgerMIN_CONSENSUS = + std::chrono::milliseconds const ledgerMIN_CONSENSUS = std::chrono::milliseconds{1950}; /** The maximum amount of time to spend pausing for laggards. @@ -86,13 +92,26 @@ struct ConsensusParms * validators don't appear to be offline that are merely waiting for * laggards. */ - std::chrono::milliseconds ledgerMAX_CONSENSUS = std::chrono::seconds{15}; + std::chrono::milliseconds const ledgerMAX_CONSENSUS = + std::chrono::seconds{15}; //! Minimum number of seconds to wait to ensure others have computed the LCL - std::chrono::milliseconds ledgerMIN_CLOSE = std::chrono::seconds{2}; + std::chrono::milliseconds const ledgerMIN_CLOSE = std::chrono::seconds{2}; //! How often we check state or change positions - std::chrono::milliseconds ledgerGRANULARITY = std::chrono::seconds{1}; + std::chrono::milliseconds const ledgerGRANULARITY = std::chrono::seconds{1}; + + //! How long to wait before completely abandoning consensus + std::size_t const ledgerABANDON_CONSENSUS_FACTOR = 10; + + /** + * Maximum amount of time to give a consensus round + * + * Does not include the time to build the LCL, so there is no reason for a + * round to go this long, regardless of how big the ledger is. + */ + std::chrono::milliseconds const ledgerABANDON_CONSENSUS = + std::chrono::seconds{120}; /** The minimum amount of time to consider the previous round to have taken. @@ -104,38 +123,80 @@ struct ConsensusParms twice the interval between proposals (0.7s) divided by the interval between mid and late consensus ([85-50]/100). */ - std::chrono::milliseconds avMIN_CONSENSUS_TIME = std::chrono::seconds{5}; + std::chrono::milliseconds const avMIN_CONSENSUS_TIME = + std::chrono::seconds{5}; //------------------------------------------------------------------------------ // Avalanche tuning // As a function of the percent this round's duration is of the prior round, // we increase the threshold for yes votes to add a transaction to our // position. - - //! Percentage of nodes on our UNL that must vote yes - std::size_t avINIT_CONSENSUS_PCT = 50; - - //! Percentage of previous round duration before we advance - std::size_t avMID_CONSENSUS_TIME = 50; - - //! Percentage of nodes that most vote yes after advancing - std::size_t avMID_CONSENSUS_PCT = 65; - - //! Percentage of previous round duration before we advance - std::size_t avLATE_CONSENSUS_TIME = 85; - - //! Percentage of nodes that most vote yes after advancing - std::size_t avLATE_CONSENSUS_PCT = 70; - - //! Percentage of previous round duration before we are stuck - std::size_t avSTUCK_CONSENSUS_TIME = 200; - - //! Percentage of nodes that must vote yes after we are stuck - std::size_t avSTUCK_CONSENSUS_PCT = 95; + enum AvalancheState { init, mid, late, stuck }; + struct AvalancheCutoff + { + int const consensusTime; + std::size_t const consensusPct; + AvalancheState const next; + }; + //! Map the consensus requirement avalanche state to the amount of time that + //! must pass before moving to that state, the agreement percentage required + //! at that state, and the next state. "stuck" loops back on itself because + //! once we're stuck, we're stuck. + //! This structure allows for "looping" of states if needed. + std::map const avalancheCutoffs{ + // {state, {time, percent, nextState}}, + // Initial state: 50% of nodes must vote yes + {init, {0, 50, mid}}, + // mid-consensus starts after 50% of the previous round time, and + // requires 65% yes + {mid, {50, 65, late}}, + // late consensus starts after 85% time, and requires 70% yes + {late, {85, 70, stuck}}, + // we're stuck after 2x time, requires 95% yes votes + {stuck, {200, 95, stuck}}, + }; //! Percentage of nodes required to reach agreement on ledger close time - std::size_t avCT_CONSENSUS_PCT = 75; + std::size_t const avCT_CONSENSUS_PCT = 75; + + //! Number of rounds before certain actions can happen. + // (Moving to the next avalanche level, considering that votes are stalled + // without consensus.) + std::size_t const avMIN_ROUNDS = 2; + + //! Number of rounds before a stuck vote is considered unlikely to change + //! because voting stalled + std::size_t const avSTALLED_ROUNDS = 4; }; +inline std::pair> +getNeededWeight( + ConsensusParms const& p, + ConsensusParms::AvalancheState currentState, + int percentTime, + std::size_t currentRounds, + std::size_t minimumRounds) +{ + // at() can throw, but the map is built by hand to ensure all valid + // values are available. + auto const& currentCutoff = p.avalancheCutoffs.at(currentState); + // Should we consider moving to the next state? + if (currentCutoff.next != currentState && currentRounds >= minimumRounds) + { + // at() can throw, but the map is built by hand to ensure all + // valid values are available. + auto const& nextCutoff = p.avalancheCutoffs.at(currentCutoff.next); + // See if enough time has passed to move on to the next. + XRPL_ASSERT( + nextCutoff.consensusTime >= currentCutoff.consensusTime, + "ripple::getNeededWeight : next state valid"); + if (percentTime >= nextCutoff.consensusTime) + { + return {nextCutoff.consensusPct, currentCutoff.next}; + } + } + return {currentCutoff.consensusPct, {}}; +} + } // namespace ripple #endif diff --git a/src/xrpld/consensus/ConsensusTypes.h b/src/xrpld/consensus/ConsensusTypes.h index dc7fb69bf4..ccb421abed 100644 --- a/src/xrpld/consensus/ConsensusTypes.h +++ b/src/xrpld/consensus/ConsensusTypes.h @@ -188,6 +188,7 @@ struct ConsensusCloseTimes enum class ConsensusState { No, //!< We do not have consensus MovedOn, //!< The network has consensus without us + Expired, //!< Consensus time limit has hard-expired Yes //!< We have consensus along with the network }; @@ -237,7 +238,7 @@ struct ConsensusResult ConsensusTimer roundTime; // Indicates state in which consensus ended. Once in the accept phase - // will be either Yes or MovedOn + // will be either Yes or MovedOn or Expired ConsensusState state = ConsensusState::No; // The number of peers proposing during the round diff --git a/src/xrpld/consensus/DisputedTx.h b/src/xrpld/consensus/DisputedTx.h index cd7e17d653..513f240829 100644 --- a/src/xrpld/consensus/DisputedTx.h +++ b/src/xrpld/consensus/DisputedTx.h @@ -82,6 +82,51 @@ public: return ourVote_; } + //! Are we and our peers "stalled" where we probably won't change + //! our vote? + bool + stalled(ConsensusParms const& p, bool proposing, int peersUnchanged) const + { + // at() can throw, but the map is built by hand to ensure all valid + // values are available. + auto const& currentCutoff = p.avalancheCutoffs.at(avalancheState_); + auto const& nextCutoff = p.avalancheCutoffs.at(currentCutoff.next); + + // We're have not reached the final avalanche state, or been there long + // enough, so there's room for change. Check the times in case the state + // machine is altered to allow states to loop. + if (nextCutoff.consensusTime > currentCutoff.consensusTime || + avalancheCounter_ < p.avMIN_ROUNDS) + return false; + + // We've haven't had this vote for minimum rounds yet. Things could + // change. + if (proposing && currentVoteCounter_ < p.avMIN_ROUNDS) + return false; + + // If we or any peers have changed a vote in several rounds, then + // things could still change. But if _either_ has not changed in that + // long, we're unlikely to change our vote any time soon. (This prevents + // a malicious peer from flip-flopping a vote to prevent consensus.) + if (peersUnchanged < p.avSTALLED_ROUNDS && + (proposing && currentVoteCounter_ < p.avSTALLED_ROUNDS)) + return false; + + // Does this transaction have more than 80% agreement + + // Compute the percentage of nodes voting 'yes' (possibly including us) + int const support = (yays_ + (proposing && ourVote_ ? 1 : 0)) * 100; + int total = nays_ + yays_ + (proposing ? 1 : 0); + if (!total) + // There are no votes, so we know nothing + return false; + int const weight = support / total; + // Returns true if the tx has more than minCONSENSUS_PCT (80) percent + // agreement. Either voting for _or_ voting against the tx. + return weight > p.minCONSENSUS_PCT || + weight < (100 - p.minCONSENSUS_PCT); + } + //! The disputed transaction. Tx_t const& tx() const @@ -100,8 +145,12 @@ public: @param peer Identifier of peer. @param votesYes Whether peer votes to include the disputed transaction. + + @return bool Whether the peer changed its vote. (A new vote counts as a + change.) */ - void + [[nodiscard]] + bool setVote(NodeID_t const& peer, bool votesYes); /** Remove a peer's vote @@ -135,12 +184,18 @@ private: bool ourVote_; //< Our vote (true is yes) Tx_t tx_; //< Transaction under dispute Map_t votes_; //< Map from NodeID to vote + //! The number of rounds we've gone without changing our vote + std::size_t currentVoteCounter_ = 0; + //! Which minimum acceptance percentage phase we are currently in + ConsensusParms::AvalancheState avalancheState_ = ConsensusParms::init; + //! How long we have been in the current acceptance phase + std::size_t avalancheCounter_ = 0; beast::Journal const j_; }; // Track a peer's yes/no vote on a particular disputed tx_ template -void +bool DisputedTx::setVote(NodeID_t const& peer, bool votesYes) { auto const [it, inserted] = votes_.insert(std::make_pair(peer, votesYes)); @@ -158,6 +213,7 @@ DisputedTx::setVote(NodeID_t const& peer, bool votesYes) JLOG(j_.debug()) << "Peer " << peer << " votes NO on " << tx_.id(); ++nays_; } + return true; } // changes vote to yes else if (votesYes && !it->second) @@ -166,6 +222,7 @@ DisputedTx::setVote(NodeID_t const& peer, bool votesYes) --nays_; ++yays_; it->second = true; + return true; } // changes vote to no else if (!votesYes && it->second) @@ -174,7 +231,9 @@ DisputedTx::setVote(NodeID_t const& peer, bool votesYes) ++nays_; --yays_; it->second = false; + return true; } + return false; } // Remove a peer's vote on this disputed transaction @@ -211,21 +270,26 @@ DisputedTx::updateVote( bool newPosition; int weight; + // When proposing, to prevent avalanche stalls, we increase the needed + // weight slightly over time. We also need to ensure that the consensus has + // made a minimum number of attempts at each "state" before moving + // to the next. + // Proposing or not, we need to keep track of which state we've reached so + // we can determine if the vote has stalled. + auto const [requiredPct, newState] = getNeededWeight( + p, avalancheState_, percentTime, ++avalancheCounter_, p.avMIN_ROUNDS); + if (newState) + { + avalancheState_ = *newState; + avalancheCounter_ = 0; + } + if (proposing) // give ourselves full weight { // This is basically the percentage of nodes voting 'yes' (including us) weight = (yays_ * 100 + (ourVote_ ? 100 : 0)) / (nays_ + yays_ + 1); - // To prevent avalanche stalls, we increase the needed weight slightly - // over time. - if (percentTime < p.avMID_CONSENSUS_TIME) - newPosition = weight > p.avINIT_CONSENSUS_PCT; - else if (percentTime < p.avLATE_CONSENSUS_TIME) - newPosition = weight > p.avMID_CONSENSUS_PCT; - else if (percentTime < p.avSTUCK_CONSENSUS_TIME) - newPosition = weight > p.avLATE_CONSENSUS_PCT; - else - newPosition = weight > p.avSTUCK_CONSENSUS_PCT; + newPosition = weight > requiredPct; } else { @@ -236,13 +300,16 @@ DisputedTx::updateVote( if (newPosition == ourVote_) { - JLOG(j_.info()) << "No change (" << (ourVote_ ? "YES" : "NO") - << ") : weight " << weight << ", percent " - << percentTime; + ++currentVoteCounter_; + JLOG(j_.info()) << "No change (" << (ourVote_ ? "YES" : "NO") << ") on " + << tx_.id() << " : weight " << weight << ", percent " + << percentTime + << ", round(s) with this vote: " << currentVoteCounter_; JLOG(j_.debug()) << Json::Compact{getJson()}; return false; } + currentVoteCounter_ = 0; ourVote_ = newPosition; JLOG(j_.debug()) << "We now vote " << (ourVote_ ? "YES" : "NO") << " on " << tx_.id();