From 00c60d408a887d8a986db81afbb5ead121e8310c Mon Sep 17 00:00:00 2001 From: Brad Chase Date: Fri, 3 Feb 2017 12:31:08 -0500 Subject: [PATCH] Improve Consensus interface and documentation (RIPD-1340): - Add Consensus::Result, which represents the result of the establish state and includes the consensus transaction set, final proposed position and disputes. - Add Consensus::Mode to track how we are participating in consensus and ensures the onAccept callback can distinguish when we entered the round with consensus versus when we recovered from a wrong ledger during a round. - Rename Consensus::Phase to Consensus::State and eliminate the processing phase. Instead, accept is a terminal phase which notifies RCLConsensus via onAccept callbacks. Even if clients dispatch accepting to another thread, all future calls except to startRound will not change the state of consensus. - Move validate_ status from Consensus to RCLConsensus, since generic implementation does not directly reference whether a node is validating or not. - Eliminate gotTxSetInternal and handle externally received TxSets distinct from locally generated positions. - Change ConsensusProposal::changePosition to always update the internal close time and position even if we have bowed out. This enforces the invariant that our proposal's position always matches our transaction set. --- CMakeLists.txt | 1 + docs/Jamfile.v2 | 12 +- docs/consensus.qbk | 663 +++++++ docs/images/consensus/EffCloseTime.png | Bin 0 -> 1975 bytes docs/images/consensus/block_chain.png | Bin 0 -> 5217 bytes docs/images/consensus/consensus_modes.png | Bin 0 -> 14411 bytes docs/images/consensus/consensus_overview.png | Bin 0 -> 12297 bytes docs/images/consensus/disputes.png | Bin 0 -> 14639 bytes docs/images/consensus/ledger_chain.png | Bin 0 -> 10403 bytes docs/images/consensus/threshold.png | Bin 0 -> 6074 bytes docs/main.qbk | 6 +- src/ripple/app/consensus/RCLConsensus.cpp | 810 ++++---- src/ripple/app/consensus/RCLConsensus.h | 352 ++-- src/ripple/app/consensus/RCLCxLedger.h | 14 +- src/ripple/app/consensus/RCLCxPeerPos.cpp | 59 +- src/ripple/app/consensus/RCLCxPeerPos.h | 60 +- src/ripple/app/consensus/RCLCxTx.h | 50 +- src/ripple/app/misc/NetworkOPs.cpp | 16 +- src/ripple/app/misc/NetworkOPs.h | 2 +- src/ripple/consensus/Consensus.h | 1754 ++++++++---------- src/ripple/consensus/ConsensusProposal.h | 106 +- src/ripple/consensus/DisputedTx.h | 116 +- src/ripple/consensus/LedgerTiming.cpp | 77 +- src/ripple/consensus/LedgerTiming.h | 82 +- src/test/consensus/Consensus_test.cpp | 293 +-- src/test/csf/BasicNetwork.h | 385 ++-- src/test/csf/BasicNetwork_test.cpp | 83 +- src/test/csf/Ledger.h | 51 +- src/test/csf/Peer.h | 290 ++- src/test/csf/Sim.h | 28 +- src/test/csf/Tx.h | 101 +- src/test/csf/UNL.h | 107 +- src/test/csf/impl/UNL.cpp | 56 +- 33 files changed, 2935 insertions(+), 2639 deletions(-) create mode 100644 docs/consensus.qbk create mode 100644 docs/images/consensus/EffCloseTime.png create mode 100644 docs/images/consensus/block_chain.png create mode 100644 docs/images/consensus/consensus_modes.png create mode 100644 docs/images/consensus/consensus_overview.png create mode 100644 docs/images/consensus/disputes.png create mode 100644 docs/images/consensus/ledger_chain.png create mode 100644 docs/images/consensus/threshold.png diff --git a/CMakeLists.txt b/CMakeLists.txt index 2d8e17bac..407fbf89e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -353,6 +353,7 @@ if (WIN32 OR is_xcode) docs/ Jamfile.v2 boostbook.dtd + consensus.qbk index.xml main.qbk quickref.xml diff --git a/docs/Jamfile.v2 b/docs/Jamfile.v2 index 9946f10bf..98a025e63 100644 --- a/docs/Jamfile.v2 +++ b/docs/Jamfile.v2 @@ -44,6 +44,15 @@ install callouts explicit callout ; +install consensus_images + : + [ glob images/consensus/*.png ] + : + $(out)/html/images/consensus + ; + +explicit consensus_images ; + xml doc : main.qbk @@ -60,7 +69,7 @@ boostbook boostdoc boost.root=$(broot) chunk.first.sections=1 # Chunk the first top-level section? chunk.section.depth=8 # Depth to which sections should be chunked - generate.section.toc.level=1 # Control depth of TOC generation in sections + generate.section.toc.level=2 # Control depth of TOC generation in sections toc.max.depth=2 # How many levels should be created for each TOC? toc.section.depth=2 # How deep should recursive sections appear in the TOC? generate.toc="chapter toc section toc" @@ -68,4 +77,5 @@ boostbook boostdoc temp stylesheets images + consensus_images ; diff --git a/docs/consensus.qbk b/docs/consensus.qbk new file mode 100644 index 000000000..8077f97be --- /dev/null +++ b/docs/consensus.qbk @@ -0,0 +1,663 @@ +[section Consensus and Validation] + +[*This section is a work in progress!!] + +Consensus is the task of reaching agreement within a distributed system in the +presence of faulty or even malicious participants. This document outlines the +[@https://ripple.com/files/ripple/consensus/whitepaper.pdf Ripple Consensus +Algorithm] as implemented in [@https://github.com/ripple/rippled rippled], but +focuses on its utility as a generic consensus algorithm independent of the +detailed mechanics of the Ripple Consensus Ledger. Most notably, the algorithm +does not require fully synchronous communication between all nodes in the +network, or even a fixed network topology, but instead achieves consensus via +collectively trusted subnetworks. + +[heading Distributed Agreement] + +A challenge for distributed systems is reaching agreement on changes in shared +state. For the Ripple network, the shared state is the current ledger--account +information, account balances, order books and other financial data. We will +refer to shared distributed state as a /ledger/ throughout the remainder of this +document. + +[$images/consensus/ledger_chain.png [width 50%] [height 50%] ] + +As shown above, new ledgers are made by applying a set of transactions to the +prior ledger. For the Ripple network, transactions include payments, +modification of account settings, updates to offers and more. + +In a centralized system, generating the next ledger is trivial since there is a +single unique arbiter of which transactions to include and how to apply them to +a ledger. For decentralized systems, participants must resolve disagreements on +the set of transactions to include, the order to apply those transactions, and +even the resulting ledger after applying the transactions. This is even more +difficult when some participants are faulty or malicious. + +The Ripple network is a decentralized and _trust-full_ network. Anyone is free +to join and participants are free to choose a subset of peers that are +collectively trusted to not collude in an attempt to defraud the participant. +Leveraging this network of trust, the Ripple algorithm has two main components. + +* /Consensus/ in which network participants agree on the transactions to apply + to a prior ledger, based on the positions of their chosen peers. +* /Validation/ in which network participants agree on what ledger was + generated, based on the ledgers generated by chosen peers. + +These phases are continually repeated to process transactions submitted to the +network, generating successive ledgers and giving rise to the blockchain ledger +history depicted below. In this diagram, time is flowing to the right, but +links between ledgers point backward to the parent. Also note the alternate +Ledger 2 that was generated by some participants, but which failed validation +and was abandoned. + +[$images/consensus/block_chain.png] + +The remainder of this section describes the Consensus and Validation algorithms +in more detail and is meant as a companion guide to understanding the generic +implementation in =rippled=. The document *does not* discuss correctness, +fault-tolerance or liveness properties of the algorithms or the full details of +how they integrate within =rippled= to support the Ripple Consensus Ledger. + +[section Consensus Overview] + +[heading Definitions] + +* The /ledger/ is the shared distributed state. Each ledger has a unique ID to + distinguish it from all other ledgers. During consensus, the /previous/, + /prior/ or /last-closed/ ledger is the most recent ledger seen by consensus + and is the basis upon which it will build the next ledger. +* A /transaction/ is an instruction for an atomic change in the ledger state. A + unique ID distinguishes a transaction from other transactions. +* A /transaction set/ is a set of transactions under consideration by consensus. + The goal of consensus is to reach agreement on this set. The generic + consensus algorithm does not rely on an ordering of transactions within the + set, nor does it specify how to apply a transaction set to a ledger to + generate a new ledger. A unique ID distinguishes a set of transactions from + all other sets of transactions. +* A /node/ is one of the distributed actors running the consensus algorithm. It + has a unique ID to distinguish it from all other nodes. +* A /peer/ of a node is another node that it has chosen to follow and which it + believes will not collude with other chosen peers. The choice of peers is not + symmetric, since participants can decide on their chosen sets independently. +* A /position/ is the current belief of the next ledger's transaction set and + close time. Position can refer to the node's own position or the position of a + peer. +* A /proposal/ is one of a sequence of positions a node shares during consensus. + An initial proposal contains the starting position taken by a node before it + considers any peer positions. If a node subsequently updates its position in + response to its peers, it will issue an updated proposal. A proposal is + uniquely identified by the ID of the proposing node, the ID of the position + taken, the ID of the prior ledger the proposal is for, and the sequence number + of the proposal. +* A /dispute/ is a transaction that is either not part of a node's position or + not in a peer's position. During consensus, the node will add or remove + disputed transactions from its position based on that transaction's support + amongst its peers. + +Note that most types have an ID as a lightweight identifier of instances of that +type. Consensus often operates on the IDs directly since the underlying type is +potentially expensive to share over the network. For example, proposal's only +contain the ID of the position of a peer. Since many peers likely have the same +position, this reduces the need to send the full transaction set multiple times. +Instead, a node can request the transaction set from the network if necessary. + +[heading Overview ] +[$images/consensus/consensus_overview.png [width 50%] [height 50%] ] + +The diagram above is an overview of the consensus process from the perspective +of a single participant. Recall that during a single consensus round, a node is +trying to agree with its peers on which transactions to apply to its prior +ledger when generating the next ledger. It also attempts to agree on the +[link effective_close_time network time when the ledger closed]. There are +3 main phases to a consensus round: + +* A call to =startRound= places the node in the =Open= phase. In this phase, +the node is waiting for transactions to include in its open ledger. +* At some point, the node will =Close= the open ledger and transition to the +=Establish= phase. In this phase, the node shares/receives peer proposals on +which transactions should be accepted in the closed ledger. +* At some point, the node determines it has reached consensus with its peers on +which transactions to include. It transitions to the =Accept= phase. In this +phase, the node works on applying the transactions to the prior ledger to +generate a new closed ledger. Once the new ledger is completed, the node shares +the validated ledger hash with the network and makes a call to =startRound= to +start the cycle again for the next ledger. + +Throughout, a heartbeat timer calls =timerEntry= at a regular frequency to drive +the process forward. Although the =startRound= call occurs at arbitrary times +based on when the initial round began and the time it takes to apply +transactions, the transitions from =Open= to =Establish= and =Establish= to +=Accept= only occur during calls to =timerEntry=. Similarly, transactions can +arrive at arbitrary times, independent of the heartbeat timer. Transactions +received after the =Open= to =Close= transition and not part of peer proposals +won't be considered until the next consensus round. They are represented above +by the light green triangles. + +Peer proposals are issued by a node during a =timerEntry= call, but since peers +do not synchronize =timerEntry= calls, they are received by other peers at +arbitrary times. Peer proposals are only considered if received prior to the +=Establish= to =Accept= transition, and only if the peer is working on the same +prior ledger. Peer proposals received after consensus is reached will not be +meaningful and are represented above by the circle with the X in it. Only +proposals from chosen peers are considered. + +[#effective_close_time] +[heading Effective Close Time] + +In addition to agreeing on a transaction set, each consensus round tries to +agree on the time the ledger closed. Each node calculates its own close time +when it closes the open ledger. This exact close time is rounded to the nearest +multiple of the current /effective close time resolution/. It is this +/effective close time/ that nodes seek to agree on. This allows servers to +derive a common time for a ledger without the need for perfectly synchronized +clocks. As depicted below, the 3 pink arrows represent exact close times from 3 +consensus nodes that round to the same effective close time given the current +resolution. The purple arrow represents a peer whose estimate rounds to a +different effective close time given the current resolution. + +[$images/consensus/EffCloseTime.png] + +The effective close time is part of the node's position and is shared with peers +in its proposals. Just like the position on the consensus transaction set, a +node will update its close time position in response to its peers' effective +close time positions. Peers can agree to disagree on the close time, in which +case the effective close time is taken as 1 second past the prior close. + +The close time resolution is itself dynamic, decreasing (coarser) resolution in +subsequent consensus rounds if nodes are unable to reach consensus on an +effective close time and increasing (finer) resolution if nodes consistently +reach close time consensus. + +[heading Modes] + +Internally, a node operates under one of the following consensus modes. Either +of the first two modes may be chosen when a consensus round starts. + +* /Proposing/ indicates the node is a full-fledged consensus participant. It + takes on positions and sends proposals to its peers. +* /Observing/ indicates the node is a passive consensus participant. It + maintains a position internally, but does not propose that position to its + peers. Instead, it receives peer proposals and updates its position + to track the majority of its peers. This may be preferred if the node is only + being used to track the state of the network or during a start-up phase while + it is still synchronizing with the network. + +The other two modes are set internally during the consensus round when the node +believes it is no longer working on the dominant ledger chain based on peer +validations. It checks this on every call to =timerEntry=. + +* /Wrong Ledger/ indicates the node is not working on the correct prior ledger + and does not have it available. It requests that ledger from the network, but + continues to work towards consensus this round while waiting. If it had been + /proposing/, it will send a special "bowout" proposal to its peers to indicate + its change in mode for the rest of this round. For the duration of the round, + it defers to peer positions for determining the consensus outcome as if it + were just /observing/. +* /Switch Ledger/ indicates that the node has acquired the correct prior ledger + from the network. Although it now has the correct prior ledger, the fact that + it had the wrong one at some point during this round means it is likely behind + and should defer to peer positions for determining the consensus outcome. + +[$images/consensus/consensus_modes.png] + +Once either wrong ledger or switch ledger are reached, the node cannot +return to proposing or observing until the next consensus round. However, +the node could change its view of the correct prior ledger, so going from +switch ledger to wrong ledger and back again is possible. + +The distinction between the wrong and switched ledger modes arises because a +ledger's unique identifier may be known by a node before the ledger itself. This +reflects that fact that the data corresponding to a ledger may be large and take +time to share over the network, whereas the smaller ID could be shared in a peer +validation much more quickly. Distinguishing the two states allows the node to +decide how best to generate the next ledger once it declares consensus. + +[heading Phases] + +As depicted in the overview diagram, consensus is best viewed as a progression +through 3 phases. There are 4 public methods of the generic consensus algorithm +that determine this progression + +* =startRound= begins a consensus round. +* =timerEntry= is called at a regular frequency (=LEDGER_MIN_CLOSE=) and is the + only call to consensus that can change the phase from =Open= to =Establish= + or =Accept=. +* =peerProposal= is called whenever a peer proposal is received and is what + allows a node to update its position in a subsequent =timerEntry= call. +* =gotTxSet= is called when a transaction set is received from the network. This + is typically in response to a prior request from the node to acquire the + transaction set corresponding to a disagreeing peer's position. + +The following subsections describe each consensus phase in more detail and what +actions are taken in response to these calls. + +[h6 Open] + +The =Open= phase is a quiescent period to allow transactions to build up in the +node's open ledger. The duration is a trade-off between latency and throughput. +A shorter window reduces the latency to generating the next ledger, but also +reduces transaction throughput due to fewer transactions accepted into the +ledger. + +A call to =startRound= would forcibly begin the next consensus round, skipping +completion of the current round. This is not expected during normal operation. +Calls to =peerProposal= or =gotTxSet= simply store the proposal or transaction +set for use in the coming =Establish= phase. + +A call to =timerEntry= first checks that the node is working on the correct +prior ledger. If not, it will update the mode and request the correct ledger. +Otherwise, the node checks whether to switch to the =Establish= phase and close +the ledger. + +['Ledger Close] + +Under normal circumstances, the open ledger period ends when one of the following +is true + +* if there are transactions in the open ledger and more than =LEDGER_MIN_CLOSE= + have elapsed. This is the typical behavior. +* if there are no open transactions and a suitably longer idle interval has + elapsed. This increases the opportunity to get some transaction into + the next ledger and avoids doing useless work closing an empty ledger. +* if more than half the number of prior round peers have already closed or finished + this round. This indicates the node is falling behind and needs to catch up. + + +When closing the ledger, the node takes its initial position based on the +transactions in the open ledger and uses the current time as +its initial close time estimate. If in the proposing mode, the node shares its +initial position with peers. Now that the node has taken a position, it will +consider any peer positions for this round that arrived earlier. The node +generates disputed transactions for each transaction not in common with a peer's +position. The node also records the vote of each peer for each disputed +transaction. + +In the example below, we suppose our node has closed with transactions 1,2 and 3. It creates disputes +for transactions 2,3 and 4, since at least one peer position differs on each. + +[#disputes_image] +[$images/consensus/disputes.png [width 20%] [height 20%]] + + +[h6 Establish] + +The establish phase is the active period of consensus in which the node +exchanges proposals with peers in an attempt to reach agreement on the consensus +transactions and effective close time. + +A call to =startRound= would forcibly begin the next consensus round, skipping +completion of the current round. This is not expected during normal operation. +Calls to =peerProposal= or =gotTxSet= that reflect new positions will generate +disputed transactions for any new disagreements and will update the peer's vote +for all disputed transactions. + +A call to =timerEntry= first checks that the node is working from the correct +prior ledger. If not, the node will update the mode and request the correct +ledger. Otherwise, the node updates the node's position and considers whether +to switch to the =Accepted= phase and declare consensus reached. However, at +least =LEDGER_MIN_CONSENSUS= time must have elapsed before doing either. This +allows peers an opportunity to take an initial position and share it. + +['Update Position] + +In order to achieve consensus, the node is looking for a transaction set that is +supported by a super-majority of peers. The node works towards this set by +adding or removing disputed transactions from its position based on an +increasing threshold for inclusion. + +[$images/consensus/threshold.png [width 50%] [height 50%]] + +By starting with a lower threshold, a node initially allows a wide set of +transactions into its position. If the establish round continues and the node is +"stuck", a higher threshold can focus on accepting transactions with the most +support. The constants that define the thresholds and durations at which the +thresholds change are given by `AV_XXX_CONSENSUS_PCT` and +`AV_XXX_CONSENSUS_TIME` respectively, where =XXX= is =INIT=,=MID=,=LATE= and +=STUCK=. The effective close time position is updated using the same +thresholds. + +Given the [link disputes_image example disputes above] and an initial threshold +of 50%, our node would retain its position since transaction 1 was not in +dispute and transactions 2 and 3 have 75% support. Since its position did not +change, it would not need to send a new proposal to peers. Peer C would not +change either. Peer A would add transaction 3 to its position and Peer B would +remove transaction 4 from its position; both would then send an updated +position. + +Conversely, if the diagram reflected a later call to =timerEntry= that occurs in +the stuck region with a threshold of say 95%, our node would remove transactions +2 and 3 from its candidate set and send an updated position. Likewise, all the +other peers would end up with only transaction 1 in their position. + +Lastly, if our node were not in the proposing mode, it would not include its own +vote and just take the majority (>50%) position of its peers. In this example, +our node would maintain its position of transactions 1, 2 and 3. + +['Checking Consensus] + +After updating its position, the node checks for supermajority agreement with +its peers on its current position. This agreement is of the exact transaction +set, not just the support of individual transactions. That is, if our position +is a subset of a peer's position, that counts as a disagreement. Also recall +that effective close time agreement allows a supermajority of participants +agreeing to disagree. + +Consensus is declared when the following 3 clauses are true: + +* `LEDGER_MIN_CONSENSUS` time has elapsed in the establish phase +* At least 75% of the prior round proposers have proposed OR this establish + phase is `LEDGER_MIN_CONSENSUS` longer than the last round's establish phase +* =minimumConsensusPercentage= of ourself and our peers share the same position + +The middle condition ensures slower peers have a chance to share positions, but +prevents waiting too long on peers that have disconnected. Additionally, a node +can declare that consensus has moved on if =minimumConsensusPercentage= peers +have sent validations and moved on to the next ledger. This outcome indicates +the node has fallen behind its peers and needs to catch up. + +If a node is not proposing, it does not include its own position when +calculating the percent of agreeing participants but otherwise follows the above +logic. + +['Accepting Consensus] + +Once consensus is reached (or moved on), the node switches to the =Accept= phase +and signals to the implementing code that the round is complete. That code is +responsible for using the consensus transaction set to generate the next ledger +and calling =startRound= to begin the next round. The implementation has total +freedom on ordering transactions, deciding what to do if consensus moved on, +determining whether to retry or abandon local transactions that did not make the +consensus set and updating any internal state based on the consensus progress. + + +[h6 Accept] + +The =Accept= phase is the terminal phase of the consensus algorithm. Calls to +=timerEntry=, =peerProposal= and =gotTxSet= will not change the internal +consensus state while in the accept phase. The expectation is that the +application specific code is working to generate the new ledger based on the +consensus outcome. Once complete, that code should make a call to =startRound= +to kick off the next consensus round. The =startRound= call includes the new +prior ledger, prior ledger ID and whether the round should begin in the +proposing or observing mode. After setting some initial state, the phase +transitions to =Open=. The node will also check if the provided prior ledger +and ID are correct, updating the mode and requesting the proper ledger from the +network if necessary. + +[endsect] [/Consensus Overview] + +[section Consensus Type Requirements] + +The consensus type requirements are given below as minimal implementation stubs. +Actual implementations would augment these stubs with members appropriate for +managing the details of transactions and ledgers within the larger application +framework. + +[heading Transaction] +The transaction type =Tx= encapsulates a single transaction under consideration +by consensus. + +``` +struct Tx +{ + using ID = ...; + ID const & id() const; + + //... implementation specific +}; +``` + +[heading Transaction Set] +The transaction set type =TxSet= represents a set of [^Tx]s that are collectively +under consideration by consensus. A =TxSet= can be compared against other [^TxSet]s +(typically from peers) and can be modified to add or remove transactions via +the mutable subtype. + +``` +struct TxSet +{ + using Tx = Tx; + using ID = ...; + + ID const & id() const; + + bool exists(Tx::ID const &) const; + Tx const * find(Tx::ID const &) const ; + + // Return set of transactions that are not common with another set + // Bool in map is true if in our set, false if in other + std::map compare(TxSet const & other) const; + + // A mutable view that allows changing transactions in the set + struct MutableTxSet + { + MutableTxSet(TxSet const &); + bool insert(Tx const &); + bool erase(Tx::ID const &); + }; + + // Construct from a mutable view. + TxSet(MutableTxSet const &); + + // Alternatively, if the TxSet is itself mutable + // just alias MutableTxSet = TxSet + + //... implementation specific +}; +``` + +[heading Ledger] The =Ledger= type represents the state shared amongst the +distributed participants. Notice that the details of how the next ledger is +generated from the prior ledger and the consensus accepted transaction set is +not part of the interface. Within the generic code, this type is primarily used +to know that peers are working on the same tip of the ledger chain and to +provide some basic timing data for consensus. + +``` +struct Ledger +{ + using ID = ...; + + ID const & id() const; + + // Sequence number that is 1 more than the parent ledger's seq() + std::size_t seq() const; + + // Whether the ledger's close time was a non-trivial consensus result + bool closeAgree() const; + + // The close time resolution used in determing the close time + NetClock::duration closeTimeResolution() const; + + // The (effective) close time, based on the closeTimeResolution + NetClock::time_point closeTime() const; + + // The parent ledger's close time + NetClock::time_point parentCloseTime() const; + + Json::Value getJson() const; + + //... implementation specific +}; +``` +[heading Generic Consensus Interface] + +Following the +[@https://en.wikipedia.org/wiki/Curiously_recurring_template_pattern CRTP] +idiom, generic =Consensus= relies on a deriving class implementing a set of +helpers and callbacks that encapsulate implementation specific details of the +algorithm. Below are excerpts of the generic consensus implementation and of +helper types that will interact with the concrete implementing class. + +``` + +// Represents our proposed position or a peer's proposed position +template class ConsensusProposal; + +// Represents a transction under dispute this round +template class DisputedTx; + +template class Consensus +{ +protected: + enum class Mode { proposing, observing, wrongLedger, switchedLedger}; + + // Measure duration of phases of consensus + class Stopwatch + { + public: + std::chrono::milliseconds read() const; + // details omitted ... + }; + + // Initial ledger close times, not rounded by closeTimeResolution + // Used to gauge degree of synchronization between a node and its peers + struct CloseTimes + { + std::map peers; + NetClock::time_point self; + }; + + // Encapsulates the result of consensus. + struct Result + { + //! The set of transactions consensus agrees go in the ledger + TxSet_t set; + + //! Our proposed position on transactions/close time + Proposal_t position; + + //! Transactions which are under dispute with our peers + using Dispute_t = DisputedTx; + hash_map disputes; + + // Set of TxSet ids we have already compared/created disputes + hash_set compares; + + // Measures the duration of the establish phase for this consensus round + Stopwatch roundTime; + + // Indicates state in which consensus ended. Once in the accept phase + // will be either Yes or MovedOn + ConsensusState state = ConsensusState::No; + }; + +public: + // Kick-off the next round of consensus. + void startRound( + NetClock::time_point const& now, + typename Ledger_t::ID const& prevLedgerID, + Ledger_t const& prevLedger, + bool proposing); + + // Call periodically to drive consensus forward. + void timerEntry(NetClock::time_point const& now); + + // A peer has proposed a new position, adjust our tracking. Return true if the proposal + // was used. + bool peerProposal(NetClock::time_point const& now, Proposal_t const& newProposal); + + // Process a transaction set acquired from the network + void gotTxSet(NetClock::time_point const& now, TxSet_t const& txSet); + + // ... details +}; +``` +[heading Adapting Generic Consensus] + +The stub below shows the set of callback/helper functions required in the implementing class. + +``` +struct Traits +{ + using Ledger_t = Ledger; + using TxSet_t = TxSet; + using NodeID_t = ...; // Integer-like std::uint32_t to uniquely identify a node + +}; + +class ConsensusImp : public Consensus +{ + // Attempt to acquire a specific ledger from the network. + boost::optional acquireLedger(Ledger::ID const & ledgerID); + + // Acquire the transaction set associated with a proposed position. + boost::optional acquireTxSet(TxSet::ID const & setID); + + // Get peers' proposed positions. Returns an iterable + // with value_type convertable to ConsensusPosition<...> + auto const & proposals(Ledger::ID const & ledgerID); + + // Whether any transactions are in the open ledger + bool hasOpenTransactions() const; + + // Number of proposers that have validated the given ledger + std::size_t proposersValidated(Ledger::ID const & prevLedger) const; + + // Number of proposers that have validated a ledger descended from the + // given ledger + std::size_t proposersFinished(Ledger::ID const & prevLedger) const; + + // Return the ID of the last closed (and validated) ledger that the + // application thinks consensus should use as the prior ledger. + Ledger::ID getPrevLedger(Ledger::ID const & prevLedgerID, + Ledger const & prevLedger, + Mode mode); + + // Called when ledger closes. Implementation should generate an initial Result + // with position based on the current open ledger's transactions. + Result onClose(Ledger const &, Ledger const & prev, Mode mode); + + // Called when ledger is accepted by consensus + void onAccept(Result const & result, + RCLCxLedger const & prevLedger, + NetClock::duration closeResolution, + CloseTimes const & rawCloseTimes, + Mode const & mode); + + // Propose the position to peers. + void propose(ConsensusProposal<...> const & pos); + + // Relay a received peer proposal on to other peer's. + void relay(ConsensusProposal<...> const & pos); + + // Relay a disputed transaction to peers + void relay(TxSet::Tx const & tx); + + // Realy given transaction set with peers + void relay(TxSet const &s); + + //... implementation specific +}; +``` + +The implementing class hides many details of the peer communication +model from the generic code. + +* The =relay= member functions are responsible for sharing the given type with a + node's peers, but are agnostic to the mechanism. Ideally, messages are delivered + faster than =LEDGER_GRANULARITY=. +* The generic code does not specify how transactions are submitted by clients, + propagated through the network or stored in the open ledger. Indeed, the open + ledger is only conceptual from the perspective of the generic code---the + initial position and transaction set are opaquely generated in a + `Consensus::Result` instance returned from the =onClose= callback. +* The calls to =acquireLedger= and =acquireTxSet= only have non-trivial return + if the ledger or transaction set of interest is available. The implementing + class is free to block while acquiring, or return the empty option while + servicing the request asynchronously. Due to legacy reasons, the two calls + are not symmetric. =acquireTxSet= requires the host application to call + =gotTxSet= when an asynchronous =acquire= completes. Conversely, + =acquireLedger= will be called again later by the consensus code if it still + desires the ledger with the hope that the asynchronous acquisition is + complete. + +[endsect] [/Consensus Type Requirements] + +[section Validation] + +Coming Soon! + +[endsect] [/Validation] + +[endsect] [/Consensus and Validation] diff --git a/docs/images/consensus/EffCloseTime.png b/docs/images/consensus/EffCloseTime.png new file mode 100644 index 0000000000000000000000000000000000000000..922320de50b71d9ab5b487775301f69d85b29899 GIT binary patch literal 1975 zcma)5dpz4&7XKxJNl~^_)ATXZ?RrdiHTv^NTJe~VVl;$NWlCEF6}1&bsAnl%rnV|W zFbPK0jM3I>P>py+MoWjJt)#>w5#te}$dXJ|d0O05VurIo-?Tj5{V0aOHT(-Bjq2=(> zI*CN;oE{&OKVyl9UdeifrHVF@P{^HV=g9?O8FvDUhkui|4!s)vx$nnv=1AMi`wu#+s;bP*&1p2+!oosEMh2hH zN1;$po;*>hR8dh;N~KZ*28YAoFc?fCk@)-jx3si4J3DuGcef9s;n2xoA?~iuz%9*B zPeTA1ecGP_0I1TP0mH}NECc{8UA(iSPvY2;NGD+oi@Fh9gJ%{S$9oiDWns{%`yFwzoWQ3TmsULR|lMoz^PVgl*;# z)CY7rMOP}Gy&7)FomEMYBsMW`I+1e-;202IWUl|q5o;>``T)0&iVzGCzcd=9$6bg3 z&z;*r^$lU6d~y{>Br+fzzPY|P`>wGXr{-0@KnQw>Uk0Bn$i*{@v?zuyUBkwiEBm?e zs6hALAIIHpvU2_sbY%}GQMb4DZ^ws=gleUQSpgH1L$uSfYD#!_dF?-;u&A%pj-*Jr za%I^<_LxtMB7;nXpq;q9_x)B5@&ECGs#aZ(_d1jYgeC{WK4I^RG?E=>&WU2<%2XiLhu7ItyZo) zOjdhsT+~+3h{=1oJ?}izFCU~undwPmKFy^<`KD?rzA?;Rb{bcoJN^8b83zLbH8bWH zubtW(q zY%C`rvxzgDqg8R1MySku?NIJ@j1$15 zU~{mG?Jye=(;c$4Cl4yA=cL57B!IeQ!xt{$=tdTtL#hiXQ;9pj(IEHpuEWX@V(J%l zz1?rqe%q}vtzv7v=Mz+%=xaW-?0 z+25a4^uKf)vWzQOfqZk5~8y40|N&3PWZwgxoCVNwDJlirvXMPYH6e<_PFlq&sf zlEtvLI>^RkjCuIgc~i(3VM@~aCxJZ^K@v;KZtY$y|R@a

dSDGT+MLC}nc{#I*C*V^t|9#U0o2Jw;EvYd7yt#O)58auIf)2 za$H$QAfri~i(lNO;J7ecXlxiH087x~1RAtLTVQ-FtZ|lDoQ0(s2;x9+&)dq%|00mD aUWy|+uq$46qM?Sf)C`b<-QIMikDS}jy79v3)R3Q;WM5U;Jv=BO>g(^J+ zM8OxSQbJ9z08%1SLJ1_^<6U>%weDT({(Jv?KeFeUCo|99v(KJ6vreLwg)tAO2qypl zJfJ9)}nE)UhnA>Kp1O9O$(8Mtq0M6w6`9VyPacLk4xqHLd z5O~TaZUT1BhZx-sx#tn$rR5pq1vY@Pq7qbIQCVJD)mBMEOG!yfP4%LpqL!kf-mY-f z|C(?ganH{?;{TsuSy{sm0D^Z+4GnC=U8(aYLWJz{`Zg(_uAZ!~<&%0|B`0U{EjFp? zJ=+^rc?~&Vt{K=E)jA9EUK*aH$}#o$!}XNVYtMN!UidakJ@8HUU}Gth$d9|3um%jYsOl0I z1m>rpW^CuB52v2;OA3y-QRK$Vi~9Uy=y0OYgaKnhj3%tJTC)VJWL^~s^c*X#Sbuxg zDKHG0zWC;#8utwM_^%}9wz`pvmgBa!<@OzG@o~37YgBg;6NYbEz5E^W2CM0kLLu8t zv%wcuO%hW+N8f0^D2?RQ&J-zR!jkD+5;TY4w|1zxdo2HaO8)5-{~tY@+icD@&Ry7w zY~Stq^L-A8tc|f2>#oB~hgnZ=(i~B{4HfM%EhTzumlb4n_D~+%i!(_%O{-Sv(E54) z_LQ_Aq+v0RNExVk;^bFS$|_mtc$%9nK2ve_BZq7Bq+6k*PiRb*EHS)gMq9I`z;zFm zXdOwzMu@aNByMqjdMEe&-O`Pu#OI&qkdEZ-#BB9sWl`OJF6pSWFK^EqUC>mTfj>p7 zKg_7{9SeZkj2#=3K;#WJ?seYENJ_$Us2t)hCCYkdMP>XPd+}SZFxb++c=DURgUzc5 znR8~*6K)-Tzh|RbR~0w8N@}#Xs%bLHy4(3NLdhCnQ4=$TA-@mFtlCUvu zXz-np-7WTN{;4o7oM4s~LQ&s9UWIqfetdKt!gz42X;){`ya?})H&|Am3`Q)5Yv;Aq zN77&=rKJ^a#F!W^k=QmI@oRk$(qE{uzO1dXvFa`XnMC)`TrFejS#$iUxUmoy z;`e7l2`_7TG@`Nt56O3jJ}wRnHX$rPr(%9Npc9JJ@DpCv`B~d;K-r3c?Ee2Ri}<{(>H@}$p`xQ8-e8r^u>c*!R zpSNAS*gmfPF5EjfxmO2u#UXe4w%5|E2GO&lExTymyz~z9`_oCfuS3Dl+ijNL@UT*( zo9~Os5U=Rb18Fvl@zo0NYQd?c$B$ zv@J8_+E>CVMb{vVsrun!td(hEip>nyp_)&TBVtsENV_Ml9XkH&OHX=E&RTQ+tNGTB z{XJS+=}JPjb|XfG<})2vo{2-hx{lEAWLgz5W4{G4)YQ76(Sy>}7%{LLALN%X8lT;6GmH1~ zutRx};@0N%CfC#@b?hr@BscJ9FY@^gRT_s>)jA_|rd*@jd;PzEW;nrFR+_k8r`uUo zyr`q9MNLTSwyxm3c>}%>eiIxqmQJDD`@G~Dmbr$nX!RFp8GZ-5p$EBllQ$Lx9#YyJaK zO`vlj`4r$RL-nGLC6tG)O``V^IZ26X7fz)HZJXJ_nWI#UK4dKZjc&YeUeEw3=*>!n z{+mhbJ-yC5!WfB~JA=K=UyxtxcirBR?QB~gT%6uJ4^>}NA3ndt(RDvR#@j=UX!FNUQTPFPD^qy_S8lCIOL4Qwz z?j9TpTkNU#caY?A%4+x@5fL%GA8a|_9Qa8h1T34O6dh%{{BGAVHu z)Am3$CV+q*uv+=pg|+I>fCwm>lo$gM|bR%<+PHS`ATgRhZm zZOm8!dx*m=Hg8YY2zyV0etD5SqqKS&MCks{S!?UN5EB6L- zbO#YeNGp^YSUoB9%GQ|*Vk-Mw<^#XTm~NT2Iz$?$+11@T0ckzid}60vY0mW&($UUz z%*BeL0+O2)ztwBm3!xQ-UygnF@Imgn zz2R5D&evv9US1x!go+GBOWy_-vceOS;wvRM{7;eb%$B#C@^+g$kjT#9@KnuCWwZ^fm3y1uHSI#&# z`o0tnInF1s6C1ETRR=1Som)UFHpVa_#q}QEuC%Y-iX9>+o>;gC-mV_E^6Bj_?7wYw zaErEca6zU+c?`8!cmv(zJU`>p;Y1>K`xJ)ojT7n%;9AslD5GrU4Wo1o|h@o%5mG2&BaaeT+b z(5Ve5Ojll0BkENjb0Rh^?bL=P4L~M>uKd?2ISmF-zt8`Ai=dDXX)O78{y+dG4*Jsr z!74Wh6hMEgVt>FNh7ANwe;A=Z0Axgtfl=Z>MV@Q3*q2x`Hl^N4_BNw?I4 z_n{=w13|k}La8S!TL%*f2V$_BG zRR5DTzoUZ_er9D9#2$UE<dd`s|H>X(w=`=MPW?{k`NmmA$Ry8H>2?SA!w z!RV+p9otM)rYe#B&-d9q#$XVbHu#G~R_%FZ(~5!GVhzR`V;Dy64^ur_ZAcGXWNCQl ziSq6-?t)wqVl221Eg><=1}g9!GnZe_%T9q8&Zd|S2Nn~_$5>va2v^;z^p<^b zvz8{_WFpS4Q?X-IQ@)MXH^JP=ZCuyapRK_}CCeQgWpVC=n>e*Y;+(6{XyGZZhP;boUOd~ydUS+=n5&6tR%02mdKf@2WqBi|N{%~id=<{bA z-4F+Tb8%#ANYw~dltp&rdNCn?@iMCT5OI_kwDs(is9;I;ET8Xkr)IE8It;POI1riaJZOy!^)DwJ4@OSJV+QKu`@^FXhStbt*oP7O`}I_(fBBKR zJ)M3Jy%7mrbnKV^Yu6z_D}L}K?eOdnDo8?EFYP@00%yjylCH4D}U+s~E1Iroa2X6C@s4OzwwC-P=c1%qFohfXMtJ2BN@qvo2QR$wHM2@+626e*j-LAlV=kv z`wJpaT4{M{-&Es-#@kxbypwGC%aso< zv(2wrl=fh2O>wY^O9rr^&W7u@}|vlV_>VR6qGjkSg(9 zKDDUU(2~$-C%LYXk9AhOq+qZ z`ve3Q_mOPp7;(*{>4bqc#kHv6zEOz~wKcwt+U&nG@XXDxYNf#Cp~W2@!cXT5Gdl&AH#r3#%c*0<&2|<5saWy;YL2)430^(Dc?vpkiS#c%JrkWI z_H-V6rf*V5=bItoz2s2p@aTx>VYqGsLjDd+qU=eIm%hx}W7-p~IhX+W7!_1JGG>|^ z$Pod>_6jYUJEUO|MR0RF`&c^6cR69Rc znkZ8h$bZtS^DEz8-jq~Z8C8*w8%G$~O$A2nrdNvuEo%&7wfhh}|5f9u;5Q(<%ibo| zbunBOB~`bt;U$a5$8Cwy;m5>)E>q_?RzL%Mef;yrgpNx2_)FH_)msewE(e$zSs3E4 Hy2t(}nf7(l literal 0 HcmV?d00001 diff --git a/docs/images/consensus/consensus_modes.png b/docs/images/consensus/consensus_modes.png new file mode 100644 index 0000000000000000000000000000000000000000..e575631dacbab2110b3d273b7f01d692510ecae0 GIT binary patch literal 14411 zcmd6Obx>Ph*De$arC5Pd+@(m-;;s!+oE9tY8r-$T3WeYf!Ci{GySq#95GaHaTra=( z`|iyB{&??y_nn)WGiTPy-Y4g*oF&h**Z!%h48VR(`Wgud30q!HMjZ(WS>XAwdxiF_ z(KafGd_JLCN-9YrAytDg?~PwR%g8S304b!(33Ax8Ky50gu7rf-&4`2)7>tB;`>YB) zKtgikL_#_=LP8QsK|&&O%xF;)dHw>!QBKDN35huUp95Lm?MK4164^u%AcGYCl0xoT zU~rYyb~Q70H5W29j$$12)EcqW3?15(1 z7GD2;LRw8BArcb5th|h*rl-O2I=b&$^IP;&LvM!QjeH{J@AwijFQll?S-uC(5K5}4 zsNimWutfO~K=y*R0U`62P*x~t$_E|Rz8Dj;fe6U*@ZeGx*`l{F^oO*zuV-eaW{%%Y z0JA(St}_79NO-X!BAJsiG7H9Cc(L`wjxuI!XykZ0k;nj<1(W}0Dpr+85EuZWj5?5w z+)HdUeX;j3HQZ_gi|*4e#|{tRAK;Dt#zmtDM$^#HLbw4Zm3?}mq^@_83(Fq5e}_Rz z?X>`{ZiAHOoCs9&YTz*`V9zaJG(FIEp382O_clGj{lQ zW_E@IB4WZC?K+OO7;sfGQzl>DAaxi_@b9|UfxL!$2bbZMbI|X8K}bModS?pwyqt3$%|O3;po30C=spqU&+u6;n$~< z%3t~Lw`YIbF6v~6WCi}J`wU2F{;u3}oQj9Z={3TK(cRe;WiEbge9JC!H~DgCXsCr* zdO{RsVv$U7fn2)!H%Rq58iR>tGQ@!j6nJRr1nGh;{T)p;lxFwHGq}|i-b;+`qMWd< z^s3|>g|k<=)>J~{Kz_SJa2V{qHr3t=Pt@_ zUg#qi%Zv+ip{z{(hB!ogX;h?Y7Hb``JLwWdQNF$r3`OEf52ZC<+zDU3Wx`=23)a6g zRV|xOmOvJt4b+9DM!yFvXZ#@tn8Hq_gv=D2(Pq>O4iU){C9?^>StS7BJar9>8ULqnqE0YaUK_MGfpwRJjfu}V6p{?9k{#{M2s}DKYa@fN{ z-pweKza>fmY#uf}`I(ox>VkiFcgQ)xN%U_YH(y~{PfA!9Q6HnI|FwSNe2|BpysUb5=B=k03` z0IO9?>&{sFnaMibRR09w@_g1q?~nMiQ_X(7zbKdAZg_VkFONNDA0=$|oaWYs7yTUZ+MjPB~T3lCq6u4!!L2Aw8zvT8FS?b*KT)q*$h&T5TAEQQpH!U|32 zS4^84;XHLz*958}uLh&PNerR~NTh~R<^5R)XFvVc2^USaBfTmY(Xzi3r+YV<8Kc zr>=9q?(wl#{X$a3;RB~!qo?cTqS4WEUeYV=qjcgsm!z^>0hd3RzP2Fw=2KnEkzsk4 z({xZlC};Cz@8tdDpc$XqoE91HWjevs&leT@%}^+ZvNDBel*)t!R>oAs4Ne z{R$NN*?{8T^AaZU!+VV}tRIWOhLt~~uZ;uV-$+&0+ZnL*Q&>0v`|Y{3xA!|73eccF zz5_Op<0r?4!;eHVP~I1?^^o#dOrA+gs>k@g#L*;>!MbLj={l&UMD$$P)j)Qm zYnp?7nX!+ic-yIrw8+Jwx(qIpF*n==$UdkE)BGp~`{iI?x?N3_ zS1s8?7=khdkh(5|wN5wMVz=F1ee%>4^EukV7I5Q(H3IZooai~@qTg@!!RSS1T5)0+ z6QjiHY-Ym9-?YM9xdKYC6bo*r5AD|i8jPyzE^@0{^Xy7qc<%k+4bSQ%V!{hp3R(1j4l{^oKscKAZ zMoFR5WhR+5KTM(e^6#nB=x-_@6?vo>OMAA;M(*d5YpWeB?tg?q3^f$qU2Si`p;Y^S zWVJAkEW>!Z%2kAScB&e;qd*JkHf&$NSMErQ)2UtN2tIWXSxy!SyhgPQn0r07-t=H# zqF{~D$iv(Sz5k7}9y-A@dnAV*l(5jj?RuX0$*}3n*wrT+$3humEoR79k50~2qh}x&+KWhVA%ni>gIkaI>aB+ABT!zHMY^&*ZEBfB5HK| zr3zIZLQl@|UFTUsOx71>&jtum&pM2d4efNMsOt0ep#cHnHec92%qMA{ZTK%Wi@)oS zM5WGmx7M`R#OWLgay%FB84tQ0|9RvX_Dg2twh2=z{|Sp~mQtFk{D+4)1a-)9asCqG zyM!~@IrHKo{fk64>fu-jy4m_n$-Up{`kE+{`VQic|e8d+Pttl>00;2Kj z2fKxO<5d#t&x7E6KN$uc#+OYop+ej|YGJ!_{(PSn=pyD=*unBC^4as{hX-=+C+KND z)Nl&{zDlW(M0{nr!x1BI$9gG!|AqzNpkclrM-#LtArvQgX0ulUHIY%!WyGgfy|7wm zu8Fw8S&t^2k4&AcH!2XJwf`zH360<>IjIu)JSu7TEe!)bkv-8@Ca)&is~#jTVHFTL z{}`7LTmW%RR_m50xex_O`uWHF*b&s65@YJ!Xja5A?IgFe)$~XIQrgBb3EYlnUVZca zi2p$;EFxDe{w(n8S&h;Ye+~1-EYj!X)5Mdzeh*>JlloW3aux>3b})es{1^(|ude6J zFqeLMt8o_d+n~I^OopFu#w1s$K7KA(kt*%ed|01(s=QxC@HgT%>0{n)ZJtRkb(&E} zS9-sKJhqmR(=#xg_f^Jm%1ye>jVCO6zdc_>SWjf1x@Kk1$2U$q?qo22ltcHx^4KeI zjfo^lY^~PEgjwRJh!0PB+<07#p5~HL23X^<@ZP*dFCZ}VemjSwrTc?pw!R`d@kpyk z-f~_+f^{k~V-kCV8EPrT^QXatm{)|nN_3W(W2D<_Wa+Zd zWELg6`AxwKhR<&E&YXXiYT|At5SIAW)-ak`%QPM)(;m3G5cmY8l0AXJZ>o2`-z*Jm zaC71Q%OoMW_OP|w*^R(~$}!|MdbUIk62h+1ePA?pU>@UJMpOy*h7L=4%97?Eabjq3 z%5HJ1OTe$2CjZRsAxFO59>TD3LA+zRR-6p}S(aqnM7k{0IJp(N+4COBm@1OA(w>~h za6xBe^s6j@DT!{hKOMf{@FX3VAPO*`W*Uo%y@2A6x=@8xiBSm>rpZd1U-WTIsSCWa zcHc$xr;?Y7o@Z^8GMAg)oTEo~9%W4nnUn?1JqbqF}k)WUwT?YrP_o30Of9#XfN zb@3DNnNjihVj)syf*9=;WYKw^sV_XGLL(xR#DguLBTp=_0?CBH37F2Z)b@bYa#_MC)X-Q0^ zfuGhpx+O6(Dr`Y%WuMIz(DE$mbMb!xR9nZBwHptzmn&2BqY?ZUz<8PpyD)>ksgcjp zT4CHmpW73Z0g?eahQ_|)mK+=YPIyTi0)ayv;%SrmotbBHQbLST?io}S3-=#Cr~Sr( zPBNI@aTHA}hNi%Wx^4TW(3muu!v!bP*qB|fJw+%VSM4aoHet|rCJzlCiP;ec{ROM5 z{vl!DB-J^8FK*;8dY(uavaT4}0u<+)+*sG|up+vhZ}^wx;?b|Sz+1=iM;E(CResB| zt>4K{y**k8sRG)u#mAPzlAjiaoJORr0{9-}E#~~=X-1(ZVyQJxR$v(gGy~sU8j~H3 zX@@Qz@??KV(GzHi4+QWFr z_veBRpnX@RR%>JY$6yJ9CFyfs71=Fnn|EhgyQyY|RmyXR)k{?ZLiT>i&qaZKm>^SP z60|R8*y^pe#z<2r972QqElx@d z&$U+`NwfGXs$dxSQ(jBK=|mBkYW@i-&%%$4;MqTutwwXJ6%;wNh=mFr@+vXE#%5=@ z+N1thqVY~02E|m4hTCi|d$di1T#n*viAwv}i ztjC>dgI>MDWsR}~1JUS<(h>bDfU(4HL?oA&#|D_jXYJlWv>)2qc3d#3&eWOH5N$J8 zmph!?x>2I> zv#=VIFtN)?$m#sW2+u9*`vs9->IKq{3emqAE?nB-WAh~$6uV=?GA`z&#c0K$MXbeU zOKEJGUw}|1%_&@DZ`}B-x4M*-L+MsBI(W_rChYyU zI8cz6c2F;v1EPuAJzOJLL~|104^5Brx_9%rxR3CK#?23^{+{q` z6Lnxq8e5Ahr07VpnNcb2$nAU(8zCc=!dIu9;cHA_3x9#{Sa__zRhW;`+#^+%YvSxU z+u_KymH^|>VfscdK{*q!OYQqCe}J+|>ejd-Umv#ctR?uihn;7R_9$7)4RaIqDh%Rt!pi{J)#a^fk8malWwv}-kamgR=h@>rcu%x zg^X@zlFY*H!pwz>W-;4AS5cUy&}3o<;63anW^EK&IQq?cc@21RQ83!lpe5wn>SMcE z1QOn=$6d_+0YZ=Uvh~3xUUbcbD%cmqpVuD2#Gf^pC$IU5 z{L+195*MH=27*hB_h-a$3`Xf2;`86?uHEf?coy^wW?Vv=$eiGTceV0iEOdbJ3_6uO zsT)-Q`W@@=n*x_2Jv@&yo&5f&zPMt4CO^d?Am~yi;2lh>`{hHm3qrLIOc|Y#51q~f zunqb<%Com%qOBG1%$_rO)aRKQ%x-}mg|$Qt3DSFVnAf#QEx2YT&lydQUDU^$ZJU_U z))JeXe&A16{iiszf5#pDi?tJR4n|yH6D*gC-g7sY?B@akuirv7M~I|eOy%6jUIUbuH4U0{dOR0}( zz-Z@LOXr?^R}`fv>2&0~QC8p}D*rwd4mt?I96t(w#}<>6W@FIW_*sSr7>Y;tNdVHy zf7y7?4!_mQLd~!jay>B0vYg1U&id&Ttg8TOhFi)cvncc;Gie}IGhzuNC# ze@>Ady-$l#DsQcc6JLkyrb%$keP{XXlRIvjxJoZM(i~YvFi(L z#qpZoO5?MSFdt14gsAz?<;xR%&~KT$ITf=kKshig}lznS%cqyYQ*@{Og!_ND3I?tRyT4So(U|! ziklG`8bBZ7~O7{iR^^ z+lWR37b`1fC;3X)QeW^W%lMk>9SX{eaiOZWBbD|+V??Y$St)ULk7y$oKIEPKvvwfx z7&u+0^L|5dv1)bqmg1I7o3MqWA%kpWXiTPe^)PLb?CNB$eOXg#P&u=2BQH!XR~4^( zt%H!VM*1i^{T)@9DRd%x`S>D+c?-YZxPJLBKO7Q;_)9bvT_t?JqQ83DU020rInGUx zMv6r00GP<=i|)O{qc;BS4&_Lxv8wb1L;I2DEvcx0V+OV4)9DMe5>l|CRE;Itqbp}L zWd~C7Lz}Ha2nmy9+eH^;+wC4|`AW5HK<@IV0^@>S#6@dkx_uC;$Ij}Bgu8q6%^KkG zolq6~)x!*v*Krg9m*sD(QE!Cag%8S)@8dwyiwpV>%Po>gBV5KM;LKk?VX$_JW&d+>r`BrCki0Dde_Eq&kkxTn+=p#Qt3TLY0cf3GQbxjn?76aJ_Y)`k5-sE zyvmFg1)UC3O5_dRZ9iDYZ%w!HEy$gwy;k}m*TE^*kTVQH+pH*g3biyNKJkN*$cWv& zYLmiDe|=U-BkFXwNg2M;^iJRKp6ta6Z{Ir%&CVAz*G~OXb#a|)t58bWB|6dSy8{}6 zv_y((qru`f_C>Gr;#wf+ftvYdK^%8A8$52{O!i z_hS<)1x_w=AGAo8@xKqN*UR-7&33F?r0O;<@CkR)fWh|;$S0)~p~j{FI$kAqr&Iz|q@GZzLCXPadJc@{U39;2USX`foWIP&Ya38lc-Gu+%` zx9+}w=>~9oa4dIB)U?H1zSssNfCW3RGfOru19&kp89z&oWYiby7LtehaiXbHZtv)b z8})Of|2~d~SMTQme{bb6YtzjafxafFi#{GjVldJZbt0G-{6q*9A$Ol2(S+4h{UaYU zI}96C_n%&-Gx|rqtho;%VO8tYRl@SR;w6ZYJfl!3TO)vc%8}3HyzzrstWXu$m)e)v zm)lp^D|>;EJu)lZ2FJ%C5JPt}@xFlplkV>?!^6WlkB;U_2pE+!!arKlrvy*9@RI1H z7VGCkc(r>Xys7i42OfD|+QaRBaW^Ia`qdjZ-2mz+%A7vJLYbM^x$kJ z9#`P4SL;%S>gcrQ4nWDLe@{~`DeyhNX4WOX#Shaw#nI#h$1^aAWVr^2I4Q zq5KJgD9VdKEzD|6XFD=%^J42^IPjnn4!(A=G%s?jJ8lJuMXJk2>fb+TZQIb<*Mh`u zhw5@3-R(8(_3TYE0ju}k4;+5S7K6{3aAk`--$c#H$!YOiPEM|?*#$aQlbpt7iWZ7R z@_i(cHBUaCajDLl;NC3Ui$MpUvfr%9ZSwfy^UFj!X{9ek>bwLt#$QU)vjee|`2Vc` zl(c8anDM7zTvsm-z7KWnV_-;Qt*%5^w-_)#sg}yAAt=?E59?m2lbD;E(`md=x(eXH zRT=#0m3mc-rKUIEskg7>{(<9T0*Bgb9l=FBNBo{uoZ~?pFITTEW)V&sR3XtPdW#Aj zG$4sWdfC8Pd!tszm%-cU(^%C?jgr!x*1ap6-!_|)q2-L{yMFWBgkv`V{in+xch&G~ zytS*dj|Xf=@Uv}=Yr6^1t0eR7IQ%iV#$H6COeyN+yw$1|nYEsEIyQyST-{cpRl4n} zVTsE9hP1Vz>5x9vb6Fs#q}0FNAE{Kts#!e6D=H-=rCOjsm?7r3(%dl0z{qHFbG9eo zbWYrM(KbVP5Pi%htUbGO;dna70dq>8`<%C)@3wm-g)RSNkg%0^({8kJ%g}4ab;9IJ9}G$K`3<~vE)j~P(U7uT z?QYi<6HJOd)o=7^ZN5grkW}GN$sc<21%J6(lzXy+06>4W=b|4FGQ$0z^J zx{B(XnVDH9j|!n04XXZH0*Gunnn$_~$DpQG=gj8W*B`=FRxt0l&fY2M&Prot$KXrJoBj~Ozol! z)U2kp_et2ojon&3z?}_>rJ4YcX30iPhpTJS1zf-xskPCbQ_JY$Z+AW}=if3==KLxg z*cjOm)ezf|*swfrrDyx!Z&YL0CGW9|>oKw8qNK3PR!;~XDXI53=*7#|*;Z=MQ#d|_ zusaXb=@JeBJy)nX4(lz9<&~c%d%qNGzZ}MQB{C}(%vQ489kQ(u5ov21|7~CNXXv7I zJ~&y^hg1_*s6MNfPWNXfpZogsXKalxzQZwuw|!RTgHD{&NH(=fxA&T0JJx|emDZ?U0M9efSMJ+|#6%64_5 zoC)uQM4T7R(kNaW&X1qw8MJ#73p&%M@dHop&$VLhb?J^|uyO}gJ~_AT_>L8vFT4_6 z?nDepx~zs*YNTtzBUd?kYbww$X=^t0_b*=;T(!f-yG7|%4i0AK%)KAkn0c4ia4!)f z46my6i89G}rIAa}cwb#nh)1*!;-(|)S{yP1`Y*>Bi+g~rC>{RDa&o^0BMkXQIj0C> z$5(S?%=FLW;GV!GM|xW1j=O&YD*SRqpulK2348z0 zA?(vt_TA`Y3L;EADm9P?!{VV;$*}$vVxdy1m$Y%%IqEqOJLM}HQ3<)Jmv$Vh%QrN6 zQa2?$YSe}Ur~5`n8+?wXb|$@pORS&}je2RIO>GLZH@1SYeGQ2$zZ~ zXt3p6rbbh#*?I`3RF^z6qS2`p@B^*>s~zQ;s9=q#cdq_qd-K8N^{voCEcFDyJac%MpD+jD_c_Qia08-zyw@Z7s| zlhTTs@l_i6@$uSNK0%PEn21L=c-^`(x23eJ&5WflP-0BK_U+(grmQ4}Nc&Xo(@O7` zf`@0q;v@(;8Pu%U)qK#oH}`usr!Z^u7TBn>W+l`&&sw(9fAKBDHs#UMsH+*>K(;_U zDyw!T63LtJtO|#}0E*@2AFsc$Aj%cSLVG4d z#1P)ZV!~vd!zvmxK-qL-l>jgO4Ln2PMl$CGjn%^#-76-o7l{2aH|2u+&}VD#+ONDF zO=5HD$h$9oE>yHzTuTrXd;C~Uz8|=_@L(|5>3LhVCBBrj^cGET1vuc5BsMnoIg7jb z$34N^i9Tpgi9lQ?MYWq8Y4Gv!Gr$|#&(+%L#f7Y+<8PPO&leKl=Ao|B#lpq)Pkt&Dup+RK zAs*M}(6~LMg906g+_3|N&u%F&4*(K^Opiw+100TbK;rHl&f!0Qc1KePXTT4ht1`OA zq`Uj{(R}&%jH>^V>+M(h%*@OLAPZM<3HaUe{F7P!?hK||ySkWm#p<11 zUVb`WsaLBukwryCO|fQBNRaru-LF=n&hR{U;B#I&8A^0{Y+~Y`uk3Eu|I62h{p|GA zv4ge-1tpr@`Jqz-Z}~g2NBnd6rfhLPju7*^H9DBeu~@<6XbbdzxXtl^ZwR>T|4RL| zJMj;d#=9<)o0m78D~kzj9c|WYoZ7m05BB{o`2$U`=uP$aR_lK_`5^mQO*gE{2WkzpCB&*8t|8=9S=e~L%O3{p}@HHxw6GpAPPuOo+zmBy<|N6x< zDbrxowZ5y6$P(gyypp=8BK}b4<*wD;-94ctgVQ$6Pemr`&67*S7S}r!b;qCkOzbVS zn6pMxiv7C9hRZxZRcvZ1?0!xZ^(C6gEl4~^H|ntXTQm3bCj-1@v@H1CnMeLlXP$EQ zzg$DM!6qa`P0fm8;gvNBY@#GU?3o6;zP_gB{*^1Btr02 zN@0>L0LH=pxEnSKJ5{9JEa`?N0Vgz)DhShQq9lRiEtc%+qkgPTIa2Qago|?izkbKM z%RDn&^89*OF50qd?|l$WPPpeevS%WD&vW4Pj1BrX?*AWSxG+7vyn7Z`(ZvilPA|{C zhuuTHQPEIorzRmcCm;mkucVP`knQc)+DyvD#!Rj4o#UvkF6#ec!pLR=U!5YPopnk1 zj#q!86h-g2+F|)JIP#oUhM-N%?rd}kL!JhMGm66Azj#2&02^e6u5PoDO&#qOZEclo z-9oIXsl+|dNK#Xuc`fUv3P>%5O52mry`OKU#!q5L2gM(fdB*1nDr$AM8f!O8Kh8Xn z_PpyQL0Vb;IkSUGL;9rD7R`!<)+sO^s#L=tIA?OoirwiDUv@YJcrWR$J^4fJ!<5)F zw4iJ=Zz&2+u7umT>WE`ZDBv~>j?F8+**t>Y?>Y-rh2}sRAn;s`i&odxD=P5_`P!mx zjTei&jqG5b&hG3O{5ss;wSAd}*ftUg-(;izQ7|l$3wZ+mO zEkde}CMjGit#z?2k07IhnQPD7p^}$;@t96)Pg_T$JSOT8&I#*m#9O9tI*tA9yW~gQ zK@e59ABXr4+4O}p3NdOX;z?y+^l>mHeGb&v-5WcGvPvw{tu>vA=rOTE5gqnS|D zkPPRiRLQGVlf`N2S0A)rt>T=>hT(y>qPs<>{C9Uw@k-YW!J8Bbj=-(^2;tuxhvKPH_XVqFcZ$1H3IB?o6laIblZbx8 ziO*Ja`S2kfpE~FrTW9k!1U+YPv;tNF>o*^#;0Vz|ZHt=?~>J zNp?CQ>{gG>O5u~fsp$3bd(4g7SM@BK18sYlZ}{@x$C(`n+0KQX@p8M`JE2F*Pg(I6 zcy|hsgWO<0nI}7cVtg#z{$XvH4sUv=dqE-U^V$RZ$h34Yib|nvraZMcR67XOMnT~3!xcP&-HkUqiRIRb4jQi+K+WczkNaljK!loa()mdY+2p0&bt*4fPdf zuBz5Js?{~ua#>UwuoNg~V+eoj!(uaG5#pfuiuN+cHP&R`=Ws(;F*FPL+U!!rVp$ga zS$bY?B;EM0X(pUMF&k%V?hYg#*1_Fpzg)*aX*W#?VT+Qo_;d>lO?ryL5zgZ)Puj8| z{p!by`Nei#w(;bFYo{Lfw^6Sy{kL%5)ynz3v#ZW7eO};Tlh%fGroxv@*3j3^hVjSu|$E32dTC-V6ouV#(7DQ=We^!`03wahxTgU zRg1p&1E!@|&4-$$p)?dH@m*pin9$sW@W2E~q-g?TPMh+cxeJj1wB?#gTcLmgEt!G{P2N^M z*}do}RfaPhEHdGhZ`VsMyD=E=3BTnF#JRQZOzb>b*pKl%jL&Mxq>r#_`Wxy-Xx8J` zu=SkqtmfuaNvnQ&2)Zm@+hW(W*2leiI3oFI55gK$dJ{pT-fz+0Pb>Pdx9$Buo?#kP zgW@A68yA-mEY(dz{6W}ku1MTS;z&7#$Ch-)Y_LkZrY_avsU&ZSVO}0OF#g70Prr|m zF~vy0^-$m>;qa3J+eb*ERegR?(fbGe&mo?%JM~3axcajf!5z(M-q`c$kzb1r6CZm| zO~5psTixBB_FbI5ye4lQ*tBT0crT7xG(KiPH|aAb^us?wAN?!HMV&J-&zp!hS~ilO z!OfF;Cym8}@w`nMCc6UbY6iLGRJ@+abFdaop1Vv9`+FeFK}VgRjSbk~}e z=JlMouz@6@V${9;z+7lLLV@jY<0q{-E3Lsk8KU1af$b}Pl+6N{^wQdM*d^x>5iOX) z?6xN?-L#}wxJ6HKakwSsg6Wfsdd*?k&jY35owE=mL-A{__dT#7(SeoLWWQm^?7Y}#)0C1oEI94+(kJ#eDl-zyL{?y8Z<9sWnX)- zkYf{qgVY7x9M;J74SW?!I|ZM)P~YlFqYsIyudPl>g+?Ai(G-NVIsWmF=Cq0kAUZoqMQ9NDblXaxF)%bm^pp3U1t{ z%Ai21__!DN+g~Z!-~K&|r)Q@<6%WFBnpf`x2z@ABcCQY8^T|Dax-c&;z1rs5T}uxm zhb?`-(KJh67!tz!aqu@jw){PuBgbR==mBVr@BZFoBIV61jMq|D-BNP#Z(OkJ&o?;Q zsdK(#uk&c!}fxSqFvb@A>P9UNT!PAxX zBtnJ5+?6@C{);dKG(T-jYPd&s>}9YFNt}%UENUDune_?QBfG_c@RAF;U5X6oRbhR% zb3Hj4`{8Krc*cI=;s|@tyX?v@$${J zl!Z^j!#~x{&GLTz>1)Y*UR2CT@O4~Mk?9QfL|=%A0cD#`&INRx0};T zyqHq_{<)~RGRwxT5IN?L)LgbyO-xE4>rzX-C7;XjL|$Q6B!~N$b7~FZi zNl!saVX_%EKNAUk8G=@Z)2JwSYRv?*l;j5cgC9d3N&FYcW=7ZQL@fRg|9bLbF`yE_ zih*RG^S#Y%%-3|et)f;}*BJLs2=&?I-RK0|5TH!FXvsUAC;>%6d)bXcBhH47XS_Hv z0ft8+&B-&IZeIl1ZxxE5`zc92!;qN^16Zt&#L=yObFUD8W+n9Y+v>Kq!KV`Z4nRk0 z7(V_r8<^x59@szddEq7yUAOLI_d%S`L$U*NSfRU7SFBrI+U%6>`Bp$oofVn$*W=#J z%c}=QEZF`tkY%;zO(qXO9dVr~3b*r{SNQ7^MRU7;1Xn1I)t<^@B8P9}f$o)53n}3a&mpdg?jyJOV literal 0 HcmV?d00001 diff --git a/docs/images/consensus/consensus_overview.png b/docs/images/consensus/consensus_overview.png new file mode 100644 index 0000000000000000000000000000000000000000..ac4fa65cab9ab86159db94e16974d244397bb839 GIT binary patch literal 12297 zcmch72Ut^Gwsx$jv`)6eeO(KDq`#RrU#xwkB1XSZyV|G(nbVWs6Q& zm6fZjtF45eX?dvS^4d6Y+i;k_HWRSNU@+=|k1jobD<~)!8yleZ_&SFS`wMRj#`A(6;<9-A{K&YV1U@)+RQ9(r}Ds*8v_*urflHpJOy z*x=W(9z{)RvBelHZuj7@yuZ}1z^hr_W5_dkvme>_JVHDqE5CRA=hm`myg^QT);@lZ zNFtI5#5Xma1R`l0UTj%jpa^8hfTn*ItMJNG3PVdjUgl_b@8QU=@(h8aI+kemywG#G7)|Ye;3(PeW6BD}+k!NORj_-6fFfgFKcuv+$ z@*+1=NX4_}=4N(w_OwBdW5KtL{={ zLPqFa&Zea3Yh&=ib8_Td?Zc#%YJ~n@TS09hcRQ2Puzx#;$jjv#(k` zhIEx%%NSaB8>YgC#7o8PhheLh>op?@qoxV#Q_<_6U1~~ZvS-L8X(zUh#a#1qJ!T$6 zgWBw@JI(kFKgR|Drk?l%AV5on^Y2_^85Z^X#dYVZ5^Gu_%HbmOn+D>4w0 z&}jds#t=BY`%_fNXI&jAg*>~6lT<#qf6^s)_dspKuM1I?E`%KL5_^VruTiI380lZZ{ zA7%J_HfX);Zan|fa~jzsr)-d%+~D6^2TLJF?};;3m`E+Q!?|^jxcJ&DXRl zYt6Tq+p`!W%P>sWx`s~XeY;M>QY^zu<6`Co?wPiXMV^)t_6aR#Fyw`^=d(@Ro3%Ra z^Tt`(`$`g}My{3J7L;?4_*UJewA(qD?F?vT8tx{1!0i(oPQomtxw})+;Gnfx38J<0 zhbSI!(0y+CI4sWNEF=%>ai`mrRrxp1qTztAEz(1$}(J6BElHhdgiqr{-k zi%E~cKK<=hvjy15;>461@`$8QX?T!e^zW66@BAsU{Y`B6oA`I5{~yF4y_)^5q+h{H z`kN@F{paZ4iASy^>HPV`eZ4kFF!e4}3eYbQRR zkz+968v|YA1A>vA7RNcRV1+T~okvK_4qY(5mE^AYxY}@zyDPG$&r|9F|J~ zn|Kke3PB6;P=~vKx>20?XvF7#U*WO?PGy(tmz`5)Z*3jd5V=IkXI*@rNl)^F1|WY0 z5fSH1(rzLKF8*1Dm~{As z{_Mv%1!4_w|8_?GCRsV5!j@z6d#0A{&#}4I&`P?*z~y69p#kBC+QY_Vo`HOzEOyPYtZ9C@uM(stbNhg<)Z+4aJ$%C zr}$GtnsEj#K|>9%@->7Mb$RaN`2J(~Xod4#AOXu0%?*iF4D-R3oHt`1l?9NUgkIQH))Z568)-y?JbI!F!w z@hD(oYkW^Vwjon|gA2o8XVyZ^0>)AySGiTQH zlcphNqWJ<&)JWXuNp2+i6MQ{4UYe*L2Z_WvClS7GP6n!q$@Mza-DP3BWO7i_c0TKu zu@y9w<|;s_CE}Qvg2i za3WK}{q4*-wAKwXt~W&b>sMPSj)nqk??%k8)0`0T*6>W8H!^tUtdGi#61`f(HNYfW zt?%_w8@h-v)e{@dR8R16p!#tt-m4Q0PlNa*5o_|{IYO@_0`U}6Wpch#_^ApnIBI7< z_!#sD!o1ER_QA1wB$M1NgwQl}khx^$%4jEZy0qmNFQaN9Q_k^nLR0CJcj@Z{s?qj1 zuVr-NOfKg#X-by2a6-eE9)W$`zW!4)@ zO90+ofuaeDW@lYSRcCiqj0ly{8lk4DCoLMQW>#OTmC#plc9k+1x|Dr?Dt3eM4-t!=font>nz-F0ShW4O5)a{b#Km`=a|zZl`d_)fOv~6aIji zx;{#Z0*)FnEREjlu+{FLN<9<{p4y|q4K^G*j*mw1+?2f0U0~=c{I& zsy2?NDn?J>Lzz!?RCo>16CA&C1`Gfbhc&#etk685kn^8yd}(TVhJJ~#x@T_ki50>l zd>v}BD~sng38Y3>BL@1tNQiSUa&hyVk^h2f$(~ks?zhT#CvVfq8;=G8;XJPw3fgON zCB=E0y)97g;*YXpT`eu2h>2+RA6OYLHY^?X?tJIM-8X%%>5S8E znx?O#b{-F-f{U~3p76oy7no@uEbNG_QC@kA8KlAicB?lL)uFQ!EhFZWY*LEaA)4J| znzzjcsbxk&7XtmJ0TDh;jj$}9y>DDek6gdmL|~$pWWR<7q4G(YW*PDUl1R>V$$$gb z1RPQ};OFRxg;idX^dzhyRswHaCTP zqlvJ=Z_qD4oB6<`Fdo+{EAMx(HRrUzE*+Pnl2Nyu4jvQYz~(7yk%1-pyi_k$8~7J{ z4)0kaqTHyhhkx-fzWPHbk>_7GKoLx5y~nxmquN$!h`uBxRq~eD@M5f=$~0B7JsJ=>#=BXy){;wALYtCJ-TTo2}n)iPy-EI64+Ogws`>0cCnME6+Kl{#=hA)e5P+P1j z*@Vx%QtCQRn;g(VqAt^=;P}u*ocl9#PsYF(YMZ3Woj&VY2hU;dTyn4URaJ?}G1jGEE-v_t~>(Qo5g(ANW ze+~S_&3P0veJZ6S6l2Y4CHgM0np`XAI$}X=-oMj3TWktqpm>ra_(Xe z{Jz2IX4i9z)J0A_m3)JRX~!QEPQ8z(t^>HDnHj*R|F6~gf$vTNdX1~MyBN!E7&=qm z0Rfty+&S!J@lB#RI%JYfZ(A-r{GXpr_o!c9doBt&;=^WVE2w8hYk?EnZ{a<^Pl)s9 ze;sdQecQsG%A|QGgb7K6th2R=%A^atc0HWochr7Hhm?A2=PfG}a2wT0;TLvVzLq|@ zy*4N#MTE@sR@|^@D@zOZwe76rw(J=e?WDvvjQq@w@3}sI9%QkA!-N}v>zlB&+vPxN zUZD7LXuLD-+^psBLedQd0Ds4X74>%rEd+4uP3a-v$vFu;Anu~e=Fyw8|I6#;v!_VD zlICb|Bwg-CtlokaQzhWT0(pQ9W*7-sBC}gMwR*pCU9U;Kcw^v{GefOJ%t7;sLH_-S zCH82&eH9PQ0App?@I}qi#r5VwwN5)Yhk6VtGt|h!sw;fJplDQ;BN{3X&kl1Y=#tWf zh~67mP8kmO8!^6EU$NfL#C`T!A(yddnEP*=WK-|?Sg+^&n1jJ!p%>85-H}2^IzP8M z|E0T=D=5lD5@tnsaeJw0-K`+(%0m zvbFGKNs)E`Ym^gXBABn`2Dp?g&KjYik;x7)t?g2O0j$!@Zg~LmvtZEr`=Ubi8)ApM z=fncE#DAosFjDq9h2n?9?>-M6e5npTRL7Nf>hW7AkQFZ!p;)?ri8Ho{+uU^#TNXwp zsP*oRs%o)>_1$_}Th&Fx{=`!E!t&?&7L(@ag_M-eZwjC2piAhbrERVSD@v@=H@oBb z$kb_B@>-1a!Hwek*+y{w(pxFtqXqKM*3b2{1{Ho5VuN#YH~8zFL0p#6yKNGT$R_&p zo6wx+2-GAqUm=f7!eGly!gsfc*=A;u@}DFA0Djr}IW`#zqjgU$^SWLHV`$GZB|Vee zZg(&Ul=jd`CbRVVXZkO#3~<8aVM7s?0dvtG2F|gapEQam2F@W#ieA7f2ljgb3DC7V z1oh0waf65_K$OW$DcoAAk_f+*CXbZFO}S?lvkm)`J1NWYb19u0a)??#>zkCE{snH$ z)ad|2$d>^8Q0?>1%SM?3%GML}XYy__+Bg@u4Z~$3QXOdX1S4(O?}`7cUXv2-#76F{ zmRqxb@YC@o2st<2$xHWlU436g&e?vAn;x?R%GTdvq|d9NHo%cfevqxLNf=+A7A+2t0XY~Bss|AO*Q1=~q`-5`ANaozoBk}@P z@tQ}vHnvxNS%Mk3QyONu8_Y^{?yh+oQV3Kd>zTsWKE12(bN|64>cLOB^Emd_>eIeU zlaUH+=2)<0I2kxSbyds!%r_d%?pNdjm}-Wu-axNd2r1XS&fiVaPbYoXD42;7Hrkny zFYj)OeH4}ZjWnQNY3}4a|CRbRNq^QoU=(!P=tSJ@d+)dsEEGA{4&HwWgnCv3=`oRM zzPdT=EDSUpQO^&#hPD=z=JC-)Zt+MtR&;~UPHHu=0{9ct7P_T4Mt60!M=R@+IU+wS z!b#ug+uJ0m1Zdpn?ierO`8k-XfY-^W2ZYMjIpT}?9N4mu`Ijx?*rfu}CA$KNy_EI%a&J`~02@ zBZE_6OhmCM$avsO(x3-fDdx7?Sj*c)^c`{>+xH9u)t4>Rja|xJ@t^jMmpVXFb?;VzRuHq5$fqmkqC?$uYsnXO&%-CU&4=L@j1<w3|*h9$PS1Zbv?#tz{se_Sg5AXtB4p6B$ zitl$6NF{{S$qmBR-elTEbF%kMHKBeo6uW31u&W=W%hkPQ7OlJ-Kr_Ud{viLe(<0Pu z^~tAwJGfXN#5 z*zqm)#+E2hd;jjpcF9~9OZH6-u~3rH@qzd;#zF{b?#I>U7xV2fn#HzwRiM5RK`0*t zKwR^%?DlMZ5PdM8v3cLRnq@VwMYrbXhiGyD>9J%9Cz27>v^T6OA==#Vc-?v|a7 z^|%)hf!!tyc)MpOo5y~))hyCLuYWMg6T?sDeC8f3YYsmC-^pqaO(S>`H{w z%)#Bh?5XqWh=Afe;uO3lS*>FgfLH4>?Ew<=UhHr$c)cXvLa*de-~Qe?&aAmD{NKH+ z`@cNF{B!k}4B;2;O@t&?+gDCM9-bAuWMME_oLJ7}^=hp^3aQoWUuz$>d z7V12Zl&qPyfX(44LNFNW{bd>$twdNI1*vn?cnw!rmXMv~@HSB(OVSutx^q!YPrOCi z`0-y|*bdlUbnvQ(9MzyYa#)E&Cn`1h<1ko39zkGo#o4izCx3AX{ZgRKsBjbmIy3kv zmFfo0Znaw|IA9$!T7;-A{!dVkFtF-YU_gOpyRo@8jpn3De{AnHR15()faNvd38W+>C-q8#gH|5iVI)5A?Y?UohNMrKk>eE|)2J3no z=wwWRd`%)hHC~v3nkp)6i0A~jLE?y7!hf7>TaefIb%+^7hJB}^q191~`vu<26uaN< zgJ9Z-GlJUb{9_3FzOv6${&W$&->;`LGK%_-xpqY}XgTJx-yDAKiG%HZb* z`A0vlsC0T+s7hi%cHmHzsAVCs{tg0LB!qYMl?WYb%=xUA{6xMY)aP@Yd9jRN?A5JX8gKEDLKFhy|X;TY2U+9F89RBSY`TL=zo8kGMHH z80?MMc@@3h8dWF&Axkv+oQ%_@MJ603O40jo)-ZCNIvLV~LHEGjD@Gm_q4b9ns8Mw_ z4_}aA#Q2EGvrg=Tdkzo}y{Wv11TWav`cx7nyMiAL#YC97W`Og|2260^aL^%nMEK-t7` zrP`z@RP=K4y7wK5Wc7@F;fa+Oxe|3Br8n>=@G?x;$a8|%azhP(MfN#4epT=zoFKQb z#mJSYk`6idtBISTJUtlsvAV*Fvo$&!tDn%yi;3C$HHT!EC*-GJBl`4%{d+4{7d#+# zadNzfoC-l{JocIY1nMxx(loh5!KF6ZSIsuk_zr@3_f)$4tB2>;|(t2O?k?? z*u>DIsSnYz;MU0|92%2NT^9!~2V?p5QoqgrXg~C$J4pCMt-L)&<;cqr;XqNXQ_=pLYR`4@cEwywRNohaP2w0-{ zBh=VJid%1Zo0$l>?*=|J^5Zv8j9z4F$6~BsjfFV1<{jEth_{IM1hU7a*Rgu`vm03? zIZi}4g6Z*@3;x0S1KuRtyz^=HCKm8XT^d8Afr$YiNtZFqI2fU%1nz$R5%_KeT|E>c zTI1mhXYQ=@Vun8=4|3%WtN#iL7w*wTyGbMLU0BSS?8)1 z3iR#6F4|CgLIH5h1_#$6uL2A^+e9l{H7!M*+r&B6o%tj7OL5E+K=M0P^}KhoN2GB4 zM_kugJc&sEDmdb1I(dGn7xC#qvM*Z9suNM=WZoo(>#Zv$i}yg;n=$$AyJf&}b+uvK z_~lN5#M#RdM&zk12A-i??TZB4_Z8G0HX^k8n`{8Xw=$%k>IO3krHPm47-QsH=L946 zK}`HECB%*0*xRwhIpIgnKVEf0W|_j>WXwE!mB@_U#LTf|le7YTWS&Ta>ho?1p4&s{D8` ziC&W@2`U@ZT&dQn`sEEmtH#Zyx4A9Mi;3wfp%>s9d0$e*k8w;|nW6NOlFwG$&U$fX7nFg>cU18k1@{ zN6mWiUQW_BKLwAtLw4*-v^#Vi|B!^|( z=Vw?ISeD7%;XXFH+$O&*zRxEr;Zr~}S9iwxhdh~buEn9SuS>}_Y;$9cx$gfC zVZ?jW-3~(H7UeY$(_m*fyJqvtTB8ts{rME|PKYdMPb~X(L}FP}njP(y{B~1MJ~bA+ z^B|L<%2X$|!)$E&V~EDxIA8)Z6u)DPhmsS&#-(Mgy*45Q8tqZ4=ZtPtZ^zkmG2dM0 zZ13KxG+N@j@Cci|mo=})(wSsw@x$S%Baj;dWjebc;OL^>n#}gut&#EX9X&gJ-~C6?g83C{+09a^s?Ix9mp4{4h&sQi+g})?$IcyF<$%-`-@_;WCq; z{`C-c$_LW(eS`5$hmrte)`pJ-UY%i$#$@tWwVx-fDXV_cER^Lt!!q+TIkh$*a#2`z zQ-+wO-D8jES1(r<#t~*pdL~vDR9x4=ZXf@>fEoP}hjNjIP2XBLS11Z!%zmm)p?4}u46@|R75TRnHI~{zd zS68gr9ukr|?#MN$Y+2T9PlcsWL4P#JL2rhukWLUN;vWx?`2>e)P!|{HQ?ZgSm+SX| z!>;ILSBta{*)Mb-gqNl0k{u^IiUeG!1bSu|AA`CCnoX{vEFD0(X3?KH#{dd@O$SCx zGXe@N^f>ESl@6UM*7-~9Cd;T|#EdVjoth)#zCv9rVSdx4e?noM3`2woFlzJ9(|j{+ zU3s^;b@^_Or-tKqSQGn~bZ1$x+fi8R3r7$IdHBx<6rZU;oDHfF&etr`V9Yu@pig$I z^4YZ}LBpuKZGZNsJ0fqU>|-x(=mkgxVneASJb?)AKNzI`!A(dm+!U7J4z6|diw?Dd z2DCmiy3`8cVreKE*4WOS#ZCV(-wFdwq|*1DQaJIh3I05V`V~K5lo?ef#rm=Ikj~nb z6{;Z@HdE4F3BVPw%Ad7#G&2l5xcXBmc`^Lkg{w0z!Yo~zIdXmz_FuZWo2>2@y-x|C zvpJxPQK!s_8r5WI$fRs3;E!9KI5dP*Y5+AeK&p>4f! zn?B6MV0zZ^PBz?=R;y=o->FC_)qmz}#`GnUi(WXo!@ddS>=}rPJtgBTRmpBi$8oEa zUUq^BMNMRTuw019VPM1$p3w+@IgsnQO$k^FSepI(V@KLfy=59*2rk9Y0+%Hz<&oMT zzx4s7hx!RAf0i&X7A5EoLM}lX&cEwP{hQ?d$R2WZXT% zprPf`c~qH+Z&GOHMAk~3oXo_FlBuq0*W!ujM#uyTnZ^3paepK_OGdVrvxoC$9=^%- z)YjF~H>zQG8#;QmvxL^qqCsI@tN2mjCT@plvB`yT@K>YdhQLwwFRg*uIzHo0$erT@ z2O<5+Xt>GW!Q5^l?9`7jZ23ikSeYwtKqF|_7D+1DAGWp zv~b7gx|38+QzK_yw+C3_ApTqKA5AZCZ(f8l69bHjXCi&D9v0j5DM1;lB5}!b8?e>j zPQ|Mv&8RyP{@K6`^_@C)cy>*N(WaM!VQbAIG#ZTy;>3IaP241E+nqHTNZDg*(emt+ za5@gjs(&4)2=SsfYmk^>ErbZ)#>w6FkgG3=r54q27%*w281!f-*)k~q-NLQnO|fW_ zVm#$(-411iCWw}lt8*F;tKr(8Hp&JDHQtud++Hp(&@`(&us*7%=(c)^8+#;lNBwixvFa}{IQ1A08WBoRZR`dn1aSa*dtuGcU@kXVM-lRj^cuC{odw=(W zd)x-~2My}bL%Wt+^^OwO8>(D6 z`piY0WK8*+Bz@r+gUx_4_p1Wq^n0hqo$qB8Kr1k5Ojw&yFvrE$ALlCkJFVW&(~9^d zM=FYo*{yb{8Kc;GkNz$G|KX*0oUD+%D!L~4>7puinOmL@jXkZcJZ+?(yW3Dd03yP7 zB?N>;1VqI2?}|$ci%8#<h}264DK#Dqt50 z3Brzqv~PlhB#?rHMEv1Pv#Jmx&O+s#hV3U%_xGiwq)13e z933440|VFB*U!$**x1w9{71cdn~R>$9|C@lS0Y#VBQ_4-v|bwO=gOs z+}11sA;G7_=z-~e-;chIt`0_Kh6e`+dU|>ig2Jh(siC2v3JMCfwY5jHD_V*w=H}*I zU0vnn*FYoW~nVFgQ_V(IJs{{oFeSLjfT3U8@ zcUf3i3JMBhVq#!0*x1-uU0q#cW23gVc41-Rz`y_<9UTJ$gQlh?7!0kG;zE)IJoS&bs%FOHN=rA%e5)%`fo0~H*FgRO=6B7__Zf-t2JkZe4+}+(}$0qRe z^KXqzLjyu0A|iZ0xc2q+g@lA;W@gIB$Sf@_EiNwRiGD0dwV-BE{>Cv(-!eCG3yFH+(Mk#c<$z>4*%n!13^P2D35bvuHOa@;GQYri%B9NA5+0knhbfu+iH_XuZY%`Kb?u&@Tz{%XWw zo)KPPBbNsz1Ax7yGx2miEE`oEbgewU0=Q`$#ajD0vsvU+zNIA=zT(AKi>BTyr%oMu zJE*}+^;-NSGLqKcPBvVcnj~(Gs3yR*ujlhs6xD}FZ3}Q2=U`7sJ2y>OBxofVQEoIJ zL%}zb_0ngIFeCcRlTlQaYIjnCEAxfz;{@eUBg*dj{3dAhv zdoFT+Hw&7ErtA$`Fo$@oGfJ&FAq-7_K`A^X zlWXrZP|rmbHM?IQ#KW>Q6ct>l`V~@`%sLpw`*SoXcSWJNuaputZ%zZB6mVVQqt`@p zNC>laB@QQY{4{G6Ss`qW*`k)Uq_~i^_^@+V8+4KtVlBp*`EFskiCC|%q!Rn3;>b>} zs(nWoO&n(KU|&4b;O>jfjrVLhtn5OvPhXNrs&j!C3WC3OjFDHFSboIUM^2rtn*7xJ z08P0EJP9VENidgsGUxa*&}?(zJ(P?*@ADMp^6Lgv%^blUNxUYI(bjX9x6$0s zVei5z-cp3(HjY-T(M)cPJJeW6rbN~1(VCbB%>Lr)CvP)t?u8v`)7*4~CKR1~7HWx- zG?Ym{qUp+!_9q&iYa#d?-&v72n`Gl0(IlqV-Svs+kpkbj`K69s&hT7Uholo63aJdY zhY3oct#&Iddq@cdV}#k5O+K8ix_Q5~|Mlvk#xPsp%tAWsel9x}@^Ex9Mm7JW8j`fl z^fCPHmEqDxhJVUIz#=Ea1e4_X-b~+!l7Rozz4thC0IUeWx;OV$Y+&^}+Vs!$OSB*d zRdUjrpEGDjcUE@~j>mXYr{YX2NyW%n-8RxI<;`<+ha|*$}tcpXr;)e~l}|aA2pEG-B^AqSFIqeyRZ!HjsI*&>tu0^V}Z220|m$AuUW{&iaLkF@khAB{l;2P@wVXdUaSsT`=Y7 zX*XbdMEo)(6zRvF0WCgy>u8lvXB_J`4M*65UI6<`Jj|oHfLCQ_hL;VuaL`9q^CX8w zmjfDMEM_Yda8q6A0v#f2HZmuLa&Zr!D8>6h)CRdpontyt)~QZ(?04m-UQEU(BT^8{xbt6I~4 z1c^IX-HDmamb(h(K#_m12(u06FhR#Ow&IR_A2tAv=^J?mDw6*F)q^+^NN$f^7hlfm z6~*Rr(Dp%8-cmtvrHyOm9nJoOZVh`j=`|AbIZv^ZD$U~K9k|@}TE|J23D6D0uuSk| zR5JbNCScPh(GGx50vZi7N3@~RKsWmC<5-yMCI^h4dM9_-3Ug_CVfK1?>SO;xQWbiV zj~E($7e%-?c}9ErZA>*Q$8l>g*2_UG?u0PIjP|dHs&b@sP3q|$vFw|_xP*F-+WKr1_JbI&gXus@AB{!hHkMjApD?_dFc$>v`q z%f2S^ZyDY{A=T8%9y4CQ?_U7i!Um)QA~gzc@x)d|Gbb)$F_I_f-%A$65>dY*!zD)7 zBQx_37cKIm=6q&i`vC_&Xm1yesk-{UAq?qErF&meDAi$AyOZ11Gn$`R5|hMrpqtB~ zAr@@BU=n>0-q$1$%;AWE!%mYZJyDvS3;^{FF6!*;oJ;y#FklaVZv!;fHc6KYt!c@4*Jt1RMIPSa~W&~;NrZ(MgR?#K>Bo1p+%w7)M zbQZH$V1S~6Gc=vfB-HUHJC^V4X)pB))*d`7v-Iw`-16_xT2WR`oNR*pTgeA8PzD!A64NK zxnviT*j5b#yjY=!sL3IQgkct^gt$9;heF4*bYPR96EPo&JLwuC*n73=>FH>BxK2Ey z;SUTQM)(Mk8^MJ8Isb|wYjc@&_oR96`pLo+=o7uroF869(&q-gvI%!7x1Oh;Qq86x z@p$hcH1GRM+Jy*&)0)xsSX|h5+7zuyR-!&?nX8WDj>CUv6dqSO#56cHMhZpkSv#iF z0#*PK+_}<+{xko)9G(z5O=7UD(cpJ0(Xh84#+yx?;cBoeQNCj#Ax3ktlYYfE``hZT z6g#oxL`LMALg)+%++H@6{%yaG#eL3Tndr;2yUaFcq~VOuJPaS#R@)&uWw<7zaVp>X z_)=48CHD86>0#5NVgq~*Lm<5Tom2(!0q+3G)EW>!VPqP#(29mb?0O|PTQ@Tt*a4lH zK)#Qe!BDq&Mnb;}O4ygAeKegokx8TM>vk`}ozKE~#-7oX1;R~QtUym|@OW7c0ah4P zM1v{2&o?X1K2UZbBk_p+Kt>8S-!WC5igS6;)nh^Jh5}}hd}9ELeesyyf|nA*I_Ho; z9C9rvEDX@Enr2aP9|k~jGiY+=7OoHqXCPB9q03Xq_xeyBElcD|71km5stz?2U5?7E z6mkHas||=I{C*`n2x2B8#W-Aff^DfKU%AXlwU5E$3UndmZLVHe$E#h>aTeGk!G&Bc z@k7m}szdx=_p1rPc8>+lT>dwML)K&Vs|-~Lo6SUAfin>6%!UJ-9neBwVkp^8G>HwV z{rn=}C4Cvs_~v|e*br6p2ZsK5ly!+n%1lpyNh!jk_qmi>Tb2Iz^vu> z%NNYj_||*{$<=kv_V%(iYy|y3;VSeEi#4Q>l29V$t#44&$=7sxEc(Z4Qc7qoP%Zs} z!LMoFQo?Dl@pb!*TvZ~+I?!TD?DS~Is}Mt2QcZ_h#Zkxmy#x(h_cCv3^V-$tKvY48 zExeNvkSn=P9jQhNFVT_cxExYi0UmGnZ?Mj_gMdzD`sNT8RR?$B=p~c^f3y4E=_VyKS)B1DBtF z!=;z#P{=YTq{t&h%qdi4()b0f9qnABK3U!Ax}4LmYw7W|)&8VnTCE8bad-TIV4fhh zPR4xG$g%cSqYGE^R(nmJXW7965}N46$Z)mp+|x@pa$yI}CXJ2>CUi|nyKeWrwwHdz zr>|DqWwTnM+27Sq>G)LH3%8TXia7 zT1!Z*u&eLzFPC0zDUfy<`*8bYlRTE8Y*_LF}*=CzbFaW9! zHef&8->54dnZSl9wB5h5hTm`N*&rwFN=OErx>kIS2wt<)X)_s2a{inmX`e{n>`EsT zA!XR5PF_X8e^YV0>L88sMm=btDHSGqn(00m_)D;F{|V-F}`Sku1=4--|Ig!70NpsA3i zQ(`z0BWm=Js-zrHP;5Y_rd8dq>uI%SGhvm5 zLJ`u8BW8K4-vYyXfrge6LiD2t!%6$Wi!=j*B;4gxxMI&}`e5PP6U*Ov0o&-g+}+12 zz}4cuk4LLQ#LsL6)eZeNU3TPU zP(?r7r&YJaupjxA4ng}v^bI*@nn#fC$gimM+Thl~0LWw9k>m6oi;!oIbhpKGha!*t5PtlVINYZ zRh6DLxFTu^&t8lBhM>Qgefn?cBzrbczTYiDm%OU`xaJJ$^Ts# z)!b7-Uar-c331;)s6_ZokoVLYs$7mbvz9%!9?K@w=SC@v6CfNJoNV{z8FxvUSVnsr z0tk^`IB1&Od_JoGH^d@){lbfx8iGHga9G@9xn@p5!V#-cMXfFy6&wFNJPNt|Q?+o< zd_&|RHB`jm@{m$XfXLCNZX3N@A>AFxA`>~ z(V(!VXtOV0K3O;#iOw7@F5{V{))n+rD@Mgd_jW9-cU0OlqPa;TXJq93!7T>lard`) z2MJ6bGyj5S-bT=B#I**&V|y9FSMvMLc_2CtTh=Aj+B3(s$o8D*7h$jxb!L z|JS=vQZUM$|6MgS{x|~+aGP8h)b};qe-dr_R}Xca;U!-ip*r9eh9kIkO>MflD>)VXrr5=Mqqi9)~u>=*&)MmniQSe#ebOYo+S3ij*R4*Gm)96p9aBa%Lvd;Of5E3 zld)A@X1kkBjbe1#4pcq|y(a{Fcicl z8TNQ`qFe0)JOwxMMlNv*NrBv~ObqWIYp>nOIm5KK-2BL%6?rHS8pT}w#3a)lXAAsc z`VN)I>9gFUW?6pO?j9+6T1^xu zh9bpZDk~;(pMm#tZDnTo2bGpMWsHxcp+^kC67}u?O+D zZB!HjWltBt+tPX*{wn?_EB9OyKP@+luLKLbEo1fF4fxZuf^u_Lz*XSyLD$`*8EW=S z3lNa4+j;Uom<=2LMo^Wm*k`Q~E5;V@?bN7W8e55Riud#q`X(N+D4Z~8PBChvfV@pF z1Vf1jKqv^X>Xkqayus07ubpywe;}3%&EA>-cU}jz>GgA1ZOgu|Zl|?umn@~nwup#7zo-Kw@$*O~28Uj$Bqbau|n(LW^F64OONba0@=tdo;7x>o6n(`lhZ7}!v!pJO$)bh(`;AH&rx0=xD%ru zHd5Y@Ca+Q3U?|5bI#7U(&3HmB+10T=LBve@^nFvi+H)g2BNe(FN( z3mzDDOB9XJ63vvobxv!xMa>g15N;4w&O~bXH{$9M;xS1x*iXRbb~>?O@24Ju=#;$Z z&}p26ubW9I@dafP=%=Alup&pHIs=2$J|#Y90{1r)d(Y7w7K^hmWWK~A*P$ugd-W=r zN~wSn>5J@Sc2`}s_2aeD#?-pR8wiS`;mrP0tg5B${O0Sl;E3C*_~2?AtdzsMIF(i8 z`O#=Ud%bVg2CWxu?Q3K}o<5m)ChDsyu?R#xpCnjb5tS5^8SkK3o$PDa%q0p#FBfF@ z6v(#gNaR^npo$mWxB5xNOO~~6D4=%6X6&goiEfcCi7fhSs|E~KmE4lx8RQSe&MMn# z)RK1F)b$Hb=}SWAyqbeKUKe5YiHSEE4?5!MXG>|SthsKJ`jA-!3YZII3EM3>@M`B~ zk`ff9jm_r;3MAMd*X<^WE0iuH@T9 zeF-w_H7OyUNIpDd-gvIxxt_Q1>0|!DoHnv8trn=h3X6l-C=#`k3}*O$7s-c zMxnzG0b{wIa757e4*_V<-yqD5YMNC$k4?g;f^3a5qB8CG-7wyp*^v)dr&evLoBloC`bqI8 zU~gbc_5~WzC_Z9Jndoo$-ruOMR!L51cjS8J8iuvWz#4GX_{3X#GUe@e6n6q0k>yZT zKJUAVJ{^GcNVGfVAG{DCkx_r68umQZPcP()y_f;U#ckVezqc%=thW1#2q9sMs1phY zb&3Ar1+EwoxnwC$^}sRNep0C4NH~aCl5q!}d}qT3roVgDR@F%|kN3$1f@ZdYOJO3# z6eVUY{s!+KbcAvfBP(|PndVqfqTjC~+5g-?>7v=Vrr8TCC$EkN+HU&185C{<6e0Z#rhe-5&elf&!!etfW08HVceMG&~F*&JR zx~yB&RyNVqy?dt~cCh zd`W30c!f>dO)`b&yLPR7ydt=bSc1W!S%T2g)xFsFH+@HUyd)|oNlf38PFC?R%+C*N z)^$mx$-f(_WmR4B19iShGu&(E#>M3q5`RCuxrxr8TPiGp7>%9XvEa!ZM=t1G$T4IG zfh|1brfz-c52A%7f{ApiPB`SQ*So&4J=BIC=?32Whzuid-!gOdTW z5ulr9z^VkaLtsa>!f8>vP;J;lGIMK{vMwQMSxxS+5!3Vbc(atYx09Ev%MBC*N!^3#B$K-BFRC6P<=Xu87wa?l$ zUe1$Au(6GAYL>g`3lx2^g70eM=|^>42_2@+XGQ-wjIj6|(LJzHw0Gy#Vywi# z{WJyY375XW^rC|c`0RB-W&dYVn|I$n*mPH$_MTwsE)lhKP(i~f87kwj0dX4pCPaF8 zzBu>d4@Q!^E_bSfV0-uEWI!YoJY3-|<6hZtUs^}l;IBK~1iqvta@iwW^D59&rvcdc6J zy64hCWv&12*!8f3I$PBF3zN2NwlIJd{8us2m4HkmOm|`M@dfT1!u}FmbkQz5)o=48 zCjE)T#hrJix4kZx;}}ip9}CIdE6lyA0462XZ2gIa@N8Ns9hM;7i1uGrLLF6c*pZyz zh1tO6A`~y|MtR+zciO~q6Do98uC9;1{i;OhBM?w7PHb@8%2F@WwSWc?I@~ou6`_=8 zvQw4fGDh!VR+X3SQOWTKd#N1V75;%<3f|2hs*XWL*RLmdE$DA{l& z3m&oWi_9V;uibDylKxCd_r!X4Ibx(+v#vliU%YR{(kaLPlBMA2uytqSFZ9;&8}_B0 z1NOR&4Vw3IIA5%G+yc@9mACys&brrSX3u{Eg2J?fh*fU$BZ9M6_U{Lm z!A(7lR(JB%}>$3B(n%bBuOxm^-CQVl^>PL#1v-fRBkHb;DBsoh40+= z-@m`;8SF%2?@RI*qw|PB?tMaSy`QUZD@%e?Nopt=9&@vl6@{-1Tr{2!ZV$9)bt8uy_4vyGt`P?j8H%8 zY=GyI82HTms~-6~vKqIUJdl^f4;CivZla1F-)cr^8#|<@gpDbw_&I@$|5NQnCA=ss z_4je2k$Ixn0*BsMPjUM5p2Un$Kkwy0a!aP!3+$(fE0-har(&V;wO0MbtP1Ql2@GQe zlWAKIKj(X)0z@{IjAC}6g)eF1?PP<{x*bzk>3{1iaDLPYfRiYt%v=+#Wsws(3;?^F zRe)wGEpY9gh34c^27kg{4lFBa?{hViY!a5eH#)Dq(t_b6a-6V*pznAKC2tc@2>Pvq z!dJzgK>f$^@m`lj1jADTQU+-wpI=S*WrG-~Epq}}((I8iw9A;TIFmduc@UK}m^cii zh6n{Vfml}({0{B-s6teY(q%nk4*NNJF9)>}XEVF+voGEP1}|NOR5SZVCgg$f6}Y8M zH6Jo|gFOz`un(f@lT+B{NkN}bGI}=OH7^t$4iOcL7>r{D06H*P#uXEDCfyt|dM0n5 zSvyp+L{EDw(u_yyBM(!-zJ=%8MnXdt2zr^qee;~r zjWa*lPBy+9+H9YTp1(0jt+t*HnE?wmH+imr&qkPsiH+{}?Ytiwa|4ZCs~U9DnYoA^ z(qj3c>FM*!hW&QRZM?TJZ+_nANhjHr#OTZ;nhn!p1)+fXlXE8@7v9@(r_nCHwY!Vq zMNt7Z=I=02xRz@ZXYk}#%%dJKT}7T}+hfM@l<32Ox0AG!|0BAxU&=#XcMO*Z$Xz>` znTtd*jh*Uyh7^MI=GJ2H+*FG?JuY;iTgBf$v}iCfHJU{-0YqMP`dVBrm_q@Hs?0Rwn zg|;QP7q{sq$k8z~5D=KzdOiGmorw9FY@tjP!!zvuNw`4Cw=>7>N~Jo=0b8(zWu7$t z4SoOTMnDHvZTxd)j;uSj%ll76+LpWm+|&@@coCz>-R?Ueav8am zZZGz z?~pdMi=sh@%EQHFLtX=8*8Nu2F~qkFP1{mnKYaq6x^=aYVI{|XgNffC{JO-rxSexL zbv=ZLnvy_0v2gITL7738N6X)$FYg;>@zz!TMBjmr+O0{DhS|2ZrN~fvtN;|f_vwP< z!^XqTrpJ8F-Oz{(1YZl-AN*rb=599cq(bs4kzNqX9tXC{e?j}=8Y{BxQ;S4>yT73` zkW$2=6v(GovDNDTu*D^y9(ve4e0%jd$C%UT)X=oSe097|ZMsGw- zJzYNx92<+?4reVs8edQM``>jJwm)Y4X5K2ns0|Z{^0{aQrG(x3*tzb?3;i~UWY%DY zro`f?^ZQ6;iZBVAdN!`p`roF*&z1e|_bL;Nb#GATSw;tRWo-D*?h>VUhPkPEM<}?_ zGcEh|+zoONHW(YXjnPp-Pd1&OG9`J}{mP`G7DsdW^usUuBO|)$jSgQaxL=p5oli7+ z3qq5M2xXGAzOXn5wG>w++ulWG@d=k$d@Pua2nH~Bvd9KuM(KRj);js%Nq@9a9#N@M z48rO^5wBLR>V$l|RqGBz>8VNh!>{pDQ?%S3P+=^@Z^9FM{W_{6RP*G(ADMdb2%uY& z-Cxk=Plox(L@cUPit*6bQAdAz!=BHvwbn$+ASIdBCcv=>ADq*L{HI3_5`8+2@9wND zrn9tQt;KjUIfjBK#Nt(qtxOTLV?VZU5_PRxB|gG2hFs=JOIirlZaHpGXDO>i*M0MD zev0t> zfJB%trOhdB`3DJ0@WbXeLHq8neZUv>uWpwQXT!VM#+ADh?N0NwYqYD$ll??49ggJv zAh}9F)8wt9)vv#nw2gJzoPX)c_kiz;>?E>;TKC|K;%yt+Mt9DmRiEIvK{X8TE@@pG zp}{_#Z(8e&Giib#+2AlL2tY@IqV;m%At*ID1aiz1 z2fst2&~_x9s$lttp7`N|I5?W_f0RUKY(-UI-UlrfZrp_?WTKHW0!75K%q(TP3Op4b zXBWPBtWs+R=;iqGM?1W!5{X_Q` zJX*zIR{Y#o!oO3#VRqv^I7OFVF_->j1R_;Esu)aqZd!4ud-Pp#H)cUaq)+QEG11gi zOr6{?sq6*L-3rk%%=mS9ar(rQj4+N?L@2Yr>YMTt4FsX}w?QaQxbEF*$Q*`eOf{ju z1!|PVfnnsppvnBC?J##!LMrymT1eJyL#BIFQ*X{5+|1tFco73ZY)!FnP#2~LekE$Q zXKf0U@y2ia1^>}Laj=!54_rbx9xmEyHhahqjPMri-H*`ia2h_z^l;tVICcG%+xAaw z$|z1IG8n}4tGMv#00p0#)|ewUaxaBu#d4)2AJh;%E8Mc@Ucx4(9Fy2YV5*80aIUPg;10`(lV z@qfvcr&!tLLJ)4g&H46|(Zxg2b{_0?kJIHhKa58##5;4Tvz3M;0RlM4mih-H`I_uM z|K02KwK%T{v;EZYrr+Q9xP15M_;F(pr}mhTT`{buymnw84~PFBp+^4{J53NLt;S+M z_5@f_jk&(@O5|}@^WsNLY5$SuIkl=$j7Pu$(&C%|mknX`mZQVOnE$KR&L70W-tLt8 z1yB$8aA{5s=HspZLxWHF!DC_@vqIg)0cmlO%w3Ut8KAZtVp+pPg1)-M_{YjsQ+Tjz zF+4-<$5WPrY5Z5l9Ueo=I;m%B|5=*6k0-j$r-i=AE69Zj*J)g}WB*s%6(?pDgU@Ml zqMP0YGZxaO_bXPW3?m&t`ng zG;xl?9JID#Z`sd_=J4QHbYr!>Kj7d8bgNqiCjdrh#h?d#lArv0%->IfO${N49#b^5 z(pkAHRsQ745VIGa6t|JYf7qWkPDa04zvK0;mz75Witz26hp&5P-k+V+-k-7y=KYxtFWC%YkZFjtnQ?)eTkDt&=wtYvT$>*X8Fqp%US^$pTewUoHGXTpLT%cPW_vZS;gWDO=cCG*2X=_EyriHZVCbMqL zB5j?O$-HU;-F=9VqS4$EuK?F3Vf2q|?|1){iWhAES1M)|OA!i+NQ-rU2*Zg_EW6az zbZI;oAnzA+6_f8#wOigDkimEmVmgqyWIX@b@qUHHW7N!s@USf;siG%IB8 z05Na&TG5_2HXXGO$nX_|vSIK_megXTHifSUMR`HI*}wPVEX3fls$ObLhrvc7X)~E= zD=#cW$oJyOH%^G70(v42CpfyuR`@7!bb(e}_J?t@pg%Q0{?8oJdmr=kKd2N{fC#MXob9DAQ zFa0Rw~xB=g}QzCYw!Jc4AY$U~4UP~T-sE3NiKODwQX)kmT9FZSt8 z)BC=C3k9HD0~y3m>Gx>pB_)G>lHg;Qa(s4^&g3se{pI9K&2u!9DgE%aEH z<51X1vJ7nZ8j^Oh?z;3FdGI-xIUILiFM23rf~p~ykn-ZbR{QiXy3<-| zm_5}f^nz}XhrdjTiJ?HP-#Njd40oIHls=C~yXD7|$?NxHYvW9C3;_VS)A03y#q$2h zK5QzQSO&wJ>LUFJoCn2I|gb@`;1 zwGLp*dVQO|C^J;jq3(*Y^(jOO)(RLcxtUVUGDDy=bV2oD`0oClLq%SpLljr(syvqw z5JAWf_QIluLkD+Ama*8$FTd6!{{8o{hqVC@^(=tO($m_X>%TtaR{n`Ve1Uqve|JLu zG;EJesG4KA8;UZnu*r-Ab&ZX!e_<1vfX$`OI(&V}3MJe%^=PKd3BLJb%k-9_=}ScP zo>I6KT)KS0>)VmDrFb3GZI6@j=X6cNk?Q%1PWkc?JkTn7!S_rA1|SxB?0OK@e8DUE zl;8E6$gD^9QO;lAh!zlgqNwKS-}H{#Lc@`qH@;q$9Z#t11H(MW1gtNW$NuX}X29P{ zK+$bK{u0FtOdScz>5!^1<70u)r8!V-XgRl8uPMTW>6+<^ko3UW^&GIUtKfZN7*_!% z>)%9?{qp8WK@+DVrF&aHSnh9QCTcB;A_{Y-x}_;s+j%O6B$hLRmdszr$~ZXUEn z25_I@DO#Vaw((>jnzq(e473_>QBcTf(e=qOcl0nc#C1K}}TMTmxjp||0| z+m8{fI>tB%&7s0R1XQ-UXqPAUCw>!davpvQ?B;sO2+-B$H2@;8UGWzeA@kT`FHF{0ap?mWH$p(gd2cz##qSxF*>$I*MsR{EJC0H zXw&_);=>L!_!BL$J;cqWkRTn)jU1Y0c;7&d&Wp`B6?y&4!o5<+1uxU~n_5jVWwn^9 z%rm6h1U zzq}Jg0^tFeNO8F#s-@#1X~l+NVO)f))+LbRMd1_=3>6K3O8$;%wzG7zjvVZ3`(h5(k!XKjhvn-9Qg}bev=&V8M-8EdevCRL_++{to_Oy(6S{^{ zQS-`(h>)Se*Jd;baGM#_(S2gw5p1`U*8NkB?w*a6h9^zR>pkYc>g2ID&0oZCH9*_i ztkD!;Zw>w`yrI(q$|pf>F@+@zn3)a}&~T!T%;GC}_`B)UQM+JBGMN61G{!SKFsiV; zF2z9kSGL!*&Xi$r;XQVojLBB`dtULLpV>(LqE=JkA0BLZIpSpq$bRL7KYdR`+E_Y> z7Vu%a#)Lu3*AQ_>+GeLz-tZTHA^>AQ^1qO~)Os6qdDzqa9ZiMZiQ@hEU%pXLZwgGJ zpUkxq&k@_b;V`V2v;ne4X;4;Jp^Bkzpd|e$80xUkq{17 zS*g*2WwcY1vID5!N`_&#y~flbwT)m7R-~lU;*@SAd;cfP;sbon3&P ky-T+b^WOp-oXl-3J^y<^Es9?~A^=HFT1l!x!X)Vb0UJ?jP5=M^ literal 0 HcmV?d00001 diff --git a/docs/images/consensus/ledger_chain.png b/docs/images/consensus/ledger_chain.png new file mode 100644 index 0000000000000000000000000000000000000000..5847381cff80848a037c3429ca3735a9c61b9c77 GIT binary patch literal 10403 zcmdUV2T)Yo)8{bc3?ktX1O!AFqM%3|K@iCjBxfW^&PbSHB!hrTlAI9)0m(3gK|nx~ zFp{$%L2{a5U>J7%zxRJ%ZS8LD`?hMgzP(jb=X9SweNOki)BQVrZ|oCoRcZ=W3IG5= zt@c=14*(!U000EbWQ2H*h3rWqz9Di{)KUZhJ|$6NZHVzSfsdZ55};~?eG5-;**(_N z0sw+;0RUkU0Khq(6}AZg_=y1kzpMcOnRfsHvuAdzt{i?~_lc%~%GK2szB@ED)ZN|P z-`{WA`rf#y_~hiIq@-kNX(=KiqPDj7*RNl2INZd<v~Z(b4hVy?fc&+0-mlDiMl*h%xi3 z+*FHG1_#Ot>hP0Lk*G#1iJAyW*ofVKE{VSc1O(_fX^1I_nVFfDAPUzQuE}}bpO~0v zZEgMV;X_tdR!T}rbaZrLVq#HI(SE|sv0#o&GbVj;%I!FAM>!hMA0&aQ^lMRE<F+fEZ$q$9RI=^RbB!0Kk&<`$wSW7oGwD z+$vU6Rx}8l-OULOpKWcg!VG`2CA72KlL(70NcU}8kWb&+tJ_c3e4CnD`7HD*$%l7_ zgz$^;V=fIy9{3hlxS|J^%Oq{rkP@%|A72Q2Cso~8S*UkyTW3G0s8l$+J8k9QiT5mS-@*i}EDfV0`m^+MFdj?GH zEj+aL6px0j&;GEhpk@1Y`*ujf;3H1|0@}QdzgR2z2NO+VtUy_%Yqz&alZ=g;M_He- zVt%~f&llfk52jBD*R5zUn|`h}UF1AYj8tS7FKtR&YnEp8New>K!g|qf?-^C9pC2P-2&1G@oc*6(Dy>AV% z7{Q}kcE6g=B@>x>GVA>*yXlit{n}-3NK9-1qlYF0j^W)#J&P=#R4Gk+5Bpm<*7Sj+ z&Ich<`F$IA+YdHGyZ2VJZ|vQ0>Ufbmwz}YM6a3|wy~X~8h(tplvAcv(-n%7qh{{gF zL)2!5<-M$#3IEsJH9XIN$d^r@2AP-}M)T^{zuOI0@?Xym3Z|SXQt7#&`uCDR;jOR!k-(&aq!nauV#k(gtoQ1C-D)vAs@M8%K{a7{8CrlDf*VH0+- zY;Zc3lv+0!aw8NvFm(JvV-QF3k!D-S1MQp~X+C-~T#nGt9e1?l_b`uny0*AU!*8~H zf_dmMJhXk7mUhW;5S$_B_8+n?x<`l}?Zq=8!#BY^L?rzEL`>bfmflYy=A;JHU(5uu zt}`Vzs^;8*g%?K@p5(thgFh3s>@C=$cs^QTY%R_{p>^{IO5UNQ%@y~3L^#_mNyZ*X z`}P`ZDQRlY(%s<0!F5w+01T1%n)&{^o!-i0vK=>#?!fDx-oqmq78kS^+uUF+h?Ne+ z4BJAX<}x*B^7GLM#Dbf1umoEzUM8sx9{LUF1Dm@5I(kGP@*~<18XZ1!aWq6sw1u8c zp4?0#Z6k)uffq+@2&~FnRSPJEylq{!DWIDW)*EgU+;m_f6tcJILA>Z{mCO9D(`oK* zG$Q|X8-l?!awWwtq&o+p_}$UXS#m_!RRxsCsA3bpE%2zodBQnzFdJj(!uUykS^GOL zjB~4G0b3XBM^1$VwHFhKB_hGSzGNfXDGA{g-HLDOnX$rsDRIHwAeY0>iYDN!Zw<~Jo*E6v znpy2$7664>c2YTJVU-|2LTEpZgKAm>qhxu+wOBiVJCdrZ{wYdCGEBtmAN@Z>rd>W? zH_KExbzsxBj~yS`Je>^gI~%d0vUDLA%L}AKPJbRZ=pb~zT$YV03WJRb@2b3bwyq2z zn`g&vd8?FErXe3$(8cSw)9=cEwEB7+Yy$qHg^P(afZvyOiPC>}xBu7{th)2*(TllT zq{FQrEL2{=!!3ekxBaEs*|S7kQkvY35JOLQf5*aV9hRg!e(fr;*|^Vxk~gFSCdbr9 zD#jLv?)bq+`MgoN@Qg2Y6*+!HFt$hcwRSi}SJf|?a{{Hc*>mr8ulS%8_)sHLS$G~g z>r~F`Y^#yxp*8bix^uYT-3JqDSv&R&=i?=YCzJnT_~qT3*)~I!aM$=8fRGlm9NO>& zJ9O3xQvGl8ch=&bnVLO~`h_Nsd?;d&H8`^{#>EGd(`XCI>o z(x`z}C}>$qj|B+tG1#oIad@P9TkffquA4ypirR1bOZ$^EwNMqhX2mm**131YXHYC}O_(!D?GSO&(#bZ#`$ITx#ub0-IAd^NZ1qu*&NBq%Q{Q zMw>~oUnoyc*GvxlECajiGktguy0e{dA#arVU#xGp9Yi_PHJcn%$ZvD{QS?67n%ZS$ zrx>b6RYJd)3ma0xy8cj_oa3;vhfX*qcrIH@1z9cM+uj|iuI-$)OE_4LTW})V0bVkW zZK|h5@)V{9FKo)gI4xrwjA|?8d4gIjrSS3p0r+O?5BfX--L>ql7ckB!Wh1#C$I6mz znO$_#5GHyRqxKz_A%zFW2k*SRfnZwa=9b*ffB-fWL1uIxSB-29s;L;Zb35fJmJBpQ zJ~X@ZmgLB?z*PgI`e6By&!eisf{!2UQ^7q-jljG&6;4hS&u4_Gq7Z35k3QaqSI-dJ zU*ojn%s@DPpM8eD3(w!VMe@)ZPHv=ZZ<`Q{)fhx=#?KYBJQVc2h%D2rl-AuJuMG|F z-TlTTHtncZ(!>#u30bEcOth+%v@lC323D7!G}^H#b9+18mZ3k(*!Ti?$A3+HQBZ)>c#M}dNlOy?6m^=kkE>s*Hxvftb4q`YWdc?Ys-_ut z3wqC(KQTmfA5i2tk-3WFG8m`>yU+I)IB!$@9MrBH%{UHkL9|Qov-%>08+1_jiIu>5v5lKW*y-gr`FE2*M+x~sbXvb%!n7vU+L|upo z8qb){4kadXr_wL1y&<}7edSYY(S9+Qhg@0xV9!rp9klpE%Nb0cdKwrU+)`QcReO9A zx_EO>+X=MlJqJ8rfWPW2rFNXCMGB4H5N;%%%n@#ItH~Q|*PL;lbWCCzwC%Sdt5i2u z5RbcC37vB{u5@NmOG&M{?ST?5v``TL5b%0Xv#>H^1Rs_|coX8y;{a6qQtw zmP#ihk$3b;T{ODote57vfZp08Bt@QYlC7e(hJcG;_h!CLVBf~9 zkuXTUbqDSZsHrS>rG0udmx7s;?RZZ~BQ@4WJ21s2L08Q!ZHw0 zKNexpi(Ti7`^ChfFW~mG9+A_@DMi(J<36ZiQi{g8`1EL4InA@ z?yd#2l23-RkOSww4#cv5pcb<_LMLxXHs5&@u5Fp5*FdnYf%7`bCRt#R=BoLaovFUX zEJ+3ZV4SAsZ(0D-7tpcnZ+XwxnanSRRZy~PBwM?mSVXV%hT3rI`Z98VT%o^DD2DNFS92N1w63Sg%qo(sx6z=y zQk`Mv+}>V_^%(U0rI(3G&Fb0bJsU+rZ2HnuknFNm{A|FK=+0v9_RxNFXkX-lVs4RH zMWFJPcQfB*gY68r%6rCEh)ZhKBsMX+!qbKE^~S-vL*#pUT#M-pJ@M~BpSsUTGR7lW z*Oy*C5;db;yDAgfJVH({(W)&uo$Y0_&Jn~~2e{i(?xGYl)16&SKb#|?*>emx~Y zb$EpOJ+87z=9k&nYxC#7KRc~sp|NTTFkR_MHA#87R@t$E6`=m*D9+yDzF+@YbvHdC z3W2Jd-z^u+dt`j@*_?qr`60)qruD5|f1Rfo$`_{Xx-;H-oNr1 zg5sDrIw{kzI$f+cs&+-I)Scy}UJoiln}AmORc0YXM2)$`NRXk`u(K_SfQI@3|IYP5 zhak7At^KG_1S3g4cMYh9Bx3bD-F!TgLlI_Bk@jb2RMsof}>{jY1pmV`kgW`blwYdbWx zAyMHbP4I!dkDXiojv1}W;d|7ZC1T(7fXawD~?P~>m6@e-Igj>85${n()KF%Mo~ucujbX_<{spOb zO=~21&LAxiL{j!!juxo#lN82C_}yaZ%LXZ|Wy|9vO8fd> zWEs(;3giO$cHYt(gde;tXDeoe7;k*Pj@Es$5P3GXT7nM~%y<(s{xUHh-)jD4p$fje z^_PJ+@%APEW#1dTc>{l$*G~%^T>I0BR`}NbFCU8H+dKc^LL0#n1IRn|^8ebd`oDMM zt=6L93}4p*c*R7X^E2b0mz&_bD_jSsuCDzgWrqKA$2B~iSfz`Fe!jI4P~c)g^X5zy zpLc=V#W;fJ6~Ep!zGBN+imVzAzzmN8ry|cu%RYWcULd*e>6&BNmP>@{s6A(Pup+5$IX|1C z5Dotl-ByCq&VIp)C|o?Cz7LcCKKV9x*Y_0eqbw|naWn1cBN)D1}@_hphmX^mZZ zC$?j_`EtYGEXseg9Pu5pc?9(n*yDiT3a>jnr13-Nb9)j>Js?hN{jKah*MVsOEh|vU zlhEM7Nx^E=#(S=s&4(Ubo(W~pFw=^`i*K{q&uS#WsSI&}oThWJRCLW5Z21(GwDQk` zAcDX;g%i#9cw1m>%aIX>cOH{`dTBquh9nog-rR0N`&O~CD+q}vxogbBf`0vx&Z6esChEIg!Rx{S)d-wBIK^u5GKk2nHBqfeu#kiMCnirsT2yGE&b z4_inr5KN`>Qz|43QCbcS`*8PFMkmrdw2AipkoC{s`$)bXC?S`#1L;zig--EjKBvfQrr-9kx z>RlfqxOq4Uws#gJ<}K?=~uN)NhCFUmluI(7sLo~&{jbq)bwYU(P$%zCnf$BMHpY5hrZwkxqT5erLXHm>oW#)Z;a6)~2!9ANe$~!>>2Oad~nFJqX44&Z_ zLUB)t&VKTrpX21A^%mza$ff$g-aytFn)I@;l_2*sn+_@{MCJL7YSENm4>=)UsE>m$ zw^lDIDCTp(dluF1`gc3E{A_5!5A`8n!8PQM%{Gvum3DQC-$VOY zQ=?&>AA%;)d0ul^%NLYF8F@v(PUcH`92}i|)_XC8!_g(hTrLRP6F}x2x~?GbD#VLU z;L!)yu-u`7<$r0bZlb%tKb8L+axMBLEqJvNbBKP*A~ZiZstbb_?D@m*x6M?r3_DQ}51DEVZ{8HQBtT$2&Oj!p3*+#Kcr;MiWy#`w$*g0Jgn3fRQa zg_<{w(%U>UeSszdTz<8cWoKI`?(iIRWpZ(7L3Oh&)CSs9K51>x@f??Nc6*uG5QRyg<=;u$ zy8_mxt0e&~ZyMb}81yL$MPfVJ+e3fu8_6eHDlR_@D^CnJrodckTJ;9=9s6i3H#G?% zrbIhKpch<_aYzW{;$U~T4+Jh9fgVHF(r|P?O(Hy&z-oT^IkK;c-!3(sv`;ZLk3_#Q zIJ)>*i=DsLoFcmaV*d;ofJuz|PGT%*G|cb)ox4*H=A!VX`vHXd+Swd>zY|jFYnq@L zcFlVNDrnDfBrEKB`CgB;zgiJkM>Mm)9Z;_L;h4a%dT?dZKOy-K8ms3v5Po6>IsO$^ zehK!DdK~7in)PHX1`~a5iXt>9Gr!d4uY3rZAWhf~kSTy+S@~)p?6L)&G?=H%%@F<* zHatqeM=U+rV3Bg4ZY(d&+^mmU&i22+mq9s)$n)s<+4A+@D8k^w4G~cM;8(%Fshvbh zIOtERg46w@-a#rKQTIpuG0QeAjYcbkvP5Uh_uy0}ih!Qjt9cI4ZG62v!D=QmuM!cktLOzGY&ZmwRfhi+ z{`0yUk4hEZfKT5#s`o*KNATet4qqJff{5Zt?wVWp=$ISyrRb8f?uXR=Va-Y}Pq7;^1})C2QBP5`s+a{&uXU1|^KGgJ6=J?N~xsn8^K%#F*c8 zXtchbu6gB8apbtta+F7y-{&v?TVb)z_)GXd?RVNwl4LoT36ZdNI zubKaiZnq+`@tFBfs4SWD+UIXTT*}}r@^9Gt%~xpze?R}k(D+@q{Ed|VOD!@#sB?Td zka_D*9JUpKk9GfNZWIUqXM7IUEJ^uFt1rMv`2V{D{|_F8cC2TBQU)Fmd{Nn51L7SJ zjLAr20!oDtRx;dMcQiF6n?Ing(Vl*n=j_bw*)4^Tv9W`cmP8_`mjK%l3I9;oR0`2* zC&P_Ck`UhQK=|R$g0{b(=!zMGudVvW)fGOAa=zh7PL-NNy;%m{`2?A(t3{;vEw$kT5^&9}%9?_=6KksUGk@ z@xs3MTVghqYW*OS<qm2qCBFd zYb7u3wV{JfF1smC`_b$DQ=2vU;f>7q2V)BB-RwyEgcplYK+!GjV9V&!YW}UG$c-2A zJvm~zujHyiD^_0&UAm!V9jXtLNg<8zR_cBjJ4*%E?n)(L_BA(yi=1nfL_E*Mg`qtP z_Z}1N>7*yGRR}o`rCnGGb&UCs|Qh+1(QEjyTV&6`6Ps39lE1|4M*w{@N z$$*Lx#wA2oonY={W^qGFZiPn~XDr8Fcf@7lR_i09zS#QYEHh^w5kj+$`vu({}0T2_EkUp4X^E;zCGb>ZHz0Pc0&I|<*gtuJHnA`tjw z#v-kKKE+erZ4jeuHf;5^(&P|*h`Q^>cfsRfwXFMN@!5~Tkh)dTubq?^hcRY2^^k#B zmdr&8IV}kHEv4l*PDZ^(nsb(i0V%J($C{oyPu+BxBdAI04hPM2I3NZEqyr^ZlG0-v z25O6u(%QVtI)>Q1Lw7D&QO{PNkB@l`-v$Z2i%UE;PLnvpoy^M0WhLd|Roa-^X1jC$^8AIZIPl&tcCvY^N48M?tt|h*CL^q~-&vos`RHUWV!()bX_n#P z(=Heyr_zt+8Vvn`DlF93n_~pBQARvME>!BOtCv0ZgH~u$lW@iFTy6xy$&`G13ez4# zZ)Sh=^j3nCtqALorG#K4m?{I@q!#%=*GyT!M~Id=&6$>*=Zb0Cd%8J_dB z7T)^$S!r;R4xP9{p=l$HsByy|j~T%V-(^10X7Bln(hzH-1L{JTC%cSD5%;DY+B8kS zgeCnbE&68D3WYN^gVhs%926AW-|^;1Z1cQeb?>@KcPpk~=~+@kZ-9QcJ(q%Yu?*5h zb8v0LUI@Y=Q!jeJI~FuRSvp{3UJ6V?n(~qsL`-v#QDuki&L+02$f@%?znZzXbodtM zUc+$Te5~K&RE2Oji7@!byMqoysDnIMA}0Q;Q0~d`cA+z}I(5*E87d`HqiTvFz)l#GNJNK8ydObk>JM)KcHaPxTP?C|1$J;4-pzxB6u NYAV{wRZ7+`{~PfEd_e#J literal 0 HcmV?d00001 diff --git a/docs/images/consensus/threshold.png b/docs/images/consensus/threshold.png new file mode 100644 index 0000000000000000000000000000000000000000..87777e49b253de5d3fbe9bc5a73089c959d3c065 GIT binary patch literal 6074 zcmd^DS5#Bm*4`Wo2nr%ngiSyZEJ(Ewx&`S)kRlck0nsCp4G0M}M-8B&0gj3w?IvaFs}r<|ZlH4^pXHT8cKa`Z-7jU9kO;j$mjEa{`^^6 zTkGfN$K`Sn@x_Z5Wn^SNeE6WQuFhaEZr;3^m6he{>e|!OQ&LirS=wM?Vv<(-evgUy z(+=j<+|u|W(qBbyBVyA`ZI8$+ZVtSdK+GsQaV8jTZ02y>M^$sr?d&Qsaf#^LIcI|i zDfzYDkui5)G>EQ~z$N|#Dk&d@aNBn29B@2(@!k_V&$Ee@R7~uxmLX1fN^UaggTfY7 zSHF;ZZ@Sd=_O4woWoYe~*E$qm{MtM6TF(1#{x?#r+302>EIl8B~E4g zj`dQ~?|O&t7M$r0@$@HBY`;c}oPkRlTW zU8fxrP}~+GWR~A83~X!nFOg>j1LkkeNQok5;pO3T&!?PZVZ`33^NjecJm-`+aeuWk z;Gh+3<;f}q%xj0YgO!Z)0`XT)NzQ5{X+!zE2i;P@HhC%6Bfvu zb?03RdV`SQp5D=yH0tBq{!UK%e%OLf+r})$6gZX_M*g~wdVMP=&goobQJAcLKF9SY zfh{V`ojA$x{OF8lx(JBaIzb0%l%#d6!rya^@xGbQKbN&34j1f{bhcy1>pFLn_U(XQ zn9eJnH{e?tN>6PUUJ_xFZhO0)`1yJpVVRu0n5QL;x?P2Q@kUOJmNF7bxBxu3n&Dx6 zb@E6FS=CJZa;VJV`$o=M;-bDLhS#^*E9BZvN|Um8E&0=si!vVD{d?pFLHNb#`LJqL z{<4o}a=cmOOt_6l3p>u9TjV3|taA<9WvlDXBe9FD2)&JkDeODDxD&QMM6myD}F$ zPqL`Y@Qsv!E)rlNue=M*T~EL-(pBew$t=$}tSRg4n7kWUR@)lc^~?zI7{=P zNyL+*DdkUe0z%+mSi8prTJyt{WE*( z*$$sO!kc5Yv?z)HC{^B#Oey2iA6({bGrS%@4bj587hF*&33YnSt z)z4;7(!K_xCx1IMepSo~MoGJyGunk^Dm_Kq0e}B~Q!R6b(ALxEU@cJvQ55}(x#1fE zMVcL#P~&|CB#JDT4-wc_cO z1xKZ?CA#Cc=`92=5#c@O{7@le$}LJP(wW_b36BJmp|&iylssm%HFI&GhR;gS@w#R@ zpUw5F(1iymFS5s-(DScN(0UH@_>ak*zPu{Vzcy=ru5?eY7V-NcglIp96Tx!i_X!KQ zTiTd)?-%D=R?e1{{9;trSX#Z`{$Jd;uW5QX`iq6L$^a(%WzqKcIrFHs#EHa(aKUx; z84ms8#KuK|(dRh-pF(CnjD0F#@T1n#`UHCBp#|zUf(lU!xQHpvRt+P1HcO#9;QqF_ z{R@jpmHg^3Cb5H%skFeJJc>TCbuo|-)$HB^1qaYZ!V6kyB9TSs*TCNws^Z`A=(!18 zWZ~D8l8`c70B^k%eW%J2ryC^x_uP-c(6_EbP1>-grJ=rmpF{PEJAQu!5tj8GZnEvH zaKyY+_LId9v}7|}RY226K3kU3ZNGXIS}RHxzZx2RY7gY9|CYT}rTriH1k~>GU`4+HyC9_GDF%bb*aux0q=+stXR7`U_uGS4_--(+iztoUYeQG5$bB-%~K zOz(uz=z!sUII?o;pCGNkQcUBZJN_*vEs?(oE(d10_v9Ddq|@mgv3)kgJK2it5&1Nf zhFf0{E@0~O7fq8dJmz58kztfVE=tj9KEOKS&%aqqiN-BfnAj$bFPx8{ce=_g#SHFgV1nk{CtQ}9Gx#1QYC|8(Bu z*c!I@o=cFg?W|=nnfZwOi5~}T-nTHmL8@fl<|aZ9_RdNkkeLtCtXWvJJW+DtI&Ij^#Hgkmga;kAh&IFYN^f3L3> z*Wz#Q5}jf*@K*yh+^a*sCR%I+2}&zjkOa{d#`QD8q>Lvt5@Foe0^D@775fshqnx2p zki#d69>vuS=e6VgRlpi$jjo*32CM>hk>MtDH&QUEbLt$r zdjhj=mZhG=ue&A))3l1J!8?lVq?_!;bl~qY#FQSs3`%j@3f=wuPQfB1#JGHP z>-R8t0`AMjtythk1w27vVx`@dkAHh-)mVL}D&W>UDX(%zHDAY$u6tir5N_zk(U&u> z5{``Q=6rM~ER#NjgwnL0J8vNgQOexW`hQ%QB1$b7ZXJacimZma2Jd41V0pOQdG>M( zWv?ZUA_YLI4r32v*_HeWxnHM@c@ur1=)|6WYx$K3)1Mr~u)^>}>tHGYI*y&~gYv^d z)FSLKX%$uK(9ACvN}36pN1bMFfy;Sw5w?hn@_R4@`sCrcc~dlRs%tt0IvzIXpc!$1 zuf(qs>O+e72KZ^z>^pMvC0{h1 zDY9U3(OnM6>OGBx{z?cfN3PCg%=J+adKF3w;JRf^J67ONSI^E-S|?LePw0uLG=j z3DGbi!Nx*iGtKhoAqcoH=M}-qkq}kf-Kq&P)J}38o}K4*IB?u{=hGiu3l(Xrx#ioz zD=ikL21P(Ly#bMrljDV?bzkqxwvP878F-@MIM6}mSrVZ2jtAaM?k}l{7%VVx9VCA2 zc-l2Tc}msQJ*TBCZu*t;vzkMB7zJ)@o>#=Zqdi9B(#PJ;*=Ush$%^_tg4^cqt2`g0 z&9X)MF>A;ZUwHfflcQkS$2y8kByAmXI z$&}l~dTU@0RyB_HJ=p=Hj&xEW*93`IP+pai4K{IE^h`{g@Yw zZ{>%DQJ|v7+gl=q0eTC$kp9qh59E#ia_t$zrfaxyYbRT17+yJ zvoiP`I~i`xtq`6Ydg}J{JyL*AuW5B1|IIUnn)mG15lMA1ofN6COam7_Uf}aG4p(Yt z?BVO$#V`9#fy4GcRq_?cu-)EgA-l-wQ%BZc1!=Ok?jDXHa+KzVZY;@hFD|jqz#fM?+$`54zGj zr@p(_BNiG+Z>npDR)6C=I@$EGk1{at{8BkwwXBXIrnN|XJd27ptx;(9MQa??6|vl1 zaGGxmqtvc~4das%cv2JuTy?Dp*yX6Ga9!B=4KJO%YwWQ57j+ShuTVwGdi9Y4W3-;E zT9dvr{GEGDQr#f8x5@I_*}~8y8b9?>eT44#6jp_0WjICplRK8Cr$kt0Ug!0;$3wQQIwy2(?d9d(m(uG0-3 zd$t8rs{`U$!Bb)xCv;Jg-nc`XQ)o>EsUaWr|JGlh|GCj|7rs~#X8-axAC5O5*2)(S z9=YJ9Gr${p0NR>bI%=9b)wDGowRH3}(Rw>IRW&vBG&LoMTigD{;5^3b?CFqy VZ{Ssg@tVPBanRNzfB(r#{{%yMc#i-8 literal 0 HcmV?d00001 diff --git a/docs/main.qbk b/docs/main.qbk index 3def9ca34..896cb1abe 100644 --- a/docs/main.qbk +++ b/docs/main.qbk @@ -1,5 +1,5 @@ [/ - Copyright (c) Copyright (c) 2012, 2013 Ripple Labs Inc. + Copyright (c) Copyright (c) 2012-2017 Ripple Labs Inc. Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above @@ -16,7 +16,7 @@ [library rippled [quickbook 1.6] - [copyright 2012 - 2016 Ripple Labs Inc.] + [copyright 2012 - 2017 Ripple Labs Inc.] [purpose C++ Library] [license Distributed under the ISC License @@ -30,6 +30,8 @@ [template indexterm1[term1] ''''''[term1]''''''] [template indexterm2[term1 term2] ''''''[term1]''''''[term2]''''''] +[include consensus.qbk] + [section:ref Reference] [include temp/reference.qbk] [endsect] diff --git a/src/ripple/app/consensus/RCLConsensus.cpp b/src/ripple/app/consensus/RCLConsensus.cpp index 7d077d1ce..b7e448395 100644 --- a/src/ripple/app/consensus/RCLConsensus.cpp +++ b/src/ripple/app/consensus/RCLConsensus.cpp @@ -18,97 +18,67 @@ //============================================================================== #include -#include #include -#include -#include -#include #include -#include +#include +#include +#include #include -#include -#include #include #include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include namespace ripple { - RCLConsensus::RCLConsensus( - Application& app, - std::unique_ptr && feeVote, - LedgerMaster& ledgerMaster, - LocalTxs& localTxs, - InboundTransactions& inboundTransactions, - typename Base::clock_type const & clock, - beast::Journal journal) - : Base(clock, journal) - , app_ (app) - , feeVote_ (std::move(feeVote)) - , ledgerMaster_ (ledgerMaster) - , localTxs_(localTxs) - , inboundTransactions_{ inboundTransactions } - , j_ (journal) - , nodeID_{ calcNodeID(app.nodeIdentity().first) } + Application& app, + std::unique_ptr&& feeVote, + LedgerMaster& ledgerMaster, + LocalTxs& localTxs, + InboundTransactions& inboundTransactions, + typename Base::clock_type const& clock, + beast::Journal journal) + : Base(clock, journal) + , app_(app) + , feeVote_(std::move(feeVote)) + , ledgerMaster_(ledgerMaster) + , localTxs_(localTxs) + , inboundTransactions_{inboundTransactions} + , j_(journal) + , nodeID_{calcNodeID(app.nodeIdentity().first)} { - -} - -void -RCLConsensus::onStartRound(RCLCxLedger const & ledger) -{ - inboundTransactions_.newRound(ledger.seq()); -} - -// First bool is whether or not we can propose -// Second bool is whether or not we can validate -std::pair -RCLConsensus::getMode () -{ - bool propose = false; - bool validate = false; - - if (! app_.getOPs().isNeedNetworkLedger() && (valPublic_.size() != 0)) - { - // We have a key, and we have some idea what the ledger is - validate = true; - - // propose only if we're in sync with the network - propose = app_.getOPs().getOperatingMode() == NetworkOPs::omFULL; - } - return { propose, validate }; } boost::optional -RCLConsensus::acquireLedger(LedgerHash const & ledger) +RCLConsensus::acquireLedger(LedgerHash const& ledger) { - // we need to switch the ledger we're working from auto buildLCL = ledgerMaster_.getLedgerByHash(ledger); - if (! buildLCL) + if (!buildLCL) { if (acquiringLedger_ != ledger) { // need to start acquiring the correct consensus LCL - JLOG (j_.warn()) << - "Need consensus ledger " << ledger; + JLOG(j_.warn()) << "Need consensus ledger " << ledger; // Tell the ledger acquire system that we need the consensus ledger acquiringLedger_ = ledger; auto app = &app_; auto hash = acquiringLedger_; - app_.getJobQueue().addJob ( - jtADVANCE, "getConsensusLedger", - [app, hash] (Job&) { + app_.getJobQueue().addJob( + jtADVANCE, "getConsensusLedger", [app, hash](Job&) { app->getInboundLedgers().acquire( hash, 0, InboundLedger::fcCONSENSUS); }); @@ -116,131 +86,126 @@ RCLConsensus::acquireLedger(LedgerHash const & ledger) return boost::none; } - assert (!buildLCL->open() && buildLCL->isImmutable ()); - assert (buildLCL->info().hash == ledger); + assert(!buildLCL->open() && buildLCL->isImmutable()); + assert(buildLCL->info().hash == ledger); + + // Notify inbound transactions of the new ledger sequence number + inboundTransactions_.newRound(buildLCL->info().seq); return RCLCxLedger(buildLCL); } - std::vector -RCLConsensus::proposals (LedgerHash const& prevLedger) +RCLConsensus::proposals(LedgerHash const& prevLedger) { - std::vector ret; + std::vector ret; { - std::lock_guard _(peerPositionsLock_); + std::lock_guard _(peerPositionsLock_); for (auto const& it : peerPositions_) for (auto const& pos : it.second) if (pos->proposal().prevLedger() == prevLedger) - ret.emplace_back (*pos); + ret.emplace_back(*pos); } return ret; } void -RCLConsensus::storeProposal ( - RCLCxPeerPos::ref peerPos, - NodeID const& nodeID) +RCLConsensus::storeProposal(RCLCxPeerPos::ref peerPos, NodeID const& nodeID) { - std::lock_guard _(peerPositionsLock_); + std::lock_guard _(peerPositionsLock_); auto& props = peerPositions_[nodeID]; - if (props.size () >= 10) - props.pop_front (); + if (props.size() >= 10) + props.pop_front(); - props.push_back (peerPos); + props.push_back(peerPos); } void -RCLConsensus::relay(RCLCxPeerPos const & peerPos) +RCLConsensus::relay(RCLCxPeerPos const& peerPos) { protocol::TMProposeSet prop; - auto const & proposal = peerPos.proposal(); + auto const& proposal = peerPos.proposal(); - prop.set_proposeseq ( - proposal.proposeSeq ()); - prop.set_closetime ( - proposal.closeTime ().time_since_epoch().count()); + prop.set_proposeseq(proposal.proposeSeq()); + prop.set_closetime(proposal.closeTime().time_since_epoch().count()); - prop.set_currenttxhash ( + prop.set_currenttxhash( proposal.position().begin(), proposal.position().size()); - prop.set_previousledger ( + prop.set_previousledger( proposal.prevLedger().begin(), proposal.position().size()); auto const pk = peerPos.getPublicKey().slice(); - prop.set_nodepubkey (pk.data(), pk.size()); + prop.set_nodepubkey(pk.data(), pk.size()); auto const sig = peerPos.getSignature(); - prop.set_signature (sig.data(), sig.size()); + prop.set_signature(sig.data(), sig.size()); - app_.overlay().relay (prop, peerPos.getSuppressionID ()); + app_.overlay().relay(prop, peerPos.getSuppressionID()); } void -RCLConsensus::relay(DisputedTx const & dispute) +RCLConsensus::relay(RCLCxTx const& tx) { - // If we didn't relay this transaction recently, relay it to all peers - auto const & tx = dispute.tx(); - if (app_.getHashRouter ().shouldRelay (tx.id())) + // If we didn't relay this transaction recently, relay it to all peers + if (app_.getHashRouter().shouldRelay(tx.id())) { auto const slice = tx.tx_.slice(); protocol::TMTransaction msg; - msg.set_rawtransaction (slice.data(), slice.size()); - msg.set_status (protocol::tsNEW); - msg.set_receivetimestamp ( + msg.set_rawtransaction(slice.data(), slice.size()); + msg.set_status(protocol::tsNEW); + msg.set_receivetimestamp( app_.timeKeeper().now().time_since_epoch().count()); - app_.overlay ().foreach (send_always ( - std::make_shared ( - msg, protocol::mtTRANSACTION))); + app_.overlay().foreach (send_always( + std::make_shared(msg, protocol::mtTRANSACTION))); } } void -RCLConsensus::propose (RCLCxPeerPos::Proposal const& proposal) +RCLConsensus::propose(RCLCxPeerPos::Proposal const& proposal) { - JLOG (j_.trace()) << "We propose: " << - (proposal.isBowOut () ? std::string ("bowOut") : - to_string (proposal.position ())); + JLOG(j_.trace()) << "We propose: " + << (proposal.isBowOut() + ? std::string("bowOut") + : ripple::to_string(proposal.position())); protocol::TMProposeSet prop; - prop.set_currenttxhash (proposal.position().begin(), - proposal.position().size()); - prop.set_previousledger (proposal.prevLedger().begin(), - proposal.position().size()); - prop.set_proposeseq (proposal.proposeSeq()); - prop.set_closetime ( - proposal.closeTime().time_since_epoch().count()); + prop.set_currenttxhash( + proposal.position().begin(), proposal.position().size()); + prop.set_previousledger( + proposal.prevLedger().begin(), proposal.position().size()); + prop.set_proposeseq(proposal.proposeSeq()); + prop.set_closetime(proposal.closeTime().time_since_epoch().count()); - prop.set_nodepubkey (valPublic_.data(), valPublic_.size()); + prop.set_nodepubkey(valPublic_.data(), valPublic_.size()); auto signingHash = sha512Half( HashPrefix::proposal, std::uint32_t(proposal.proposeSeq()), proposal.closeTime().time_since_epoch().count(), - proposal.prevLedger(), proposal.position()); + proposal.prevLedger(), + proposal.position()); - auto sig = signDigest ( - valPublic_, valSecret_, signingHash); + auto sig = signDigest(valPublic_, valSecret_, signingHash); - prop.set_signature (sig.data(), sig.size()); + prop.set_signature(sig.data(), sig.size()); app_.overlay().send(prop); } void -RCLConsensus::share (RCLTxSet const& set) +RCLConsensus::relay(RCLTxSet const& set) { - inboundTransactions_.giveSet (set.id(), - set.map_, false); + inboundTransactions_.giveSet(set.id(), set.map_, false); } boost::optional -RCLConsensus::acquireTxSet(RCLTxSet::ID const & setId) +RCLConsensus::acquireTxSet(RCLTxSet::ID const& setId) { if (auto set = inboundTransactions_.getSet(setId, true)) { @@ -249,127 +214,122 @@ RCLConsensus::acquireTxSet(RCLTxSet::ID const & setId) return boost::none; } - bool RCLConsensus::hasOpenTransactions() const { - return ! app_.openLedger().empty(); + return !app_.openLedger().empty(); } std::size_t -RCLConsensus::proposersValidated(LedgerHash const & h) const +RCLConsensus::proposersValidated(LedgerHash const& h) const { return app_.getValidations().getTrustedValidationCount(h); } std::size_t -RCLConsensus::proposersFinished(LedgerHash const & h) const +RCLConsensus::proposersFinished(LedgerHash const& h) const { return app_.getValidations().getNodesAfter(h); } uint256 -RCLConsensus::getLCL ( - uint256 const& currentLedger, - uint256 const& priorLedger, - bool believedCorrect) +RCLConsensus::getPrevLedger( + uint256 ledgerID, + RCLCxLedger const& ledger, + Mode mode) { + uint256 parentID; + // Only set the parent ID if we believe ledger is the right ledger + if (mode != Mode::wrongLedger) + parentID = ledger.parentID(); + // Get validators that are on our ledger, or "close" to being on // our ledger. - auto vals = - app_.getValidations().getCurrentValidations( - currentLedger, priorLedger, - ledgerMaster_.getValidLedgerIndex()); + auto vals = app_.getValidations().getCurrentValidations( + ledgerID, parentID, ledgerMaster_.getValidLedgerIndex()); - uint256 netLgr = currentLedger; + uint256 netLgr = ledgerID; int netLgrCount = 0; for (auto& it : vals) { // Switch to ledger supported by more peers // Or stick with ours on a tie if ((it.second.first > netLgrCount) || - ((it.second.first == netLgrCount) && (it.first == currentLedger))) + ((it.second.first == netLgrCount) && (it.first == ledgerID))) { - netLgr = it.first; - netLgrCount = it.second.first; + netLgr = it.first; + netLgrCount = it.second.first; } } - if(netLgr != currentLedger) + if (netLgr != ledgerID) { - if (believedCorrect) - app_.getOPs().consensusViewChange(); - if (auto stream = j_.debug()) - { - for (auto& it : vals) - stream << "V: " << it.first << ", " << it.second.first; - stream << getJson (true); - } + if (mode != Mode::wrongLedger) + app_.getOPs().consensusViewChange(); + + if (auto stream = j_.debug()) + { + for (auto& it : vals) + stream << "V: " << it.first << ", " << it.second.first; + stream << getJson(true); + } } return netLgr; } - -void -RCLConsensus::onClose(RCLCxLedger const & ledger, bool haveCorrectLCL) +RCLConsensus::Result +RCLConsensus::onClose( + RCLCxLedger const& ledger, + NetClock::time_point const& closeTime, + Mode mode) { - notify(protocol::neCLOSING_LEDGER, ledger, haveCorrectLCL); -} + const bool wrongLCL = mode == Mode::wrongLedger; + const bool proposing = mode == Mode::proposing; -std::pair -RCLConsensus::makeInitialPosition (RCLCxLedger const & prevLedgerT, - bool proposing, - bool correctLCL, - NetClock::time_point closeTime, - NetClock::time_point now) -{ - auto const &prevLedger = prevLedgerT.ledger_; - ledgerMaster_.applyHeldTransactions (); + notify(protocol::neCLOSING_LEDGER, ledger, !wrongLCL); + + auto const& prevLedger = ledger.ledger_; + + ledgerMaster_.applyHeldTransactions(); // Tell the ledger master not to acquire the ledger we're probably building - ledgerMaster_.setBuildingLedger (prevLedger->info().seq + 1); + ledgerMaster_.setBuildingLedger(prevLedger->info().seq + 1); auto initialLedger = app_.openLedger().current(); - auto initialSet = std::make_shared ( + auto initialSet = std::make_shared( SHAMapType::TRANSACTION, app_.family(), SHAMap::version{1}); - initialSet->setUnbacked (); + initialSet->setUnbacked(); // Build SHAMap containing all transactions in our open ledger for (auto const& tx : initialLedger->txs) { - Serializer s (2048); + Serializer s(2048); tx.first->add(s); - initialSet->addItem ( - SHAMapItem (tx.first->getTransactionID(), std::move (s)), true, false); + initialSet->addItem( + SHAMapItem(tx.first->getTransactionID(), std::move(s)), + true, + false); } // Add pseudo-transactions to the set - if ((app_.config().standalone() || (proposing && correctLCL)) - && ((prevLedger->info().seq % 256) == 0)) + if ((app_.config().standalone() || (proposing && !wrongLCL)) && + ((prevLedger->info().seq % 256) == 0)) { // previous ledger was flag ledger, add pseudo-transactions auto const validations = - app_.getValidations().getValidations ( - prevLedger->info().parentHash); + app_.getValidations().getValidations(prevLedger->info().parentHash); - std::size_t const count = std::count_if ( - validations.begin(), validations.end(), - [](auto const& v) - { + std::size_t const count = std::count_if( + validations.begin(), validations.end(), [](auto const& v) { return v.second->isTrusted(); }); - if (count >= app_.validators ().quorum ()) + if (count >= app_.validators().quorum()) { - feeVote_->doVoting ( - prevLedger, - validations, - initialSet); - app_.getAmendmentTable ().doVoting ( - prevLedger, - validations, - initialSet); + feeVote_->doVoting(prevLedger, validations, initialSet); + app_.getAmendmentTable().doVoting( + prevLedger, validations, initialSet); } } @@ -377,104 +337,120 @@ RCLConsensus::makeInitialPosition (RCLCxLedger const & prevLedgerT, initialSet = initialSet->snapShot(false); auto setHash = initialSet->getHash().as_uint256(); - return std::make_pair ( - std::move (initialSet), - RCLCxPeerPos::Proposal { + return Result{ + std::move(initialSet), + RCLCxPeerPos::Proposal{ initialLedger->info().parentHash, RCLCxPeerPos::Proposal::seqJoin, setHash, closeTime, - now, - nodeID_ }); + app_.timeKeeper().closeTime(), + nodeID_}}; } void -RCLConsensus::dispatchAccept(RCLTxSet const & txSet) +RCLConsensus::onForceAccept( + Result const& result, + RCLCxLedger const& prevLedger, + NetClock::duration const& closeResolution, + CloseTimes const& rawCloseTimes, + Mode const& mode) { - app_.getJobQueue().addJob(jtACCEPT, "acceptLedger", - [that = this->shared_from_this(), - consensusSet = txSet] - (auto &) - { - that->accept(consensusSet); + doAccept(result, prevLedger, closeResolution, rawCloseTimes, mode); +} + +void +RCLConsensus::onAccept( + Result const& result, + RCLCxLedger const& prevLedger, + NetClock::duration const& closeResolution, + CloseTimes const& rawCloseTimes, + Mode const& mode) +{ + app_.getJobQueue().addJob( + jtACCEPT, "acceptLedger", [&, that = this->shared_from_this() ](auto&) { + // note that no lock is held inside this thread, which + // is fine since once a ledger is accepted, consensus + // will not touch any internal state until startRound is called + that->doAccept( + result, prevLedger, closeResolution, rawCloseTimes, mode); + that->app_.getOPs().endConsensus(); }); } -bool -RCLConsensus::accept( - RCLTxSet const& set, - NetClock::time_point consensusCloseTime, - bool proposing_, - bool validating_, - bool haveCorrectLCL_, - bool consensusFail_, - LedgerHash const &prevLedgerHash_, - RCLCxLedger const & previousLedger_, - NetClock::duration closeResolution_, - NetClock::time_point const & now_, - std::chrono::milliseconds const & roundTime_, - hash_map> const & disputes_, - std::map closeTimes_, - NetClock::time_point const & closeTime_ - ) +void +RCLConsensus::doAccept( + Result const& result, + RCLCxLedger const& prevLedger, + NetClock::duration closeResolution, + CloseTimes const& rawCloseTimes, + Mode const& mode) { bool closeTimeCorrect; + const bool proposing = mode == Mode::proposing; + const bool haveCorrectLCL = mode != Mode::wrongLedger; + const bool consensusFail = result.state == ConsensusState::MovedOn; + + auto consensusCloseTime = result.position.closeTime(); + if (consensusCloseTime == NetClock::time_point{}) { // We agreed to disagree on the close time - consensusCloseTime = previousLedger_.closeTime() + 1s; + consensusCloseTime = prevLedger.closeTime() + 1s; closeTimeCorrect = false; } else { // We agreed on a close time - consensusCloseTime = effectiveCloseTime(consensusCloseTime, - closeResolution_, previousLedger_.closeTime()); + consensusCloseTime = effCloseTime( + consensusCloseTime, closeResolution, prevLedger.closeTime()); closeTimeCorrect = true; } - JLOG (j_.debug()) - << "Report: Prop=" << (proposing_ ? "yes" : "no") - << " val=" << (validating_ ? "yes" : "no") - << " corLCL=" << (haveCorrectLCL_ ? "yes" : "no") - << " fail=" << (consensusFail_ ? "yes" : "no"); - JLOG (j_.debug()) - << "Report: Prev = " << prevLedgerHash_ - << ":" << previousLedger_.seq(); + JLOG(j_.debug()) << "Report: Prop=" << (proposing ? "yes" : "no") + << " val=" << (validating_ ? "yes" : "no") + << " corLCL=" << (haveCorrectLCL ? "yes" : "no") + << " fail=" << (consensusFail ? "yes" : "no"); + JLOG(j_.debug()) << "Report: Prev = " << prevLedger.id() << ":" + << prevLedger.seq(); //-------------------------------------------------------------------------- // Put transactions into a deterministic, but unpredictable, order - CanonicalTXSet retriableTxs{ set.id() }; - - auto sharedLCL = buildLCL(previousLedger_, set, consensusCloseTime, - closeTimeCorrect, closeResolution_, now_, roundTime_, retriableTxs); + CanonicalTXSet retriableTxs{result.set.id()}; + auto sharedLCL = buildLCL( + prevLedger, + result.set, + consensusCloseTime, + closeTimeCorrect, + closeResolution, + result.roundTime.read(), + retriableTxs); auto const newLCLHash = sharedLCL.id(); - JLOG (j_.debug()) - << "Report: NewL = " << newLCLHash - << ":" << sharedLCL.seq(); + JLOG(j_.debug()) << "Report: NewL = " << newLCLHash << ":" + << sharedLCL.seq(); // Tell directly connected peers that we have a new LCL - notify (protocol::neACCEPTED_LEDGER, sharedLCL, haveCorrectLCL_); + notify(protocol::neACCEPTED_LEDGER, sharedLCL, haveCorrectLCL); if (validating_) - validating_ = ledgerMaster_.isCompatible(*sharedLCL.ledger_, - app_.journal("LedgerConsensus").warn(), "Not validating"); + validating_ = ledgerMaster_.isCompatible( + *sharedLCL.ledger_, + app_.journal("LedgerConsensus").warn(), + "Not validating"); - if (validating_ && ! consensusFail_) + if (validating_ && !consensusFail) { - validate(sharedLCL, now_, proposing_); - JLOG (j_.info()) - << "CNF Val " << newLCLHash; + validate(sharedLCL, proposing); + JLOG(j_.info()) << "CNF Val " << newLCLHash; } else - JLOG (j_.info()) - << "CNF buildLCL " << newLCLHash; + JLOG(j_.info()) << "CNF buildLCL " << newLCLHash; // See if we can accept a ledger as fully-validated - ledgerMaster_.consensusBuilt (sharedLCL.ledger_, getJson(true)); + ledgerMaster_.consensusBuilt(sharedLCL.ledger_, getJson(true)); //------------------------------------------------------------------------- { @@ -490,36 +466,34 @@ RCLConsensus::accept( // in the previous consensus round. // bool anyDisputes = false; - for (auto& it : disputes_) + for (auto& it : result.disputes) { - if (!it.second.getOurVote ()) + if (!it.second.getOurVote()) { // we voted NO try { - JLOG (j_.debug()) + JLOG(j_.debug()) << "Test applying disputed transaction that did" << " not get in"; - SerialIter sit (it.second.tx().tx_.slice()); + SerialIter sit(it.second.tx().tx_.slice()); auto txn = std::make_shared(sit); - retriableTxs.insert (txn); + retriableTxs.insert(txn); anyDisputes = true; } catch (std::exception const&) { - JLOG (j_.debug()) + JLOG(j_.debug()) << "Failed to apply transaction we voted NO on"; } } } // Build new open ledger - auto lock = make_lock( - app_.getMasterMutex(), std::defer_lock); - auto sl = make_lock( - ledgerMaster_.peekMutex (), std::defer_lock); + auto lock = make_lock(app_.getMasterMutex(), std::defer_lock); + auto sl = make_lock(ledgerMaster_.peekMutex(), std::defer_lock); std::lock(lock, sl); auto const lastVal = ledgerMaster_.getValidatedLedger(); @@ -528,50 +502,60 @@ RCLConsensus::accept( rules.emplace(*lastVal, app_.config().features); else rules.emplace(app_.config().features); - app_.openLedger().accept(app_, *rules, - sharedLCL.ledger_, localTxs_.getTxSet(), anyDisputes, retriableTxs, tapNONE, - "consensus", - [&](OpenView& view, beast::Journal j) - { - // Stuff the ledger with transactions from the queue. - return app_.getTxQ().accept(app_, view); - }); + app_.openLedger().accept( + app_, + *rules, + sharedLCL.ledger_, + localTxs_.getTxSet(), + anyDisputes, + retriableTxs, + tapNONE, + "consensus", + [&](OpenView& view, beast::Journal j) { + // Stuff the ledger with transactions from the queue. + return app_.getTxQ().accept(app_, view); + }); - // Signal a potential fee change to subscribers after the open ledger - // is created - app_.getOPs().reportFeeChange(); + // Signal a potential fee change to subscribers after the open ledger + // is created + app_.getOPs().reportFeeChange(); } //------------------------------------------------------------------------- { - ledgerMaster_.switchLCL (sharedLCL.ledger_); + ledgerMaster_.switchLCL(sharedLCL.ledger_); // Do these need to exist? - assert (ledgerMaster_.getClosedLedger()->info().hash == sharedLCL.id()); - assert (app_.openLedger().current()->info().parentHash == sharedLCL.id()); + assert(ledgerMaster_.getClosedLedger()->info().hash == sharedLCL.id()); + assert( + app_.openLedger().current()->info().parentHash == sharedLCL.id()); } //------------------------------------------------------------------------- - if (haveCorrectLCL_ && ! consensusFail_) + // we entered the round with the network, + // see how close our close time is to other node's + // close time reports, and update our clock. + if ((mode == Mode::proposing || mode == Mode::observing) && !consensusFail) { - // we entered the round with the network, - // see how close our close time is to other node's - // close time reports, and update our clock. - JLOG (j_.info()) - << "We closed at " << closeTime_.time_since_epoch().count(); + auto closeTime = rawCloseTimes.self; + + JLOG(j_.info()) << "We closed at " + << closeTime.time_since_epoch().count(); using usec64_t = std::chrono::duration; - usec64_t closeTotal = std::chrono::duration_cast(closeTime_.time_since_epoch()); + usec64_t closeTotal = + std::chrono::duration_cast(closeTime.time_since_epoch()); int closeCount = 1; - for (auto const& p : closeTimes_) + for (auto const& p : rawCloseTimes.peers) { // FIXME: Use median, not average - JLOG (j_.info()) - << std::to_string(p.second) - << " time votes for " + JLOG(j_.info()) + << std::to_string(p.second) << " time votes for " << std::to_string(p.first.time_since_epoch().count()); closeCount += p.second; - closeTotal += std::chrono::duration_cast(p.first.time_since_epoch()) * p.second; + closeTotal += std::chrono::duration_cast( + p.first.time_since_epoch()) * + p.second; } closeTotal += usec64_t(closeCount / 2); // for round to nearest @@ -581,47 +565,37 @@ RCLConsensus::accept( using duration = std::chrono::duration; using time_point = std::chrono::time_point; auto offset = time_point{closeTotal} - - std::chrono::time_point_cast(closeTime_); - JLOG (j_.info()) - << "Our close offset is estimated at " - << offset.count() << " (" << closeCount << ")"; + std::chrono::time_point_cast(closeTime); + JLOG(j_.info()) << "Our close offset is estimated at " << offset.count() + << " (" << closeCount << ")"; app_.timeKeeper().adjustCloseTime(offset); } - - return validating_; } -void -RCLConsensus::endConsensus(bool correctLCL) -{ - app_.getOPs ().endConsensus (correctLCL); -} - - void RCLConsensus::notify( protocol::NodeEvent ne, - RCLCxLedger const & ledger, + RCLCxLedger const& ledger, bool haveCorrectLCL) { - protocol::TMStatusChange s; if (!haveCorrectLCL) - s.set_newevent (protocol::neLOST_SYNC); + s.set_newevent(protocol::neLOST_SYNC); else s.set_newevent(ne); - s.set_ledgerseq (ledger.seq()); - s.set_networktime (app_.timeKeeper().now().time_since_epoch().count()); - s.set_ledgerhashprevious(ledger.parentID().begin (), + s.set_ledgerseq(ledger.seq()); + s.set_networktime(app_.timeKeeper().now().time_since_epoch().count()); + s.set_ledgerhashprevious( + ledger.parentID().begin(), std::decay_t::bytes); - s.set_ledgerhash (ledger.id().begin (), - std::decay_t::bytes); + s.set_ledgerhash( + ledger.id().begin(), std::decay_t::bytes); std::uint32_t uMin, uMax; - if (! ledgerMaster_.getFullValidatedRange (uMin, uMax)) + if (!ledgerMaster_.getFullValidatedRange(uMin, uMax)) { uMin = 0; uMax = 0; @@ -629,17 +603,15 @@ RCLConsensus::notify( else { // Don't advertise ledgers we're not willing to serve - uMin = std::max(uMin, ledgerMaster_.getEarliestFetch ()); + uMin = std::max(uMin, ledgerMaster_.getEarliestFetch()); } - s.set_firstseq (uMin); - s.set_lastseq (uMax); - app_.overlay ().foreach (send_always ( - std::make_shared ( - s, protocol::mtSTATUS_CHANGE))); - JLOG (j_.trace()) << "send status change to peer"; + s.set_firstseq(uMin); + s.set_lastseq(uMax); + app_.overlay().foreach ( + send_always(std::make_shared(s, protocol::mtSTATUS_CHANGE))); + JLOG(j_.trace()) << "send status change to peer"; } - /** Apply a set of transactions to a ledger. Typically the txFilter is used to reject transactions @@ -652,35 +624,33 @@ RCLConsensus::notify( */ CanonicalTXSet -applyTransactions ( +applyTransactions( Application& app, RCLTxSet const& cSet, OpenView& view, std::function txFilter) { - auto j = app.journal ("LedgerConsensus"); + auto j = app.journal("LedgerConsensus"); auto& set = *(cSet.map_); - CanonicalTXSet retriableTxs (set.getHash().as_uint256()); - + CanonicalTXSet retriableTxs(set.getHash().as_uint256()); for (auto const& item : set) { - if (! txFilter (item.key())) + if (!txFilter(item.key())) continue; // The transaction wan't filtered // Add it to the set to be tried in canonical order - JLOG (j.debug()) << - "Processing candidate transaction: " << item.key(); + JLOG(j.debug()) << "Processing candidate transaction: " << item.key(); try { - retriableTxs.insert ( + retriableTxs.insert( std::make_shared(SerialIter{item.slice()})); } catch (std::exception const&) { - JLOG (j.warn()) << "Txn " << item.key() << " throws"; + JLOG(j.warn()) << "Txn " << item.key() << " throws"; } } @@ -688,43 +658,41 @@ applyTransactions ( // Attempt to apply all of the retriable transactions for (int pass = 0; pass < LEDGER_TOTAL_PASSES; ++pass) { - JLOG (j.debug()) << "Pass: " << pass << " Txns: " - << retriableTxs.size () - << (certainRetry ? " retriable" : " final"); + JLOG(j.debug()) << "Pass: " << pass << " Txns: " << retriableTxs.size() + << (certainRetry ? " retriable" : " final"); int changes = 0; - auto it = retriableTxs.begin (); + auto it = retriableTxs.begin(); - while (it != retriableTxs.end ()) + while (it != retriableTxs.end()) { try { - switch (applyTransaction (app, view, - *it->second, certainRetry, tapNO_CHECK_SIGN, j)) + switch (applyTransaction( + app, view, *it->second, certainRetry, tapNO_CHECK_SIGN, j)) { - case ApplyResult::Success: - it = retriableTxs.erase (it); - ++changes; - break; + case ApplyResult::Success: + it = retriableTxs.erase(it); + ++changes; + break; - case ApplyResult::Fail: - it = retriableTxs.erase (it); - break; + case ApplyResult::Fail: + it = retriableTxs.erase(it); + break; - case ApplyResult::Retry: - ++it; + case ApplyResult::Retry: + ++it; } } catch (std::exception const&) { - JLOG (j.warn()) - << "Transaction throws"; - it = retriableTxs.erase (it); + JLOG(j.warn()) << "Transaction throws"; + it = retriableTxs.erase(it); } } - JLOG (j.debug()) << "Pass: " - << pass << " finished " << changes << " changes"; + JLOG(j.debug()) << "Pass: " << pass << " finished " << changes + << " changes"; // A non-retry pass made no changes if (!changes && !certainRetry) @@ -737,20 +705,19 @@ applyTransactions ( // If there are any transactions left, we must have // tried them in at least one final pass - assert (retriableTxs.empty() || !certainRetry); + assert(retriableTxs.empty() || !certainRetry); return retriableTxs; } RCLCxLedger RCLConsensus::buildLCL( - RCLCxLedger const & previousLedger, - RCLTxSet const & set, + RCLCxLedger const& previousLedger, + RCLTxSet const& set, NetClock::time_point closeTime, bool closeTimeCorrect, NetClock::duration closeResolution, - NetClock::time_point now, std::chrono::milliseconds roundTime, - CanonicalTXSet & retriableTxs) + CanonicalTXSet& retriableTxs) { auto replay = ledgerMaster_.releaseReplay(); if (replay) @@ -760,14 +727,13 @@ RCLConsensus::buildLCL( closeTimeCorrect = ((replay->closeFlags_ & sLCF_NoConsensusTime) == 0); } - JLOG (j_.debug()) - << "Report: TxSt = " << set.id () - << ", close " << closeTime.time_since_epoch().count() - << (closeTimeCorrect ? "" : "X"); - + JLOG(j_.debug()) << "Report: TxSt = " << set.id() << ", close " + << closeTime.time_since_epoch().count() + << (closeTimeCorrect ? "" : "X"); // Build the new last closed ledger - auto buildLCL = std::make_shared(*previousLedger.ledger_, now); + auto buildLCL = + std::make_shared(*previousLedger.ledger_, closeTime); auto const v2_enabled = buildLCL->rules().enabled(featureSHAMapV2); @@ -780,9 +746,8 @@ RCLConsensus::buildLCL( // Set up to write SHAMap changes to our database, // perform updates, extract changes - JLOG (j_.debug()) - << "Applying consensus set transactions to the" - << " last closed ledger"; + JLOG(j_.debug()) << "Applying consensus set transactions to the" + << " last closed ledger"; { OpenView accum(&*buildLCL); @@ -791,21 +756,19 @@ RCLConsensus::buildLCL( { // Special case, we are replaying a ledger close for (auto& tx : replay->txns_) - applyTransaction (app_, accum, *tx.second, - false, tapNO_CHECK_SIGN, j_); + applyTransaction( + app_, accum, *tx.second, false, tapNO_CHECK_SIGN, j_); } else { // Normal case, we are not replaying a ledger close - retriableTxs = applyTransactions (app_, set, accum, - [&buildLCL](uint256 const& txID) - { - return ! buildLCL->txExists(txID); + retriableTxs = applyTransactions( + app_, set, accum, [&buildLCL](uint256 const& txID) { + return !buildLCL->txExists(txID); }); } // Update fee computations. - app_.getTxQ().processClosedLedger(app_, accum, - roundTime > 5s); + app_.getTxQ().processClosedLedger(app_, accum, roundTime > 5s); accum.apply(*buildLCL); } @@ -813,61 +776,52 @@ RCLConsensus::buildLCL( // made it into the consensus set but failed during application // to the ledger. - buildLCL->updateSkipList (); + buildLCL->updateSkipList(); { // Write the final version of all modified SHAMap // nodes to the node store to preserve the new LCL - int asf = buildLCL->stateMap().flushDirty ( + int asf = buildLCL->stateMap().flushDirty( hotACCOUNT_NODE, buildLCL->info().seq); - int tmf = buildLCL->txMap().flushDirty ( + int tmf = buildLCL->txMap().flushDirty( hotTRANSACTION_NODE, buildLCL->info().seq); - JLOG (j_.debug()) << "Flushed " << - asf << " accounts and " << - tmf << " transaction nodes"; + JLOG(j_.debug()) << "Flushed " << asf << " accounts and " << tmf + << " transaction nodes"; } buildLCL->unshare(); // Accept ledger - buildLCL->setAccepted(closeTime, closeResolution, - closeTimeCorrect, app_.config()); + buildLCL->setAccepted( + closeTime, closeResolution, closeTimeCorrect, app_.config()); // And stash the ledger in the ledger master - if (ledgerMaster_.storeLedger (buildLCL)) - JLOG (j_.debug()) - << "Consensus built ledger we already had"; - else if (app_.getInboundLedgers().find (buildLCL->info().hash)) - JLOG (j_.debug()) - << "Consensus built ledger we were acquiring"; + if (ledgerMaster_.storeLedger(buildLCL)) + JLOG(j_.debug()) << "Consensus built ledger we already had"; + else if (app_.getInboundLedgers().find(buildLCL->info().hash)) + JLOG(j_.debug()) << "Consensus built ledger we were acquiring"; else - JLOG (j_.debug()) - << "Consensus built new ledger"; + JLOG(j_.debug()) << "Consensus built new ledger"; return RCLCxLedger{std::move(buildLCL)}; - } void -RCLConsensus::validate( - RCLCxLedger const & ledger, - NetClock::time_point now, - bool proposing) +RCLConsensus::validate(RCLCxLedger const& ledger, bool proposing) { - auto validationTime = now; + auto validationTime = app_.timeKeeper().closeTime(); if (validationTime <= lastValidationTime_) validationTime = lastValidationTime_ + 1s; lastValidationTime_ = validationTime; // Build validation - auto v = std::make_shared (ledger.id(), - validationTime, valPublic_, proposing); - v->setFieldU32 (sfLedgerSequence, ledger.seq()); + auto v = std::make_shared( + ledger.id(), validationTime, valPublic_, proposing); + v->setFieldU32(sfLedgerSequence, ledger.seq()); // Add our load fee to the validation auto const& feeTrack = app_.getFeeTrack(); - std::uint32_t fee = std::max( - feeTrack.getLocalFee(), - feeTrack.getClusterFee()); + std::uint32_t fee = + std::max(feeTrack.getLocalFee(), feeTrack.getClusterFee()); if (fee > feeTrack.getLoadBase()) v->setFieldU32(sfLoadFee, fee); @@ -876,34 +830,104 @@ RCLConsensus::validate( // next ledger is flag ledger { // Suggest fee changes and new features - feeVote_->doValidation (ledger.ledger_, *v); - app_.getAmendmentTable ().doValidation (ledger.ledger_, *v); + feeVote_->doValidation(ledger.ledger_, *v); + app_.getAmendmentTable().doValidation(ledger.ledger_, *v); } - auto const signingHash = v->sign (valSecret_); - v->setTrusted (); + auto const signingHash = v->sign(valSecret_); + v->setTrusted(); // suppress it if we receive it - FIXME: wrong suppression - app_.getHashRouter ().addSuppression (signingHash); - app_.getValidations ().addValidation (v, "local"); - Blob validation = v->getSerialized (); + app_.getHashRouter().addSuppression(signingHash); + app_.getValidations().addValidation(v, "local"); + Blob validation = v->getSerialized(); protocol::TMValidation val; - val.set_validation (&validation[0], validation.size ()); + val.set_validation(&validation[0], validation.size()); // Send signed validation to all of our directly connected peers app_.overlay().send(val); } +Json::Value +RCLConsensus::getJson(bool full) const +{ + auto ret = Base::getJson(full); + ret["validating"] = validating_; + return ret; +} + PublicKey const& -RCLConsensus::getValidationPublicKey () const +RCLConsensus::getValidationPublicKey() const { return valPublic_; } void -RCLConsensus::setValidationKeys (SecretKey const& valSecret, +RCLConsensus::setValidationKeys( + SecretKey const& valSecret, PublicKey const& valPublic) { valSecret_ = valSecret; valPublic_ = valPublic; } +void +RCLConsensus::timerEntry(NetClock::time_point const& now) +{ + try + { + Base::timerEntry(now); + } + catch (SHAMapMissingNode const& mn) + { + // This should never happen + leaveConsensus(); + JLOG(j_.error()) << "Missing node during consensus process " << mn; + Rethrow(); + } +} + +void +RCLConsensus::gotTxSet(NetClock::time_point const& now, RCLTxSet const& txSet) +{ + try + { + Base::gotTxSet(now, txSet); + } + catch (SHAMapMissingNode const& mn) + { + // This should never happen + leaveConsensus(); + JLOG(j_.error()) << "Missing node during consensus process " << mn; + Rethrow(); + } +} + +void +RCLConsensus::startRound( + NetClock::time_point const& now, + RCLCxLedger::ID const& prevLgrId, + RCLCxLedger const& prevLgr) +{ + // We have a key, and we have some idea what the ledger is + validating_ = + !app_.getOPs().isNeedNetworkLedger() && (valPublic_.size() != 0); + + // propose only if we're in sync with the network (and validating) + bool proposing = + validating_ && (app_.getOPs().getOperatingMode() == NetworkOPs::omFULL); + + if (validating_) + { + JLOG(j_.info()) << "Entering consensus process, validating"; + } + else + { + // Otherwise we just want to monitor the validation process. + JLOG(j_.info()) << "Entering consensus process, watching"; + } + + // Notify inbOund ledgers that we are starting a new round + inboundTransactions_.newRound(prevLgr.seq()); + + Base::startRound(now, prevLgrId, prevLgr, proposing); +} } diff --git a/src/ripple/app/consensus/RCLConsensus.h b/src/ripple/app/consensus/RCLConsensus.h index 0848bc278..26478faee 100644 --- a/src/ripple/app/consensus/RCLConsensus.h +++ b/src/ripple/app/consensus/RCLConsensus.h @@ -1,4 +1,4 @@ - +//------------------------------------------------------------------------------ /* This file is part of rippled: https://github.com/ripple/rippled Copyright (c) 2012, 2013 Ripple Labs Inc. @@ -21,19 +21,19 @@ #define RIPPLE_APP_CONSENSUS_RCLCONSENSUS_H_INCLUDED #include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include +#include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include namespace ripple { @@ -50,37 +50,40 @@ struct RCLCxTraits using NodeID_t = NodeID; //! TxSet type presented to Consensus using TxSet_t = RCLTxSet; - //! MissingTxException type neede by Consensus - using MissingTxException_t = SHAMapMissingNode; }; - /** Adapts the generic Consensus algorithm for use by RCL. @note The enabled_shared_from_this base allows the application to properly create a shared instance of RCLConsensus for use in the accept logic.. */ -class RCLConsensus : public Consensus - , public std::enable_shared_from_this - , public CountedObject +class RCLConsensus final : public Consensus, + public std::enable_shared_from_this, + public CountedObject { using Base = Consensus; - using Base::accept; -public: +public: //! Constructor RCLConsensus( Application& app, - std::unique_ptr && feeVote, + std::unique_ptr&& feeVote, LedgerMaster& ledgerMaster, LocalTxs& localTxs, InboundTransactions& inboundTransactions, - typename Base::clock_type const & clock, + typename Base::clock_type const& clock, beast::Journal journal); - RCLConsensus(RCLConsensus const&) = delete; - RCLConsensus& operator=(RCLConsensus const&) = delete; - static char const* getCountedObjectName() { return "Consensus"; } + RCLConsensus(RCLConsensus const&) = delete; + + RCLConsensus& + operator=(RCLConsensus const&) = delete; + + static char const* + getCountedObjectName() + { + return "Consensus"; + } /** Save the given consensus proposed by a peer with nodeID for later use in consensus. @@ -89,15 +92,59 @@ public: @param nodeID ID of peer */ void - storeProposal( RCLCxPeerPos::ref peerPos, NodeID const& nodeID); + storeProposal(RCLCxPeerPos::ref peerPos, NodeID const& nodeID); + + //! Whether we are validating consensus ledgers. + bool + validating() const + { + return validating_; + } + + bool + haveCorrectLCL() const + { + return mode() != Mode::wrongLedger; + } + + bool + proposing() const + { + return mode() == Mode::proposing; + } + + /** Get the Json state of the consensus process. + + Called by the consensus_info RPC. + + @param full True if verbose response desired. + @return The Json state. + */ + Json::Value + getJson(bool full) const; + + //! See Consensus::startRound + void + startRound( + NetClock::time_point const& now, + RCLCxLedger::ID const& prevLgrId, + RCLCxLedger const& prevLgr); + + //! See Consensus::timerEntry + void + timerEntry(NetClock::time_point const& now); + + //! See Consensus::gotTxSet + void + gotTxSet(NetClock::time_point const& now, RCLTxSet const& txSet); /** Returns validation public key */ PublicKey const& - getValidationPublicKey () const; + getValidationPublicKey() const; /** Set validation private and public key pair. */ void - setValidationKeys (SecretKey const& valSecret, PublicKey const& valPublic); + setValidationKeys(SecretKey const& valSecret, PublicKey const& valPublic); private: friend class Consensus; @@ -105,17 +152,6 @@ private: //------------------------------------------------------------------------- // Consensus type requirements. - /** Notification that a new consensus round has begun. - - @param ledger The ledger we are building consensus on - */ - void - onStartRound(RCLCxLedger const & ledger); - - //! @return Whether consensus should be (proposing, validating) - std::pair - getMode (); - /** Attempt to acquire a specific ledger. If not available, asynchronously acquires from the network. @@ -124,44 +160,44 @@ private: @return Optional ledger, will be seated if we locally had the ledger */ boost::optional - acquireLedger(LedgerHash const & ledger); + acquireLedger(LedgerHash const& ledger); /** Get peers' proposed positions. @param prevLedger The base ledger which proposals are based on @return The set of proposals */ std::vector - proposals (LedgerHash const& prevLedger); + proposals(LedgerHash const& prevLedger); /** Relay the given proposal to all peers @param peerPos The peer position to relay. */ void - relay(RCLCxPeerPos const & peerPos); + relay(RCLCxPeerPos const& peerPos); /** Relay disputed transacction to peers. Only relay if the provided transaction hasn't been shared recently. - @param dispute The disputed transaction to relay. + @param tx The disputed transaction to relay. */ void - relay(DisputedTx const & dispute); + relay(RCLCxTx const& tx); - /** Acquire the transaction set associated with a proposal. + /** Acquire the transaction set associated with a proposal. - If the transaction set is not available locally, will attempt acquire it - from the network. + If the transaction set is not available locally, will attempt acquire it + from the network. - @param setId The transaction set ID associated with the proposal - @return Optional set of transactions, seated if available. - */ + @param setId The transaction set ID associated with the proposal + @return Optional set of transactions, seated if available. + */ boost::optional - acquireTxSet(RCLTxSet::ID const & setId); + acquireTxSet(RCLTxSet::ID const& setId); /** Whether the open ledger has any transactions - */ + */ bool hasOpenTransactions() const; @@ -171,133 +207,93 @@ private: @return the number of proposers that validated a ledger */ std::size_t - proposersValidated(LedgerHash const & h) const; + proposersValidated(LedgerHash const& h) const; - /** Number of proposers that have validated a ledger descended from requested ledger. + /** Number of proposers that have validated a ledger descended from + requested ledger. @param h The hash of the ledger of interest. @return The number of validating peers that have validated a ledger succeeding the one provided. */ std::size_t - proposersFinished(LedgerHash const & h) const; + proposersFinished(LedgerHash const& h) const; /** Propose the given position to my peers. @param proposal Our proposed position */ void - propose (RCLCxPeerPos::Proposal const& proposal); + propose(RCLCxPeerPos::Proposal const& proposal); - /** Share the given tx set with peers. + /** Relay the given tx set to peers. @param set The TxSet to share. */ void - share (RCLTxSet const& set); + relay(RCLTxSet const& set); - /** Get the last closed ledger (LCL) seen on the network + /** Get the ID of the previous ledger/last closed ledger(LCL) on the network - @param currentLedger Current ledger used in consensus - @param priorLedger Prior ledger used in consensus - @param believedCorrect Whether consensus believes currentLedger is LCL + @param ledgerID ID of previous ledger used by consensus + @param ledger Previous ledger consensus has available + @param mode Current consensus mode + @return The id of the last closed network - @return The hash of the last closed network + @note ledgerID may not match ledger.id() if we haven't acquired + the ledger matching ledgerID from the network */ uint256 - getLCL ( - uint256 const& currentLedger, - uint256 const& priorLedger, - bool believedCorrect); + getPrevLedger( + uint256 ledgerID, + RCLCxLedger const& ledger, + Mode mode); - - /** Notification that the ledger has closed. + /** Close the open ledger and return initial consensus position. @param ledger the ledger we are changing to - @param haveCorrectLCL whether we believe this is the correct LCL + @param closeTime When consensus closed the ledger + @param mode Current consensus mode + @return Tentative consensus result */ - void - onClose(RCLCxLedger const & ledger, bool haveCorrectLCL); + Result + onClose( + RCLCxLedger const& ledger, + NetClock::time_point const& closeTime, + Mode mode); - /** Create our initial position of transactions to accept in this round - of consensus. - - @param prevLedger The ledger the transactions apply to - @param isProposing Whether we are currently proposing - @param isCorrectLCL Whether we have the correct LCL - @param closeTime When we believe the ledger closed - @param now The current network adjusted time - - @return Pair of (i) transactions we believe are in the ledger - (ii) the corresponding proposal of those transactions - to send to peers - */ - std::pair - makeInitialPosition ( - RCLCxLedger const & prevLedger, - bool isProposing, - bool isCorrectLCL, - NetClock::time_point closeTime, - NetClock::time_point now); - - - /** Dispatch a call to Consensus::accept + /** Process the accepted ledger. Accepting a ledger may be expensive, so this function can dispatch - that call to another thread if desired and must call the accept - method of the generic consensus algorithm. + that call to another thread if desired. - @param txSet The transactions to accept. + @param result The result of consensus + @param prevLedger The closed ledger consensus worked from + @param closeResolution The resolution used in agreeing on an effective + closeTiem + @param rawCloseTimes The unrounded closetimes of ourself and our peers + @param mode Our participating mode at the time consensus was declared */ void - dispatchAccept(RCLTxSet const & txSet); + onAccept( + Result const& result, + RCLCxLedger const& prevLedger, + NetClock::duration const & closeResolution, + CloseTimes const& rawCloseTimes, + Mode const& mode); + /** Process the accepted ledger that was a result of simulation/force + accept. - /** Accept a new ledger based on the given transactions. - - TODO: Too many arguments, need to group related types. - - @param set The set of accepted transactions - @param consensusCloseTime Consensus agreed upon close time - @param proposing_ Whether we are proposing - @param validating_ Whether we are validating - @param haveCorrectLCL_ Whether we had the correct last closed ledger - @param consensusFail_ Whether consensus failed - @param prevLedgerHash_ The hash/id of the previous ledger - @param previousLedger_ The previous ledger - @param closeResolution_ The close time resolution used this round - @param now Current network adjsuted time - @param roundTime_ Duration of this consensus round - @param disputes_ Disputed trarnsactions from this round - @param closeTimes_ Histogram of peers close times - @param closeTime Our close time - @return Whether we should continue validating - */ - bool - accept( - RCLTxSet const& set, - NetClock::time_point consensusCloseTime, - bool proposing_, - bool validating_, - bool haveCorrectLCL_, - bool consensusFail_, - LedgerHash const &prevLedgerHash_, - RCLCxLedger const & previousLedger_, - NetClock::duration closeResolution_, - NetClock::time_point const & now, - std::chrono::milliseconds const & roundTime_, - hash_map> const & disputes_, - std::map closeTimes_, - NetClock::time_point const & closeTime - ); - - /** Signal the end of consensus to the application, which will start the - next round. - - @param correctLCL Whether we believe we have the correct LCL + @ref onAccept */ void - endConsensus(bool correctLCL); + onForceAccept( + Result const& result, + RCLCxLedger const& prevLedger, + NetClock::duration const &closeResolution, + CloseTimes const& rawCloseTimes, + Mode const& mode); //!------------------------------------------------------------------------- // Additional members (not directly required by Consensus interface) @@ -308,42 +304,53 @@ private: @param haveCorrectLCL Whether we believ we have the correct LCL. */ void - notify(protocol::NodeEvent ne, RCLCxLedger const & ledger, bool haveCorrectLCL); + notify( + protocol::NodeEvent ne, + RCLCxLedger const& ledger, + bool haveCorrectLCL); - /** Build the new last closed ledger. + /** Accept a new ledger based on the given transactions. - Accept the given the provided set of consensus transactions and build - the last closed ledger. Since consensus just agrees on which - transactions to apply, but not whether they make it into the closed - ledger, this function also populates retriableTxs with those that can - be retried in the next round. + @ref onAccept + */ + void + doAccept( + Result const& result, + RCLCxLedger const& prevLedger, + NetClock::duration closeResolution, + CloseTimes const& rawCloseTimes, + Mode const& mode); - @param previousLedger Prior ledger building upon - @param set The set of transactions to apply to the ledger - @param closeTime The the ledger closed - @param closeTimeCorrect Whether consensus agreed on close time - @param closeResolution Resolution used to determine consensus close time - @param now Current network adjusted time - @param roundTime Duration of this consensus rorund - @param retriableTxs Populate with transactions to retry in next round - @return The newly built ledger - */ + /** Build the new last closed ledger. + + Accept the given the provided set of consensus transactions and build + the last closed ledger. Since consensus just agrees on which + transactions to apply, but not whether they make it into the closed + ledger, this function also populates retriableTxs with those that can + be retried in the next round. + + @param previousLedger Prior ledger building upon + @param set The set of transactions to apply to the ledger + @param closeTime The the ledger closed + @param closeTimeCorrect Whether consensus agreed on close time + @param closeResolution Resolution used to determine consensus close time + @param roundTime Duration of this consensus rorund + @param retriableTxs Populate with transactions to retry in next round + @return The newly built ledger + */ RCLCxLedger buildLCL( - RCLCxLedger const & previousLedger, - RCLTxSet const & set, + RCLCxLedger const& previousLedger, + RCLTxSet const& set, NetClock::time_point closeTime, bool closeTimeCorrect, NetClock::duration closeResolution, - NetClock::time_point now, std::chrono::milliseconds roundTime, - CanonicalTXSet & retriableTxs - ); + CanonicalTXSet& retriableTxs); /** Validate the given ledger and share with peers as necessary @param ledger The ledger to validate - @param now Current network adjusted time @param proposing Whether we were proposing transactions while generating this ledger. If we are not proposing, a validation can still be sent to inform peers that we know we @@ -351,16 +358,13 @@ private: around and trying to catch up. */ void - validate( - RCLCxLedger const & ledger, - NetClock::time_point now, - bool proposing); + validate(RCLCxLedger const& ledger, bool proposing); //!------------------------------------------------------------------------- Application& app_; - std::unique_ptr feeVote_; - LedgerMaster & ledgerMaster_; - LocalTxs & localTxs_; + std::unique_ptr feeVote_; + LedgerMaster& ledgerMaster_; + LocalTxs& localTxs_; InboundTransactions& inboundTransactions_; beast::Journal j_; @@ -373,11 +377,13 @@ private: // only used for our own validations. NetClock::time_point lastValidationTime_; - using PeerPositions = hash_map >; + using PeerPositions = hash_map>; PeerPositions peerPositions_; std::mutex peerPositionsLock_; -}; + bool validating_ = false; + bool simulating_ = false; +}; } #endif diff --git a/src/ripple/app/consensus/RCLCxLedger.h b/src/ripple/app/consensus/RCLCxLedger.h index 9e383ac1f..29841d472 100644 --- a/src/ripple/app/consensus/RCLCxLedger.h +++ b/src/ripple/app/consensus/RCLCxLedger.h @@ -21,8 +21,8 @@ #define RIPPLE_APP_CONSENSUS_RCLCXLEDGER_H_INCLUDED #include -#include #include +#include #include #include @@ -50,24 +50,26 @@ public: @param l The ledger to wrap. */ - RCLCxLedger(std::shared_ptr const & l) : ledger_{ l } {} + RCLCxLedger(std::shared_ptr const& l) : ledger_{l} + { + } //! Sequence number of the ledger. - auto const & + auto const& seq() const { return ledger_->info().seq; } //! Unique identifier (hash) of this ledger. - auto const & + auto const& id() const { return ledger_->info().hash; } //! Unique identifier (hash) of this ledger's parent. - auto const & + auto const& parentID() const { return ledger_->info().parentHash; @@ -114,8 +116,6 @@ public: a new ledger from a readView? */ std::shared_ptr ledger_; - }; - } #endif diff --git a/src/ripple/app/consensus/RCLCxPeerPos.cpp b/src/ripple/app/consensus/RCLCxPeerPos.cpp index 9b8a8799d..73f278fe7 100644 --- a/src/ripple/app/consensus/RCLCxPeerPos.cpp +++ b/src/ripple/app/consensus/RCLCxPeerPos.cpp @@ -19,28 +19,28 @@ #include #include -#include #include -#include #include +#include #include +#include namespace ripple { // Used to construct received proposals -RCLCxPeerPos::RCLCxPeerPos ( - PublicKey const& publicKey, - Slice const& signature, - uint256 const& suppression, - Proposal && proposal) - : proposal_{ std::move(proposal)} - , mSuppression {suppression} +RCLCxPeerPos::RCLCxPeerPos( + PublicKey const& publicKey, + Slice const& signature, + uint256 const& suppression, + Proposal&& proposal) + : proposal_{std::move(proposal)} + , mSuppression{suppression} , publicKey_{publicKey} , signature_{signature} { - } -uint256 RCLCxPeerPos::getSigningHash () const +uint256 +RCLCxPeerPos::getSigningHash() const { return sha512Half( HashPrefix::proposal, @@ -50,28 +50,25 @@ uint256 RCLCxPeerPos::getSigningHash () const proposal().position()); } -bool RCLCxPeerPos::checkSign () const +bool +RCLCxPeerPos::checkSign() const { - return verifyDigest ( - publicKey_, - getSigningHash(), - signature_, - false); + return verifyDigest(publicKey_, getSigningHash(), signature_, false); } -Json::Value RCLCxPeerPos::getJson () const +Json::Value +RCLCxPeerPos::getJson() const { auto ret = proposal().getJson(); if (publicKey_.size()) - ret[jss::peer_id] = toBase58 ( - TokenType::TOKEN_NODE_PUBLIC, - publicKey_); + ret[jss::peer_id] = toBase58(TokenType::TOKEN_NODE_PUBLIC, publicKey_); return ret; } -uint256 proposalUniqueId ( +uint256 +proposalUniqueId( uint256 const& proposeHash, uint256 const& previousLedger, std::uint32_t proposeSeq, @@ -79,15 +76,15 @@ uint256 proposalUniqueId ( Slice const& publicKey, Slice const& signature) { - Serializer s (512); - s.add256 (proposeHash); - s.add256 (previousLedger); - s.add32 (proposeSeq); - s.add32 (closeTime.time_since_epoch().count()); - s.addVL (publicKey); - s.addVL (signature); + Serializer s(512); + s.add256(proposeHash); + s.add256(previousLedger); + s.add32(proposeSeq); + s.add32(closeTime.time_since_epoch().count()); + s.addVL(publicKey); + s.addVL(signature); - return s.getSHA512Half (); + return s.getSHA512Half(); } -} // ripple +} // ripple diff --git a/src/ripple/app/consensus/RCLCxPeerPos.h b/src/ripple/app/consensus/RCLCxPeerPos.h index 2019f49c8..b358d4606 100644 --- a/src/ripple/app/consensus/RCLCxPeerPos.h +++ b/src/ripple/app/consensus/RCLCxPeerPos.h @@ -22,12 +22,12 @@ #include #include +#include +#include #include #include #include #include -#include -#include #include #include #include @@ -38,18 +38,20 @@ namespace ripple { Carries a ConsensusProposal signed by a peer. */ -class RCLCxPeerPos - : public CountedObject +class RCLCxPeerPos : public CountedObject { public: - static char const* getCountedObjectName () { return "RCLCxPeerPos"; } + static char const* + getCountedObjectName() + { + return "RCLCxPeerPos"; + } using pointer = std::shared_ptr; using ref = const pointer&; //< The type of the proposed position using Proposal = ConsensusProposal; - /** Constructor Constructs a signed peer position. @@ -60,57 +62,64 @@ public: @param proposal The consensus proposal */ - RCLCxPeerPos ( + RCLCxPeerPos( PublicKey const& publicKey, Slice const& signature, uint256 const& suppress, - Proposal && proposal); + Proposal&& proposal); //! Create the signing hash for the proposal - uint256 getSigningHash () const; + uint256 + getSigningHash() const; //! Verify the signing hash of the proposal - bool checkSign () const; + bool + checkSign() const; //! Signature of the proposal (not necessarily verified) - Slice getSignature () const + Slice + getSignature() const { return signature_; } //! Public key of peer that sent the proposal - PublicKey const& getPublicKey () const + PublicKey const& + getPublicKey() const { return publicKey_; } //! ????? - uint256 const& getSuppressionID () const + uint256 const& + getSuppressionID() const { return mSuppression; } //! The consensus proposal - Proposal const & proposal() const + Proposal const& + proposal() const { return proposal_; } /// @cond Ignore //! Add a conversion operator to conform to the Consensus interface - operator Proposal const &() const + operator Proposal const&() const { return proposal_; } /// @endcond //! JSON representation of proposal - Json::Value getJson () const; + Json::Value + getJson() const; private: template void - hash_append (Hasher& h) const + hash_append(Hasher& h) const { using beast::hash_append; hash_append(h, HashPrefix::proposal); @@ -142,14 +151,15 @@ private: @param publicKey Signer's public key @param signature Proposal signature */ -uint256 proposalUniqueId ( - uint256 const& proposeHash, - uint256 const& previousLedger, - std::uint32_t proposeSeq, - NetClock::time_point closeTime, - Slice const& publicKey, - Slice const& signature); +uint256 +proposalUniqueId( + uint256 const& proposeHash, + uint256 const& previousLedger, + std::uint32_t proposeSeq, + NetClock::time_point closeTime, + Slice const& publicKey, + Slice const& signature); -} // ripple +} // ripple #endif diff --git a/src/ripple/app/consensus/RCLCxTx.h b/src/ripple/app/consensus/RCLCxTx.h index 0441b54b4..aae8cf1fc 100644 --- a/src/ripple/app/consensus/RCLCxTx.h +++ b/src/ripple/app/consensus/RCLCxTx.h @@ -20,10 +20,10 @@ #ifndef RIPPLE_APP_CONSENSUS_RCLCXTX_H_INCLUDED #define RIPPLE_APP_CONSENSUS_RCLCXTX_H_INCLUDED +#include #include #include #include -#include namespace ripple { @@ -42,14 +42,15 @@ public: @param txn The transaction to wrap */ - RCLCxTx(SHAMapItem const& txn) : tx_{ txn } - { } + RCLCxTx(SHAMapItem const& txn) : tx_{txn} + { + } //! The unique identifier/hash of the transaction ID const& id() const { - return tx_.key (); + return tx_.key(); } //! The SHAMapItem that represents the transaction. @@ -74,13 +75,11 @@ public: { friend class RCLTxSet; //! The SHAMap representing the transactions. - std::shared_ptr map_; + std::shared_ptr map_; public: - MutableTxSet(RCLTxSet const & src) - : map_{ src.map_->snapShot(true) } + MutableTxSet(RCLTxSet const& src) : map_{src.map_->snapShot(true)} { - } /** Insert a new transaction into the set. @@ -92,8 +91,7 @@ public: insert(Tx const& t) { return map_->addItem( - SHAMapItem{ t.id(), t.tx_.peekData() }, - true, false); + SHAMapItem{t.id(), t.tx_.peekData()}, true, false); } /** Remove a transaction from the set. @@ -112,8 +110,7 @@ public: @param m SHAMap to wrap */ - RCLTxSet (std::shared_ptr m) - : map_{ std::move(m) } + RCLTxSet(std::shared_ptr m) : map_{std::move(m)} { assert(map_); } @@ -122,10 +119,8 @@ public: @param m MutableTxSet that will become fixed */ - RCLTxSet(MutableTxSet const & m) - : map_{m.map_->snapShot(false)} + RCLTxSet(MutableTxSet const& m) : map_{m.map_->snapShot(false)} { - } /** Test if a transaction is in the set. @@ -136,7 +131,7 @@ public: bool exists(Tx::ID const& entry) const { - return map_->hasItem (entry); + return map_->hasItem(entry); } /** Lookup a transaction. @@ -150,10 +145,10 @@ public: code use the shared_ptr semantics to know whether the find was succesfully and properly creates a Tx as needed. */ - std::shared_ptr const & + std::shared_ptr const& find(Tx::ID const& entry) const { - return map_->peekItem (entry); + return map_->peekItem(entry); } //! The unique ID/hash of the transaction set @@ -163,7 +158,8 @@ public: return map_->getHash().as_uint256(); } - /** Find transactions not in common between this and another transaction set. + /** Find transactions not in common between this and another transaction + set. @param j The set to compare with @return Map of transactions in this set and `j` but not both. The key @@ -171,28 +167,28 @@ public: exists in this set. */ std::map - compare (RCLTxSet const& j) const + compare(RCLTxSet const& j) const { SHAMap::Delta delta; // Bound the work we do in case of a malicious // map_ from a trusted validator - map_->compare (*(j.map_), delta, 65536); + map_->compare(*(j.map_), delta, 65536); - std::map ret; + std::map ret; for (auto const& item : delta) { - assert ( (item.second.first && ! item.second.second) || - (item.second.second && ! item.second.first) ); + assert( + (item.second.first && !item.second.second) || + (item.second.second && !item.second.first)); - ret[item.first] = static_cast (item.second.first); + ret[item.first] = static_cast(item.second.first); } return ret; } //! The SHAMap representing the transactions. - std::shared_ptr map_; + std::shared_ptr map_; }; - } #endif diff --git a/src/ripple/app/misc/NetworkOPs.cpp b/src/ripple/app/misc/NetworkOPs.cpp index 5b60b1f78..18ce779cb 100644 --- a/src/ripple/app/misc/NetworkOPs.cpp +++ b/src/ripple/app/misc/NetworkOPs.cpp @@ -322,7 +322,7 @@ private: public: bool beginConsensus (uint256 const& networkClosed) override; - void endConsensus (bool correctLCL) override; + void endConsensus () override; void setStandAlone () override { setMode (omFULL); @@ -1525,7 +1525,7 @@ bool NetworkOPsImp::beginConsensus (uint256 const& networkClosed) uint256 NetworkOPsImp::getConsensusLCL () { - return mConsensus->LCL (); + return mConsensus->prevLedgerID (); } void NetworkOPsImp::processTrustedProposal ( @@ -1565,7 +1565,7 @@ NetworkOPsImp::mapComplete ( RCLTxSet{map}); } -void NetworkOPsImp::endConsensus (bool correctLCL) +void NetworkOPsImp::endConsensus () { uint256 deadLedger = m_ledgerMaster.getClosedLedger ()->info().parentHash; @@ -2164,18 +2164,18 @@ Json::Value NetworkOPsImp::getServerInfo (bool human, bool admin) info[jss::peers] = Json::UInt (app_.overlay ().size ()); Json::Value lastClose = Json::objectValue; - lastClose[jss::proposers] = mConsensus->getLastCloseProposers(); + lastClose[jss::proposers] = Json::UInt(mConsensus->prevProposers()); if (human) { lastClose[jss::converge_time_s] = std::chrono::duration{ - mConsensus->getLastConvergeDuration()}.count(); + mConsensus->prevRoundTime()}.count(); } else { lastClose[jss::converge_time] = - Json::Int (mConsensus->getLastConvergeDuration().count()); + Json::Int (mConsensus->prevRoundTime().count()); } info[jss::last_close] = lastClose; @@ -2754,9 +2754,7 @@ std::uint32_t NetworkOPsImp::acceptLedger ( // FIXME Could we improve on this and remove the need for a specialized // API in Consensus? beginConsensus (m_ledgerMaster.getClosedLedger()->info().hash); - mConsensus->simulate ( - app_.timeKeeper().closeTime(), - consensusDelay); + mConsensus->simulate (app_.timeKeeper().closeTime(), consensusDelay); return m_ledgerMaster.getCurrentLedger ()->info().seq; } diff --git a/src/ripple/app/misc/NetworkOPs.h b/src/ripple/app/misc/NetworkOPs.h index 65106d770..4fee5292e 100644 --- a/src/ripple/app/misc/NetworkOPs.h +++ b/src/ripple/app/misc/NetworkOPs.h @@ -162,7 +162,7 @@ public: // network state machine virtual bool beginConsensus (uint256 const& netLCL) = 0; - virtual void endConsensus (bool correctLCL) = 0; + virtual void endConsensus () = 0; virtual void setStandAlone () = 0; virtual void setStateTimer () = 0; diff --git a/src/ripple/consensus/Consensus.h b/src/ripple/consensus/Consensus.h index 7ea6fe29c..973b2f2c4 100644 --- a/src/ripple/consensus/Consensus.h +++ b/src/ripple/consensus/Consensus.h @@ -1,7 +1,7 @@ //------------------------------------------------------------------------------ /* This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. + Copyright (c) 2012-2017 Ripple Labs Inc. Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above @@ -20,11 +20,11 @@ #ifndef RIPPLE_CONSENSUS_CONSENSUS_H_INCLUDED #define RIPPLE_CONSENSUS_CONSENSUS_H_INCLUDED -#include -#include -#include #include #include +#include +#include +#include #include namespace ripple { @@ -38,31 +38,30 @@ namespace ripple { 1. The set of transactions included in the ledger. 2. The close time for the ledger. - The general consensus stages: + The basic flow: - 1. Consensus finishes, we build a new last closed ledger and a new open - ledger based on it. - 2. The open ledger interval starts. This gives servers time to finish - building the new last closed ledger and fill the new ledger - with transactions. - 3. The ledger closes. Servers send their initial proposal. - 4. We do not change our position or declare a consensus for at least - LEDGER_MIN_CONSENSUS to ensure servers have a chance to make an initial - proposal. - 5. On a frequent timer event, we change our position if needed based on - received peer positions. - 6. When we have a consensus, go to step 1. + 1. A call to `startRound` places the node in the `Open` phase. In this + phase, the node is waiting for transactions to include in its open ledger. + 2. Successive calls to `timerEntry` check if the node can close the ledger. + Once the node `Close`s the open ledger, it transitions to the + `Establish` phase. In this phase, the node shares/receives peer + proposals on which transactions should be accepted in the closed ledger. + 3. During a subsequent call to `timerEntry`, the node determines it has + reached consensus with its peers on which transactions to include. It + transitions to the `Accept` phase. In this phase, the node works on + applying the transactions to the prior ledger to generate a new closed + ledger. Once the new ledger is completed, the node shares the validated + ledger with the network, does some book-keeping, then makes a call to + `startRound` to start the cycle again. This class uses CRTP to allow adapting Consensus for specific applications. The Derived template argument is used to embed consensus within the larger application framework. The Traits template identifies types that play important roles in Consensus (transactions, ledgers, etc.) and which must - conform to the generic interface outlined below. The Traits typesmust be copy + conform to the generic interface outlined below. The Traits types must be copy constructible and assignable. - @note The interface below is in flux as this code is refactored. - @code // A single transaction struct Tx @@ -113,13 +112,12 @@ namespace ripple { using ID = ...; // Unique identifier of ledgerr - ID const & id() const; + ID const id() const; auto seq() const; auto closeTimeResolution() const; auto closeAgree() const; auto closeTime() const; auto parentCloseTime() const; - auto const & parentID() const; Json::Value getJson() const; }; @@ -128,51 +126,54 @@ namespace ripple { using Ledger_t = Ledger; using NodeID_t = std::uint32_t; using TxSet_t = TxSet; - // To be removed; currently needed to handle missing SHAMap node exception - using MissingTxException_t = MissingTx; } class ConsensusImp : public Consensus { - // Whether consensus should (proposing,validating). - std::pair getMode(); - // Attempt to acquire a specific ledger. - Ledger const * acquireLedger(Ledger::ID const & ledgerID); + boost::optional acquireLedger(Ledger::ID const & ledgerID); + + // Acquire the transaction set associated with a proposed position. + boost::optional acquireTxSet(TxSet::ID const & setID); // Get peers' proposed positions. Returns an iterable // with value_type convertable to ConsensusPosition<...> auto const & proposals(Ledger::ID const & ledgerID); - // Acquire the transaction set associated with a proposed position. - TxSet const * acquireTxSet(TxSet::ID const & setID); - - // Whether the open ledger has any transactions + // Whether any transactions are in the open ledger bool hasOpenTransactions() const; - // Number of proposers that have vallidated the given ledger + // Number of proposers that have validated the given ledger std::size_t proposersValidated(Ledger::ID const & prevLedger) const; - // Number of proposers that have validated a ledger descended from requested ledger. - std::size_t proposersFinished(LedgerHash const & h) const;re + // Number of proposers that have validated a ledger descended from the + // given ledger + std::size_t proposersFinished(Ledger::ID const & prevLedger) const; - // Called when a new round of consensus has started - void onStartRound(Ledger const &); + // Return the ID of the last closed (and validated) ledger that the + // application thinks consensus should use as the prior ledger. + Ledger::ID getPrevLedger(Ledger::ID const & prevLedgerID, + Ledger const & prevLedger, + Mode mode); - // Called when ledger is closes - void onClose(Ledger const &, bool haveCorrectLCL); - // Setup later call to Consensus::accept which will call Derived::accept - // Can call Consensus::accept immediately if no interest in offloading - void dispatchAccept(TxSet const & ); + // Called when ledger closes + Result onClose(Ledger const &, Ledger const & prev, Mode mode); - // Share given transaction set with peers - void share(TxSet const &s); + // Called when ledger is accepted by consensus + void onAccept(Result const & result, + RCLCxLedger const & prevLedger, + NetClock::duration closeResolution, + CloseTimes const & rawCloseTimes, + Mode const & mode); - // Return the ID of the last closed (and validated) ledger - Ledger::ID getLCL(Ledger::ID const & currLedger, - Ledger::ID const & priorLedger, - bool haveCorrectLCL); + // Called when ledger was forcibly accepted by consensus via the simulate + // function. + void onForceAccept(Result const & result, + RCLCxLedger const & prevLedger, + NetClock::duration closeResolution, + CloseTimes const & rawCloseTimes, + Mode const & mode); // Propose the position to peers. void propose(ConsensusProposal<...> const & pos); @@ -182,26 +183,12 @@ namespace ripple { // but may be a different type. void relay(implementation_defined const & pos); - // Relay disputed transaction to peers - void relay(DisputedTx const & dispute); + // Relay a disputed transaction to peers + void relay(Txn const & tx); - // Create initial position for current consensus round - std::pair - makeInitialPosition( - Ledger const & prevLedger, - bool isProposing, - bool isCorrectLCL, - NetClock::time_point closeTime, - NetClock::time_point now) + // Share given transaction set with peers + void relay(TxSet const &s); - - // Process the accepted transaction set, generating the newly closed ledger - // and clearing out the openTxs that were included. - bool accept(TxSet const& set, ... ) - - // Called when time to end current round of consensus. Client code - // determines when to call startRound again. - void endConsensus(bool correct); }; @endcode @@ -211,19 +198,29 @@ namespace ripple { template class Consensus { - //! Current stage of consensus - enum class State - { + //! Phases of consensensus: + //! "close" "accept" + //! open ------- > establish ---------> accepted + //! ^ | | + //! |---------------| | + //! ^ "startRound" | + //! |------------------------------------| + //! + //! The typical transition goes from open to establish to accepted and + //! then a call to startRound begins the process anew. + //! However, if a wrong prior ledger is detected and recovered + //! during the establish or accept phase, consensus will internally go back + //! to open (see handleWrongLedger). + enum class Phase { //! We haven't closed our ledger yet, but others might have open, - //! Establishing consensus + //! Establishing consensus by exchanging proposals with our peers establish, - //! We have closed on a transaction set and are processing the new ledger - processing, - - //! We have accepted a new last closed ledger and need to start a new round + //! We have accepted a new last closed ledger and are waiting on a call + //! to startRound to begin the next consensus round. No changes + //! to consensus phase occur while in this phase. accepted, }; @@ -231,147 +228,240 @@ class Consensus using TxSet_t = typename Traits::TxSet_t; using NodeID_t = typename Traits::NodeID_t; using Tx_t = typename TxSet_t::Tx; - using Proposal_t = ConsensusProposal; - using Dispute_t = DisputedTx; + +protected: + //! How we participating in Consensus + //! proposing observing + //! \ / + //| \---> wrongLedger <---/ + //! ^ + //! | + //! | + //! v + //! switchedLedger + //! We enter the round proposing or observing. If we detect we + //! are working on the wrong prior ledger, we go to + //! wrongLedger and attempt to acquire the right one (ref + //! handleWrongLedger). Once we acquire the right one, we go to + //! switchedLedger. If we again detect the wrongLedger before this round + //! ends, we go back to wrongLedger until we acquire the right one. + enum class Mode { + //! We a normal participant in consensus and propose our position + proposing, + //! We are observing peer positions, but not proposing our position + observing, + //! We have the wrong ledger and are attempting to acquire it + wrongLedger, + //! We switched ledger since we started this consensus round but are now + //! running on what we believe is the correct ledger. This mode is as + //! if we entered the round observing, but is used to indicate we did + //! have the wrongLedger at some point. + switchedLedger + }; + + //! Measure duration of phases of consensus + class Stopwatch + { + using time_point = std::chrono::steady_clock::time_point; + time_point start_; + std::chrono::milliseconds dur_; + + public: + std::chrono::milliseconds + read() const + { + return dur_; + } + + void + tick(std::chrono::milliseconds fixed) + { + dur_ += fixed; + } + + void + reset(time_point tp) + { + start_ = tp; + dur_ = std::chrono::milliseconds{0}; + } + + void + tick(time_point tp) + { + using namespace std::chrono; + dur_ = duration_cast(tp - start_); + } + }; + + //! Initial ledger close times, not rounded by closeTimeResolution + struct CloseTimes + { + // Close time estimates, keep ordered for predictable traverse + std::map peers; + NetClock::time_point self; + }; + + /** Enacpsulates the result of consensus. + + While in the establish phase, represents our work in progress consensus + result. In the accept phase, represents our final consensus result + for this round. + */ + struct Result + { + using Dispute_t = DisputedTx; + + Result(TxSet_t&& s, Proposal_t&& p) + : set{std::move(s)}, position{std::move(p)} + { + assert(set.id() == position.position()); + } + + //! The set of transactions consensus agrees go in the ledger + TxSet_t set; + + //! Our proposed position on transactions/close time + Proposal_t position; + + //! Transactions which are under dispute with our peers + hash_map disputes; + + // Set of TxSet ids we have already compared/created disputes + hash_set compares; + + // Measures the duration of the establish phase for this consensus round + Stopwatch roundTime; + + // Indicates state in which consensus ended. Once in the accept phase + // will be either Yes or MovedOn + ConsensusState state = ConsensusState::No; + }; public: - //! Clock type for measuring time within the consensus code - using clock_type = beast::abstract_clock ; + using clock_type = beast::abstract_clock; - Consensus(Consensus &&) = default; + Consensus(Consensus&&) = default; /** Constructor. @param clock The clock used to internally sample consensus progress @param j The journal to log debug output */ - Consensus(clock_type const & clock, beast::Journal j); - + Consensus(clock_type const& clock, beast::Journal j); /** Kick-off the next round of consensus. Called by the client code to start each round of consensus. @param now The network adjusted time - @param prevLgrId the ID/hash of the last ledger - @param prevLgr Best guess of what the last closed ledger was. + @param prevLedgerID the ID of the last ledger + @param prevLedger The last ledger + @param proposing Whether we want to send proposals to peers this round. - @note @b prevLgrId is not required to the ID of @b prevLgr since - the ID is shared independent of the full ledger. + @note @b prevLedgerID is not required to the ID of @b prevLedger since + the ID may be known locally before the contents of the ledger arrive */ void - startRound ( + startRound( NetClock::time_point const& now, - typename Ledger_t::ID const& prevLgrId, - Ledger_t const& prevLgr); + typename Ledger_t::ID const& prevLedgerID, + Ledger_t const& prevLedger, + bool proposing); - - /** A peer has proposed a new position, adjust our tracking + /** A peer has proposed a new position, adjust our tracking. @param now The network adjusted time @param newProposal The new proposal from a peer @return Whether we should do delayed relay of this proposal. */ bool - peerProposal ( + peerProposal( NetClock::time_point const& now, Proposal_t const& newProposal); - /** Call periodically to drive consensus forward + /** Call periodically to drive consensus forward. @param now The network adjusted time */ void - timerEntry (NetClock::time_point const& now); + timerEntry(NetClock::time_point const& now); - /** - Process a transaction set, typically acquired from the network + /** Process a transaction set acquired from the network @param now The network adjusted time @param txSet the transaction set */ void - gotTxSet ( - NetClock::time_point const& now, - TxSet_t const& txSet); + gotTxSet(NetClock::time_point const& now, TxSet_t const& txSet); /** Simulate the consensus process without any network traffic. - The end result, is that consensus begins and completes as if everyone - had agreed with whatever we propose. + The end result, is that consensus begins and completes as if everyone + had agreed with whatever we propose. - This function is only called from the rpc "ledger_accept" path with the - server in standalone mode and SHOULD NOT be used during the normal - consensus process. + This function is only called from the rpc "ledger_accept" path with the + server in standalone mode and SHOULD NOT be used during the normal + consensus process. - @param now The current network adjusted time. - @param consensusDelay (Optional) duration to delay between closing and - accepting the ledger. Uses 100ms if unspecified. + Simulate will call onForceAccept since clients are manually driving + consensus to the accept phase. + + @param now The current network adjusted time. + @param consensusDelay Duration to delay between closing and accepting the + ledger. Uses 100ms if unspecified. */ void simulate( NetClock::time_point const& now, boost::optional consensusDelay); - /** Get the last closed ledger ID. + /** Get the previous ledger ID. - The last closed ledger is the last validated ledger seen by the consensus - code. + The previous ledger is the last ledger seen by the consensus code and + should correspond to the most recent validated ledger seen by this peer. - @return ID of last closed ledger. + @return ID of previous ledger */ typename Ledger_t::ID - LCL() + prevLedgerID() const { - std::lock_guard _(*lock_); - return prevLedgerID_; + std::lock_guard _(*lock_); + return prevLedgerID_; } - //! Get the number of proposing peers that participated in the previous round. - int - getLastCloseProposers() const + //! Get the number of proposing peers that participated in the previous + //! round. + std::size_t + prevProposers() const { - return previousProposers_; + return prevProposers_; } /** Get duration of the previous round. - The duration of the round is measured from closing the open ledger to - starting acceptance of the consensus transaction set. + The duration of the round is the establish phase, measured from closing + the open ledger to accepting the consensus result. @return Last round duration in milliseconds */ std::chrono::milliseconds - getLastConvergeDuration() const + prevRoundTime() const { - return previousRoundTime_; + return prevRoundTime_; } - //! Whether we are sending proposals during consensus. - bool - proposing() const + /** Get the current consensus mode. + */ + Mode + mode() const { - return proposing_; - } - - //! Whether we are validating consensus ledgers. - bool - validating() const - { - return validating_; - } - - /** Whether we have the correct last closed ledger. - - This is typically a case where we have seen the ID/hash of a newer - ledger, but do not have the ledger itself. - */ - bool - haveCorrectLCL() const - { - return haveCorrectLCL_; + return mode_; } /** Get the Json state of the consensus process. @@ -382,204 +472,119 @@ public: @return The Json state. */ Json::Value - getJson (bool full) const; + getJson(bool full) const; protected: - /** Accept a new last closed ledger. - We believe the network reached Consensus on a set of transactions. This - function accepts those new transactions, creating a new last closed - ledger. The bulk of the work is dispatched to the deriving class' accept - method. + // Prevent deleting derived instance through base pointer + ~Consensus() = default; - @param set Our consensus transaction set - - @note This is protected so the Derived class can call it from dispatchAccept. + /** Revoke our outstanding proposal, if any, and cease proposing + until this round ends. */ void - accept(TxSet_t const& set); + leaveConsensus(); private: - - /** Internal version of @ref startRound to allow restarting state when - wrong LCL acquired. - */ void - startRoundInternal ( + startRoundInternal( NetClock::time_point const& now, - typename Ledger_t::ID const& prevLgrId, - Ledger_t const& prevLgr); + typename Ledger_t::ID const& prevLedgerID, + Ledger_t const& prevLedger, + Mode mode); - /** Change our view of the last closed ledger + // Change our view of the previous ledger + void + handleWrongLedger(typename Ledger_t::ID const& lgrId); - @param lgrId The ID of the last closed ledger to switch to. + /** Check if our previous ledger matches the network's. + + If the previous ledger differs, we are no longer in sync with + the network and need to bow out/switch modes. */ void - handleLCL (typename Ledger_t::ID const& lgrId); - - /** Check if our last closed ledger matches the network's. - - If the last closed ledger differs, we are no longer in sync with - the network. If we enter the consensus round with - the wrong ledger, we can leave it with the correct ledger so - that we can participate in the next round. - */ - void - checkLCL (); + checkLedger(); /** If we radically changed our consensus context for some reason, we need to replay recent proposals so that they're not lost. */ void - playbackProposals (); + playbackProposals(); + /** Handle pre-close phase. - /** Handle pre-close state. - - In the pre-close state, the ledger is open as we wait for new - transactions. After enough time has elapsed, we will close the ledger - and start the consensus process. + In the pre-close phase, the ledger is open as we wait for new + transactions. After enough time has elapsed, we will close the ledger, + switch to the establish phase and start the consensus process. */ void - statePreClose (); + phaseOpen(); - /** Handle establish state. + /** Handle establish phase. - In the establish state, the ledger has closed and we work with peers + In the establish phase, the ledger has closed and we work with peers to reach consensus. Update our position only on the timer, and in this - state. + phase. - If we have consensus, move to the processing state. + If we have consensus, move to the accepted phase. */ void - stateEstablish (); + phaseEstablish(); - /** Close the open ledger and establish initial position. - */ + // Close the open ledger and establish initial position. void - closeLedger (); + closeLedger(); - /** Take an initial position on the consensus set. - */ + // Adjust our positions to try to agree with other validators. void - takeInitialPosition (); + updateOurPositions(); - /** Compare two proposed transaction sets and create disputed - transctions structures for any mismatches - - @param m1 One transaction set - @param m2 The other transaction set - */ - void - createDisputes (TxSet_t const& m1, TxSet_t const& m2); - - /** Add a disputed transaction (one that at least one node wants - in the consensus set and at least one node does not) to our tracking - - @param tx The disputed transaction - */ - void - addDisputedTransaction (Tx_t const& tx); - - /** Adjust the votes on all disputed transactions based - on the set of peers taking this position - - @param txSet A disputed position - @param peers Peers which are taking the position txSet - */ - void adjustCount (TxSet_t const& txSet, std::vector const& peers); - - /** Adjust our positions to try to agree with other validators. - - */ - void - updateOurPositions (); - - /** @return Whether we've reached consensus - */ bool - haveConsensus (); + haveConsensus(); - /** Revoke our outstanding proposal, if any, and cease proposing at least - until this round ends. - */ + // Create disputes between our position and the provided one. void - leaveConsensus (); + createDisputes(TxSet_t const& o); - /** Process complete transaction set. - - Called when: - * We take our initial position - * We take a new position - * We acquire a position a validator took - - We store it, notify peers that we have it, and update our tracking if - any validators currently propose it. - - @param txSet the transaction set. - @param acquired true if we have acquired the transaction set. - */ + // Update our disputes given that this node has adopted a new position. + // Will call createDisputes as needed. void - gotTxSetInternal ( TxSet_t const& txSet, bool acquired); + updateDisputes(NodeID_t const& node, TxSet_t const& other); - - /** Initiate acceptance of a the next closed ledger. - - After consensus is complete, beginAccept is called to start - accepting the consensus transaction set. In synchronous mode, this will - directly call the accept member function. However, since accepting and - generating a new ledger is likely computationally intensive, the - asynchronous mode defer to the Derived class dispatchAccept call to - schedule the call to accept. - - @param synchronous Do not dispatch to Derived and instead call accept - directly. - */ - void - beginAccept (bool synchronous); - - - /** @return The Derived class that implements the CRTP requirements. - */ - Derived & + Derived& impl() { return *static_cast(this); } + static std::string + to_string(Phase p); + + static std::string + to_string(Mode m); + +private: // TODO: Move this to clients std::unique_ptr lock_; - //------------------------------------------------------------------------- - // Consensus state variables - State state_ = State::accepted; - bool proposing_ = false; - bool validating_ = false; - bool haveCorrectLCL_ = false; - bool consensusFail_ = false; - bool haveCloseTimeConsensus_ = false; + Phase phase_ = Phase::accepted; + Mode mode_ = Mode::observing; bool firstRound_ = true; + bool haveCloseTimeConsensus_ = false; - //------------------------------------------------------------------------- - //! Clock for measuring consensus progress - clock_type const & clock_; - - // How much time has elapsed since the round started - std::chrono::milliseconds roundTime_ = std::chrono::milliseconds{0}; - // How long has this round been open - std::chrono::milliseconds openTime_ = std::chrono::milliseconds{0}; + clock_type const& clock_; // How long the consensus convergence has taken, expressed as // a percentage of the time that we expected it to take. int convergePercent_{0}; + + // How long has this round been open + Stopwatch openTime_; + NetClock::duration closeResolution_ = ledgerDefaultTimeResolution; - // When startRound was last called - clock_type::time_point openStartTime_; - // When consensus started - clock_type::time_point consensusStartTime_; // Time it took for the last consensus round to converge - std::chrono::milliseconds previousRoundTime_ = LEDGER_IDLE_INTERVAL; + std::chrono::milliseconds prevRoundTime_ = LEDGER_IDLE_INTERVAL; //------------------------------------------------------------------------- // Network time measurements of consensus progress @@ -587,12 +592,7 @@ private: // The current network adjusted time. This is the network time the // ledger would close if it closed now NetClock::time_point now_; - - // The network time this ledger closed - NetClock::time_point closeTime_; - - // Close time estimates, keep ordered for predictable traverse - std::map closeTimes_; + NetClock::time_point prevCloseTime_; //------------------------------------------------------------------------- // Non-peer (self) consensus data @@ -605,177 +605,137 @@ private: // Transaction Sets, indexed by hash of transaction tree hash_map acquired_; - boost::optional ourPosition_; - boost::optional ourSet_; - + boost::optional result_; + CloseTimes rawCloseTimes_; //------------------------------------------------------------------------- // Peer related consensus data // Convergence tracking, trusted peers indexed by hash of public key - hash_map peerProposals_; + hash_map peerProposals_; // The number of proposers who participated in the last consensus round - std::size_t previousProposers_ = 0; - - // Disputed transactions - hash_map disputes_; - - // Set of TxSet ids we have already compared/created disputes - hash_set compares_; + std::size_t prevProposers_ = 0; // nodes that have bowed out of this consensus process hash_set deadNodes_; // Journal for debugging beast::Journal j_; - }; template -Consensus::Consensus ( - clock_type const & clock, - beast::Journal journal) +Consensus::Consensus( + clock_type const& clock, + beast::Journal journal) : lock_(std::make_unique()) , clock_(clock) - , j_(journal) + , j_{journal} { - JLOG (j_.debug()) << "Creating consensus object"; + JLOG(j_.debug()) << "Creating consensus object"; } template void -Consensus::startRound ( +Consensus::startRound( NetClock::time_point const& now, - typename Ledger_t::ID const& prevLCLHash, - Ledger_t const & prevLedger) + typename Ledger_t::ID const& prevLedgerID, + Ledger_t const& prevLedger, + bool proposing) { std::lock_guard _(*lock_); - if (state_ == State::processing) - { - // We can't start a new round while we're processing - return; - } - if (firstRound_) { // take our initial view of closeTime_ from the seed ledger - closeTime_ = prevLedger.closeTime(); + prevCloseTime_ = prevLedger.closeTime(); firstRound_ = false; } - - - // Can only change proposing/validating state on non-internal startRound - // calls - - // We should not be proposing but not validating - // Okay to validate but not propose - std::tie(proposing_, validating_) = impl().getMode(); - assert (! proposing_ || validating_); - - if (validating_) - { - JLOG (j_.info()) - << "Entering consensus process, validating"; - } else { - // Otherwise we just want to monitor the validation process. - JLOG (j_.info()) - << "Entering consensus process, watching"; + prevCloseTime_ = rawCloseTimes_.self; } - - startRoundInternal(now, prevLCLHash, prevLedger); + startRoundInternal( + now, + prevLedgerID, + prevLedger, + proposing ? Mode::proposing : Mode::observing); } - template void -Consensus::startRoundInternal ( +Consensus::startRoundInternal( NetClock::time_point const& now, - typename Ledger_t::ID const& prevLCLHash, - Ledger_t const & prevLedger) + typename Ledger_t::ID const& prevLedgerID, + Ledger_t const& prevLedger, + Mode mode) { - state_ = State::open; + phase_ = Phase::open; + mode_ = mode; now_ = now; - prevLedgerID_ = prevLCLHash; + prevLedgerID_ = prevLedgerID; previousLedger_ = prevLedger; - ourPosition_.reset(); - ourSet_.reset(); - consensusFail_ = false; - roundTime_ = 0ms; + result_.reset(); convergePercent_ = 0; haveCloseTimeConsensus_ = false; - openStartTime_ = clock_.now(); - haveCorrectLCL_ = (previousLedger_.id() == prevLedgerID_); - - impl().onStartRound(previousLedger_); - + openTime_.reset(clock_.now()); peerProposals_.clear(); acquired_.clear(); - disputes_.clear(); - compares_.clear(); - closeTimes_.clear(); + rawCloseTimes_.peers.clear(); + rawCloseTimes_.self = {}; deadNodes_.clear(); - closeResolution_ = getNextLedgerTimeResolution ( + closeResolution_ = getNextLedgerTimeResolution( previousLedger_.closeTimeResolution(), previousLedger_.closeAgree(), previousLedger_.seq() + 1); - - if (! haveCorrectLCL_) + if (previousLedger_.id() != prevLedgerID_) { - // If we were not handed the correct LCL, then set our state - // to not proposing. - handleLCL (prevLedgerID_); + handleWrongLedger(prevLedgerID_); - if (! haveCorrectLCL_) + // Unable to acquire the correct ledger + if (mode_ == Mode::wrongLedger) { - JLOG (j_.info()) - << "Entering consensus with: " - << previousLedger_.id(); - JLOG (j_.info()) - << "Correct LCL is: " << prevLCLHash; + JLOG(j_.info()) + << "Entering consensus with: " << previousLedger_.id(); + JLOG(j_.info()) << "Correct LCL is: " << prevLedgerID; } } - playbackProposals (); - if (peerProposals_.size() > (previousProposers_ / 2)) + playbackProposals(); + if (peerProposals_.size() > (prevProposers_ / 2)) { // We may be falling behind, don't wait for the timer // consider closing the ledger immediately - timerEntry (now_); + timerEntry(now_); } } template bool -Consensus::peerProposal ( +Consensus::peerProposal( NetClock::time_point const& now, Proposal_t const& newProposal) { - auto const peerID = newProposal.nodeID (); + auto const peerID = newProposal.nodeID(); std::lock_guard _(*lock_); // Nothing to do if we are currently working on a ledger - if ((state_ == State::processing) || (state_ == State::accepted)) + if (phase_ == Phase::accepted) return false; now_ = now; if (newProposal.prevLedger() != prevLedgerID_) { - JLOG (j_.debug()) << "Got proposal for " - << newProposal.prevLedger() - << " but we are on " << prevLedgerID_; + JLOG(j_.debug()) << "Got proposal for " << newProposal.prevLedger() + << " but we are on " << prevLedgerID_; return false; } - if (deadNodes_.find (peerID) != deadNodes_.end ()) + if (deadNodes_.find(peerID) != deadNodes_.end()) { using std::to_string; - JLOG (j_.info()) - << "Position from dead node: " << to_string (peerID); + JLOG(j_.info()) << "Position from dead node: " << to_string(peerID); return false; } @@ -785,25 +745,26 @@ Consensus::peerProposal ( if (currentPosition != peerProposals_.end()) { - if (newProposal.proposeSeq () - <= currentPosition->second.proposeSeq()) + if (newProposal.proposeSeq() <= + currentPosition->second.proposeSeq()) { return false; } } - if (newProposal.isBowOut ()) + if (newProposal.isBowOut()) { using std::to_string; - JLOG (j_.info()) - << "Peer bows out: " << to_string (peerID); - - for (auto& it : disputes_) - it.second.unVote (peerID); + JLOG(j_.info()) << "Peer bows out: " << to_string(peerID); + if (result_) + { + for (auto& it : result_->disputes) + it.second.unVote(peerID); + } if (currentPosition != peerProposals_.end()) - peerProposals_.erase (peerID); - deadNodes_.insert (peerID); + peerProposals_.erase(peerID); + deadNodes_.insert(peerID); return true; } @@ -811,44 +772,35 @@ Consensus::peerProposal ( if (currentPosition != peerProposals_.end()) currentPosition->second = newProposal; else - peerProposals_.emplace (peerID, newProposal); + peerProposals_.emplace(peerID, newProposal); } - if (newProposal.isInitial ()) + if (newProposal.isInitial()) { // Record the close time estimate - JLOG (j_.trace()) - << "Peer reports close time as " - << newProposal.closeTime().time_since_epoch().count(); - ++closeTimes_[newProposal.closeTime()]; + JLOG(j_.trace()) << "Peer reports close time as " + << newProposal.closeTime().time_since_epoch().count(); + ++rawCloseTimes_.peers[newProposal.closeTime()]; } - JLOG (j_.trace()) << "Processing peer proposal " - << newProposal.proposeSeq () << "/" - << newProposal.position (); + JLOG(j_.trace()) << "Processing peer proposal " << newProposal.proposeSeq() + << "/" << newProposal.position(); { - auto ait = acquired_.find (newProposal.position()); + auto ait = acquired_.find(newProposal.position()); if (ait == acquired_.end()) { + // acquireTxSet will return the set if it is available, or + // spawn a request for it and return none/nullptr. It will call + // gotTxSet once it arrives if (auto set = impl().acquireTxSet(newProposal.position())) - { - ait = acquired_.emplace (newProposal.position(), - std::move(*set)).first; - } + gotTxSet(now_, *set); + else + JLOG(j_.debug()) << "Don't have tx set for peer"; } - - - if (ait != acquired_.end()) + else if (result_) { - for (auto& it : disputes_) - it.second.setVote (peerID, - ait->second.exists (it.first)); - } - else - { - JLOG (j_.debug()) - << "Don't have tx set for peer"; + updateDisputes(newProposal.nodeID(), ait->second); } } @@ -857,115 +809,111 @@ Consensus::peerProposal ( template void -Consensus::timerEntry (NetClock::time_point const& now) +Consensus::timerEntry(NetClock::time_point const& now) { std::lock_guard _(*lock_); + // Nothing to do if we are currently working on a ledger + if (phase_ == Phase::accepted) + return; + now_ = now; - try + // Check we are on the proper ledger (this may change phase_) + checkLedger(); + + if(phase_ == Phase::open) { - using namespace std::chrono; - - // checkLCL may change state_, so it needs to run prior to the - // switch (state_) below - if(state_ == State::open || state_ == State::establish) - checkLCL (); - - switch (state_) - { - case State::open: - statePreClose (); - break; - - case State::establish: - stateEstablish (); - break; - - case State::processing: - // We are processing the finished ledger - // logic of calculating next ledger advances us out of this state - // nothing to do - break; - - case State::accepted: - // NetworkOPs needs to setup the next round - // nothing to do - break; - - default: - assert (false); - } + phaseOpen(); } - catch (typename Traits::MissingTxException_t const& mn) + else if (phase_ == Phase::establish) { - // This should never happen - leaveConsensus (); - JLOG (j_.error()) << - "Missing node during consensus process " << mn; - Rethrow(); + phaseEstablish(); } } template void -Consensus::gotTxSet ( +Consensus::gotTxSet( NetClock::time_point const& now, TxSet_t const& txSet) { std::lock_guard _(*lock_); - // Nothing to do if we are currently working on a ledger - if ((state_ == State::processing) || (state_ == State::accepted)) + // Nothing to do if we've finished work on a ledger + if (phase_ == Phase::accepted) return; now_ = now; - try + auto id = txSet.id(); + + // If we've already processed this transaction set since requesting + // it from the network, there is nothing to do now + if (!acquired_.emplace(id, txSet).second) + return; + + if (!result_) { - gotTxSetInternal (txSet, true); + JLOG(j_.debug()) << "Not creating disputes: no position yet."; } - catch (typename Traits::MissingTxException_t const& mn) + else { - // This should never happen - leaveConsensus(); - JLOG (j_.error()) << - "Missing node processing complete map " << mn; - Rethrow(); + // Our position is added to acquired_ as soon as we create it, + // so this txSet must differ + assert(id != result_->position.position()); + bool any = false; + for (auto const& p : peerProposals_) + { + if (p.second.position() == id) + { + updateDisputes(p.first, txSet); + any = true; + } + } + + if (!any) + { + JLOG(j_.warn()) + << "By the time we got " << id << " no peers were proposing it"; + } } } template void -Consensus::simulate ( +Consensus::simulate( NetClock::time_point const& now, boost::optional consensusDelay) { std::lock_guard _(*lock_); - JLOG (j_.info()) << "Simulating consensus"; + JLOG(j_.info()) << "Simulating consensus"; now_ = now; - closeLedger (); - roundTime_ = consensusDelay.value_or(100ms); - beginAccept (true); - JLOG (j_.info()) << "Simulation complete"; + closeLedger(); + result_->roundTime.tick(consensusDelay.value_or(100ms)); + prevProposers_ = peerProposals_.size(); + prevRoundTime_ = result_->roundTime.read(); + phase_ = Phase::accepted; + impl().onForceAccept( + *result_, previousLedger_, closeResolution_, rawCloseTimes_, mode_); + JLOG(j_.info()) << "Simulation complete"; } template Json::Value -Consensus::getJson (bool full) const +Consensus::getJson(bool full) const { using std::to_string; using Int = Json::Value::Int; - Json::Value ret (Json::objectValue); + Json::Value ret(Json::objectValue); std::lock_guard _(*lock_); - ret["proposing"] = proposing_; - ret["validating"] = validating_; - ret["proposers"] = static_cast (peerProposals_.size ()); + ret["proposing"] = (mode_ == Mode::proposing); + ret["proposers"] = static_cast(peerProposals_.size()); - if (haveCorrectLCL_) + if (mode_ != Mode::wrongLedger) { ret["synched"] = true; ret["ledger_seq"] = previousLedger_.seq() + 1; @@ -974,90 +922,73 @@ Consensus::getJson (bool full) const else ret["synched"] = false; - switch (state_) - { - case State::open: - ret["state"] = "open"; - break; + ret["phase"] = Consensus::to_string(phase_); - case State::establish: - ret["state"] = "consensus"; - break; + if (result_ && !result_->disputes.empty() && !full) + ret["disputes"] = static_cast(result_->disputes.size()); - case State::processing: - ret["state"] = "processing"; - break; - - case State::accepted: - ret["state"] = "accepted"; - break; - } - - int v = disputes_.size (); - - if ((v != 0) && !full) - ret["disputes"] = v; - - if (ourPosition_) - ret["our_position"] = ourPosition_->getJson (); + if (result_) + ret["our_position"] = result_->position.getJson(); if (full) { - ret["current_ms"] = static_cast(roundTime_.count()); + if (result_) + ret["current_ms"] = + static_cast(result_->roundTime.read().count()); ret["converge_percent"] = convergePercent_; ret["close_resolution"] = static_cast(closeResolution_.count()); ret["have_time_consensus"] = haveCloseTimeConsensus_; - ret["previous_proposers"] = static_cast(previousProposers_); - ret["previous_mseconds"] = - static_cast(previousRoundTime_.count()); + ret["previous_proposers"] = static_cast(prevProposers_); + ret["previous_mseconds"] = static_cast(prevRoundTime_.count()); - if (! peerProposals_.empty ()) + if (!peerProposals_.empty()) { - Json::Value ppj (Json::objectValue); + Json::Value ppj(Json::objectValue); for (auto& pp : peerProposals_) { - ppj[to_string (pp.first)] = pp.second.getJson (); + ppj[to_string(pp.first)] = pp.second.getJson(); } ret["peer_positions"] = std::move(ppj); } - if (! acquired_.empty ()) + if (!acquired_.empty()) { - Json::Value acq (Json::arrayValue); + Json::Value acq(Json::arrayValue); for (auto& at : acquired_) { - acq.append (to_string (at.first)); + acq.append(to_string(at.first)); } ret["acquired"] = std::move(acq); } - if (! disputes_.empty ()) + if (result_ && !result_->disputes.empty()) { - Json::Value dsj (Json::objectValue); - for (auto& dt : disputes_) + Json::Value dsj(Json::objectValue); + for (auto& dt : result_->disputes) { - dsj[to_string (dt.first)] = dt.second.getJson (); + dsj[to_string(dt.first)] = dt.second.getJson(); } ret["disputes"] = std::move(dsj); } - if (! closeTimes_.empty ()) + if (!rawCloseTimes_.peers.empty()) { - Json::Value ctj (Json::objectValue); - for (auto& ct : closeTimes_) + Json::Value ctj(Json::objectValue); + for (auto& ct : rawCloseTimes_.peers) { - ctj[std::to_string(ct.first.time_since_epoch().count())] = ct.second; + ctj[std::to_string(ct.first.time_since_epoch().count())] = + ct.second; } ret["close_times"] = std::move(ctj); } - if (! deadNodes_.empty ()) + if (!deadNodes_.empty()) { - Json::Value dnj (Json::arrayValue); + Json::Value dnj(Json::arrayValue); for (auto const& dn : deadNodes_) { - dnj.append (to_string (dn)); + dnj.append(to_string(dn)); } ret["dead_nodes"] = std::move(dnj); } @@ -1066,409 +997,217 @@ Consensus::getJson (bool full) const return ret; } -// Handle a change in the LCL during a consensus round +// Handle a change in the prior ledger during a consensus round template void -Consensus::handleLCL (typename Ledger_t::ID const& lgrId) +Consensus::handleWrongLedger( + typename Ledger_t::ID const& lgrId) { - assert (lgrId != prevLedgerID_ || - previousLedger_.id() != lgrId); + assert(lgrId != prevLedgerID_ || previousLedger_.id() != lgrId); if (prevLedgerID_ != lgrId) { // first time switching to this ledger prevLedgerID_ = lgrId; - if (haveCorrectLCL_ && proposing_ && ourPosition_) + // Stop proposing because we are out of sync + leaveConsensus(); + + if (result_) { - JLOG (j_.info()) << "Bowing out of consensus"; - leaveConsensus(); + result_->disputes.clear(); + result_->compares.clear(); } - // Stop proposing because we are out of sync - proposing_ = false; - peerProposals_.clear (); - disputes_.clear (); - compares_.clear (); - closeTimes_.clear (); - deadNodes_.clear (); - // To get back in sync: - playbackProposals (); + peerProposals_.clear(); + rawCloseTimes_.peers.clear(); + deadNodes_.clear(); + + // Get back in sync, this will also recreate disputes + playbackProposals(); } if (previousLedger_.id() == prevLedgerID_) return; // we need to switch the ledger we're working from - if (auto buildLCL = impl().acquireLedger(prevLedgerID_)) + if (auto newLedger = impl().acquireLedger(prevLedgerID_)) { - JLOG (j_.info()) << "Have the consensus ledger " << prevLedgerID_; - startRoundInternal (now_, lgrId, *buildLCL); + JLOG(j_.info()) << "Have the consensus ledger " << prevLedgerID_; + startRoundInternal(now_, lgrId, *newLedger, Mode::switchedLedger); } else { - haveCorrectLCL_ = false; + mode_ = Mode::wrongLedger; } } - template void -Consensus::checkLCL () +Consensus::checkLedger() { - auto netLgr = impl().getLCL ( - prevLedgerID_, - haveCorrectLCL_ ? previousLedger_.parentID() : typename Ledger_t::ID{}, - haveCorrectLCL_); + auto netLgr = impl().getPrevLedger(prevLedgerID_, previousLedger_, mode_); if (netLgr != prevLedgerID_) { - // LCL change - const char* status; - - switch (state_) - { - case State::open: - status = "open"; - break; - - case State::establish: - status = "establish"; - break; - - case State::processing: - status = "processing"; - break; - - case State::accepted: - status = "accepted"; - break; - - default: - status = "unknown"; - } - - JLOG (j_.warn()) - << "View of consensus changed during " << status - << " status=" << status << ", " - << (haveCorrectLCL_ ? "CorrectLCL" : "IncorrectLCL"); - JLOG (j_.warn()) << prevLedgerID_ - << " to " << netLgr; - JLOG (j_.warn()) - << previousLedger_.getJson(); - handleLCL (netLgr); + JLOG(j_.warn()) << "View of consensus changed during " + << to_string(phase_) << " status=" << to_string(phase_) + << ", " + << " mode=" << to_string(mode_); + JLOG(j_.warn()) << prevLedgerID_ << " to " << netLgr; + JLOG(j_.warn()) << previousLedger_.getJson(); + handleWrongLedger(netLgr); } - else if(previousLedger_.id() != prevLedgerID_) - handleLCL(netLgr); + else if (previousLedger_.id() != prevLedgerID_) + handleWrongLedger(netLgr); } template void -Consensus::playbackProposals () +Consensus::playbackProposals() { - for (auto const & p : impl().proposals(prevLedgerID_)) + for (auto const& p : impl().proposals(prevLedgerID_)) { - if(peerProposal(now_, p)) + if (peerProposal(now_, p)) impl().relay(p); } } template void -Consensus::statePreClose () +Consensus::phaseOpen() { using namespace std::chrono; // it is shortly before ledger close time bool anyTransactions = impl().hasOpenTransactions(); - auto proposersClosed = peerProposals_.size (); + auto proposersClosed = peerProposals_.size(); auto proposersValidated = impl().proposersValidated(prevLedgerID_); - openTime_ = duration_cast(clock_.now() - openStartTime_); + openTime_.tick(clock_.now()); // This computes how long since last ledger's close time milliseconds sinceClose; { - bool previousCloseCorrect = haveCorrectLCL_ - && previousLedger_.closeAgree () - && (previousLedger_.closeTime() != - (previousLedger_.parentCloseTime() + 1s)); + bool previousCloseCorrect = (mode_ != Mode::wrongLedger) && + previousLedger_.closeAgree() && + (previousLedger_.closeTime() != + (previousLedger_.parentCloseTime() + 1s)); auto lastCloseTime = previousCloseCorrect - ? previousLedger_.closeTime() // use consensus timing - : closeTime_; // use the time we saw internally + ? previousLedger_.closeTime() // use consensus timing + : prevCloseTime_; // use the time we saw internally - if (now_ >= lastCloseTime ) + if (now_ >= lastCloseTime) sinceClose = duration_cast(now_ - lastCloseTime); else - sinceClose = -duration_cast(lastCloseTime - now_); + sinceClose = -duration_cast(lastCloseTime - now_); } - auto const idleInterval = std::max(LEDGER_IDLE_INTERVAL, + auto const idleInterval = std::max( + LEDGER_IDLE_INTERVAL, duration_cast(2 * previousLedger_.closeTimeResolution())); // Decide if we should close the ledger - if (shouldCloseLedger (anyTransactions - , previousProposers_, proposersClosed, proposersValidated - , previousRoundTime_, sinceClose, openTime_ - , idleInterval, j_)) + if (shouldCloseLedger( + anyTransactions, + prevProposers_, + proposersClosed, + proposersValidated, + prevRoundTime_, + sinceClose, + openTime_.read(), + idleInterval, + j_)) { - closeLedger (); + closeLedger(); } } template void -Consensus::stateEstablish () +Consensus::phaseEstablish() { - using namespace std::chrono; - roundTime_ = duration_cast - (clock_.now() - consensusStartTime_); + // can only establish consensus if we already took a stance + assert(result_); - convergePercent_ = roundTime_ * 100 / - std::max ( - previousRoundTime_, AV_MIN_CONSENSUS_TIME); + using namespace std::chrono; + result_->roundTime.tick(clock_.now()); + + convergePercent_ = result_->roundTime.read() * 100 / + std::max(prevRoundTime_, AV_MIN_CONSENSUS_TIME); // Give everyone a chance to take an initial position - if (roundTime_ < LEDGER_MIN_CONSENSUS) + if (result_->roundTime.read() < LEDGER_MIN_CONSENSUS) return; - updateOurPositions (); + updateOurPositions(); // Nothing to do if we don't have consensus. - if (!haveConsensus ()) + if (!haveConsensus()) return; if (!haveCloseTimeConsensus_) { - JLOG (j_.info()) << - "We have TX consensus but not CT consensus"; + JLOG(j_.info()) << "We have TX consensus but not CT consensus"; return; } - JLOG (j_.info()) << - "Converge cutoff (" << peerProposals_.size () << " participants)"; - state_ = State::processing; - beginAccept (false); -} - - -template -void -Consensus::closeLedger () -{ - state_ = State::establish; - consensusStartTime_ = clock_.now (); - closeTime_ = now_; - - impl().onClose (previousLedger_, haveCorrectLCL_); - - takeInitialPosition (); + JLOG(j_.info()) << "Converge cutoff (" << peerProposals_.size() + << " participants)"; + prevProposers_ = peerProposals_.size(); + prevRoundTime_ = result_->roundTime.read(); + phase_ = Phase::accepted; + impl().onAccept( + *result_, previousLedger_, closeResolution_, rawCloseTimes_, mode_); } template void -Consensus::takeInitialPosition() +Consensus::closeLedger() { - auto pair = impl().makeInitialPosition(previousLedger_, proposing_, - haveCorrectLCL_, closeTime_, now_ ); - auto const& initialSet = pair.first; - auto const& initialPos = pair.second; - assert (initialSet.id() == initialPos.position()); + // We should not be closing if we already have a position + assert(!result_); - ourPosition_ = initialPos; - ourSet_ = initialSet; + phase_ = Phase::establish; + rawCloseTimes_.self = now_; + result_.emplace(impl().onClose(previousLedger_, now_, mode_)); + result_->roundTime.reset(clock_.now()); + // Share the newly created transaction set if we haven't already + // received it from a peer + if (acquired_.emplace(result_->set.id(), result_->set).second) + impl().relay(result_->set); - // FIXME: Is it possible to have any disputes before a position is taken? - for (auto& it : disputes_) + if (mode_ == Mode::proposing) + impl().propose(result_->position); + + // Create disputes with any peer positions we have transactions for + for (auto const& p : peerProposals_) { - it.second.setOurVote (initialSet.exists (it.first)); - } - - // When we take our initial position, - // we need to create any disputes required by our position - // and any peers who have already taken positions - compares_.emplace (initialSet.id()); - for (auto& it : peerProposals_) - { - auto pos = it.second.position(); - auto iit (acquired_.find (pos)); - if (iit != acquired_.end ()) + auto pos = p.second.position(); + auto it = acquired_.find(pos); + if (it != acquired_.end()) { - if (compares_.emplace (pos).second) - createDisputes (initialSet, iit->second); + createDisputes(it->second); } } - - gotTxSetInternal (initialSet, false); - - if (proposing_) - impl().propose (*ourPosition_); -} - -template -void -Consensus::gotTxSetInternal ( - TxSet_t const& txSet, - bool acquired) -{ - auto const hash = txSet.id(); - - if (acquired_.find (hash) != acquired_.end()) - return; - - if (acquired) - { - JLOG (j_.trace()) << "We have acquired txs " << hash; - } - - // We now have a txSet that we did not have before - - if (! acquired) - { - // If we generated this locally, - // put the txSet where others can get it - // If we acquired it, it's already shared - impl().share (txSet); - } - - if (! ourPosition_) - { - JLOG (j_.debug()) - << "Not creating disputes: no position yet."; - } - else if (ourPosition_->isBowOut ()) - { - JLOG (j_.warn()) - << "Not creating disputes: not participating."; - } - else if (hash == ourPosition_->position ()) - { - JLOG (j_.debug()) - << "Not creating disputes: identical position."; - } - else - { - // Our position is not the same as the acquired position - // create disputed txs if needed - createDisputes (*ourSet_, txSet); - compares_.insert(hash); - } - - // Adjust tracking for each peer that takes this position - std::vector peers; - for (auto& it : peerProposals_) - { - if (it.second.position () == hash) - peers.push_back (it.second.nodeID ()); - } - - if (!peers.empty ()) - { - adjustCount (txSet, peers); - } - else if (acquired) - { - JLOG (j_.warn()) - << "By the time we got the map " << hash - << " no peers were proposing it"; - } - - acquired_.emplace (hash, txSet); -} - -template -void Consensus::createDisputes ( - TxSet_t const& m1, - TxSet_t const& m2) -{ - if (m1.id() == m2.id()) - return; - - JLOG (j_.debug()) << "createDisputes " - << m1.id() << " to " << m2.id(); - auto differences = m1.compare (m2); - - int dc = 0; - // for each difference between the transactions - for (auto& id : differences) - { - ++dc; - // create disputed transactions (from the ledger that has them) - assert ( - (id.second && m1.find(id.first) && !m2.find(id.first)) || - (!id.second && !m1.find(id.first) && m2.find(id.first)) - ); - if (id.second) - addDisputedTransaction (*m1.find (id.first)); - else - addDisputedTransaction (*m2.find (id.first)); - } - JLOG (j_.debug()) << dc << " differences found"; -} - -template -void Consensus::addDisputedTransaction ( - Tx_t const& tx) -{ - auto txID = tx.id(); - - if (disputes_.find (txID) != disputes_.end ()) - return; - - JLOG (j_.debug()) << "Transaction " - << txID << " is disputed"; - - bool ourVote = false; - - // Update our vote on the disputed transaction - if (ourSet_) - ourVote = ourSet_->exists (txID); - - Dispute_t dtx {tx, ourVote, j_}; - - // Update all of the peer's votes on the disputed transaction - for (auto& pit : peerProposals_) - { - auto cit (acquired_.find (pit.second.position ())); - - if (cit != acquired_.end ()) - dtx.setVote (pit.first, - cit->second.exists (txID)); - } - - impl().relay(dtx); - - disputes_.emplace (txID, std::move (dtx)); -} - -template -void Consensus::adjustCount (TxSet_t const& txSet, - std::vector const& peers) -{ - for (auto& it : disputes_) - { - bool setHas = txSet.exists (it.first); - for (auto const& pit : peers) - it.second.setVote (pit, setHas); - } } /** How many of the participants must agree to reach a given threshold? - Note that the number may not precisely yield the requested percentage. - For example, with with size = 5 and percent = 70, we return 3, but - 3 out of 5 works out to 60%. There are no security implications to - this. +Note that the number may not precisely yield the requested percentage. +For example, with with size = 5 and percent = 70, we return 3, but +3 out of 5 works out to 60%. There are no security implications to +this. - @param participants The number of participants (i.e. validators) - @param percent The percent that we want to reach +@param participants The number of participants (i.e. validators) +@param percent The percent that we want to reach - @return the number of participants which must agree +@return the number of participants which must agree */ inline int -participantsNeeded (int participants, int percent) +participantsNeeded(int participants, int percent) { int result = ((participants * percent) + (percent / 2)) / 100; @@ -1476,126 +1215,133 @@ participantsNeeded (int participants, int percent) } template -void Consensus::updateOurPositions () +void +Consensus::updateOurPositions() { + // We must have a position if we are updating it + assert(result_); + // Compute a cutoff time auto const peerCutoff = now_ - PROPOSE_FRESHNESS; auto const ourCutoff = now_ - PROPOSE_INTERVAL; // Verify freshness of peer positions and compute close times - std::map closeTimes; + std::map effCloseTimes; { - auto it = peerProposals_.begin (); - while (it != peerProposals_.end ()) + auto it = peerProposals_.begin(); + while (it != peerProposals_.end()) { - if (it->second.isStale (peerCutoff)) + if (it->second.isStale(peerCutoff)) { // peer's proposal is stale, so remove it - auto const& peerID = it->second.nodeID (); - JLOG (j_.warn()) - << "Removing stale proposal from " << peerID; - for (auto& dt : disputes_) - dt.second.unVote (peerID); - it = peerProposals_.erase (it); + auto const& peerID = it->second.nodeID(); + JLOG(j_.warn()) << "Removing stale proposal from " << peerID; + for (auto& dt : result_->disputes) + dt.second.unVote(peerID); + it = peerProposals_.erase(it); } else { // proposal is still fresh - ++closeTimes[effectiveCloseTime(it->second.closeTime(), - closeResolution_, previousLedger_.closeTime())]; + ++effCloseTimes[effCloseTime( + it->second.closeTime(), + closeResolution_, + previousLedger_.closeTime())]; ++it; } } } // This will stay unseated unless there are any changes - boost::optional ourNewSet; + boost::optional ourNewSet; // Update votes on disputed transactions { boost::optional mutableSet; - for (auto& it : disputes_) + for (auto& it : result_->disputes) { - // Because the threshold for inclusion increases, // time can change our position on a dispute - if (it.second.updateVote (convergePercent_, proposing_)) + if (it.second.updateVote( + convergePercent_, (mode_ == Mode::proposing))) { if (!mutableSet) - mutableSet.emplace(*ourSet_); + mutableSet.emplace(result_->set); - if (it.second.getOurVote ()) + if (it.second.getOurVote()) { // now a yes - mutableSet->insert (it.second.tx()); + mutableSet->insert(it.second.tx()); } else { // now a no - mutableSet->erase (it.first); + mutableSet->erase(it.first); } } } - if(mutableSet) + if (mutableSet) ourNewSet.emplace(*mutableSet); } - int neededWeight; - - if (convergePercent_ < AV_MID_CONSENSUS_TIME) - neededWeight = AV_INIT_CONSENSUS_PCT; - else if (convergePercent_ < AV_LATE_CONSENSUS_TIME) - neededWeight = AV_MID_CONSENSUS_PCT; - else if (convergePercent_ < AV_STUCK_CONSENSUS_TIME) - neededWeight = AV_LATE_CONSENSUS_PCT; - else - neededWeight = AV_STUCK_CONSENSUS_PCT; - - NetClock::time_point closeTime = {}; + NetClock::time_point consensusCloseTime = {}; haveCloseTimeConsensus_ = false; - if (peerProposals_.empty ()) + if (peerProposals_.empty()) { // no other times haveCloseTimeConsensus_ = true; - closeTime = effectiveCloseTime(ourPosition_->closeTime(), - closeResolution_, previousLedger_.closeTime()); + consensusCloseTime = effCloseTime( + result_->position.closeTime(), + closeResolution_, + previousLedger_.closeTime()); } else { - int participants = peerProposals_.size (); - if (proposing_) + int neededWeight; + + if (convergePercent_ < AV_MID_CONSENSUS_TIME) + neededWeight = AV_INIT_CONSENSUS_PCT; + else if (convergePercent_ < AV_LATE_CONSENSUS_TIME) + neededWeight = AV_MID_CONSENSUS_PCT; + else if (convergePercent_ < AV_STUCK_CONSENSUS_TIME) + neededWeight = AV_LATE_CONSENSUS_PCT; + else + neededWeight = AV_STUCK_CONSENSUS_PCT; + + int participants = peerProposals_.size(); + if (mode_ == Mode::proposing) { - ++closeTimes[effectiveCloseTime(ourPosition_->closeTime(), - closeResolution_, previousLedger_.closeTime())]; + ++effCloseTimes[effCloseTime( + result_->position.closeTime(), + closeResolution_, + previousLedger_.closeTime())]; ++participants; } // Threshold for non-zero vote - int threshVote = participantsNeeded (participants, - neededWeight); + int threshVote = participantsNeeded(participants, neededWeight); // Threshold to declare consensus - int const threshConsensus = participantsNeeded ( - participants, AV_CT_CONSENSUS_PCT); + int const threshConsensus = + participantsNeeded(participants, AV_CT_CONSENSUS_PCT); - JLOG (j_.info()) << "Proposers:" - << peerProposals_.size () << " nw:" << neededWeight - << " thrV:" << threshVote << " thrC:" << threshConsensus; + JLOG(j_.info()) << "Proposers:" << peerProposals_.size() + << " nw:" << neededWeight << " thrV:" << threshVote + << " thrC:" << threshConsensus; - for (auto const& it : closeTimes) + for (auto const& it : effCloseTimes) { - JLOG (j_.debug()) << "CCTime: seq " - << previousLedger_.seq() + 1 << ": " - << it.first.time_since_epoch().count() - << " has " << it.second << ", " - << threshVote << " required"; + JLOG(j_.debug()) + << "CCTime: seq " << previousLedger_.seq() + 1 << ": " + << it.first.time_since_epoch().count() << " has " << it.second + << ", " << threshVote << " required"; if (it.second >= threshVote) { // A close time has enough votes for us to try to agree - closeTime = it.first; + consensusCloseTime = it.first; threshVote = it.second; if (threshVote >= threshConsensus) @@ -1605,114 +1351,103 @@ void Consensus::updateOurPositions () if (!haveCloseTimeConsensus_) { - JLOG (j_.debug()) << "No CT consensus:" - << " Proposers:" << peerProposals_.size () - << " Proposing:" << (proposing_ ? "yes" : "no") - << " Thresh:" << threshConsensus - << " Pos:" << closeTime.time_since_epoch().count(); + JLOG(j_.debug()) + << "No CT consensus:" + << " Proposers:" << peerProposals_.size() + << " Mode:" << to_string(mode_) << " Thresh:" << threshConsensus + << " Pos:" << consensusCloseTime.time_since_epoch().count(); } } - // Temporarily send a new proposal if there's any change to our - // claimed close time. Once the new close time code is deployed - // to the full network, this can be relaxed to force a change - // only if the rounded close time has changed. - if (! ourNewSet && - ((closeTime != ourPosition_->closeTime()) - || ourPosition_->isStale (ourCutoff))) + if (!ourNewSet && + ((consensusCloseTime != + effCloseTime( + result_->position.closeTime(), + closeResolution_, + previousLedger_.closeTime())) || + result_->position.isStale(ourCutoff))) { // close time changed or our position is stale - ourNewSet.emplace (*ourSet_); + ourNewSet.emplace(result_->set); } if (ourNewSet) { - auto newHash = ourNewSet->id(); + auto newID = ourNewSet->id(); - // Setting ourSet_ here prevents gotTxSetInternal - // from checking for new disputes. But we only changed - // positions on existing disputes, so no need to. - ourSet_ = ourNewSet; + result_->set = std::move(*ourNewSet); - JLOG (j_.info()) - << "Position change: CTime " - << closeTime.time_since_epoch().count() - << ", tx " << newHash; + JLOG(j_.info()) << "Position change: CTime " + << consensusCloseTime.time_since_epoch().count() + << ", tx " << newID; - if (ourPosition_->changePosition ( - newHash, closeTime, now_)) + result_->position.changePosition(newID, consensusCloseTime, now_); + if (!result_->position.isBowOut()) { - if (proposing_) - impl().propose (*ourPosition_); + // Share our new transaction set if we haven't already received + // it from a peer + if (acquired_.emplace(newID, result_->set).second) + impl().relay(result_->set); - gotTxSetInternal (*ourNewSet, false); + if (mode_ == Mode::proposing) + impl().propose(result_->position); } } } template bool -Consensus::haveConsensus () +Consensus::haveConsensus() { + // Must have a stance if we are checking for consensus + assert(result_); + // CHECKME: should possibly count unacquired TX sets as disagreeing int agree = 0, disagree = 0; - auto ourPosition = ourPosition_->position (); + + auto ourPosition = result_->position.position(); // Count number of agreements/disagreements with our position for (auto& it : peerProposals_) { - if (it.second.isBowOut ()) - continue; - - if (it.second.position () == ourPosition) + if (it.second.position() == ourPosition) { ++agree; } else { - using std::to_string; - JLOG (j_.debug()) << to_string (it.first) - << " has " << to_string (it.second.position ()); + JLOG(j_.debug()) << to_string(it.first) << " has " + << to_string(it.second.position()); ++disagree; - if (compares_.count(it.second.position()) == 0) - { // Make sure we have generated disputes - auto hash = it.second.position(); - JLOG (j_.debug()) - << "We have not compared to " << hash; - auto it1 = acquired_.find (hash); - auto it2 = acquired_.find(ourPosition_->position ()); - if ((it1 != acquired_.end()) && (it2 != acquired_.end())) - { - compares_.insert(hash); - createDisputes(it2->second, it1->second); - } - } } } auto currentFinished = impl().proposersFinished(prevLedgerID_); - JLOG (j_.debug()) - << "Checking for TX consensus: agree=" << agree - << ", disagree=" << disagree; + JLOG(j_.debug()) << "Checking for TX consensus: agree=" << agree + << ", disagree=" << disagree; // Determine if we actually have consensus or not - auto ret = checkConsensus (previousProposers_, agree + disagree, agree, - currentFinished, previousRoundTime_, roundTime_, proposing_, + result_->state = checkConsensus( + prevProposers_, + agree + disagree, + agree, + currentFinished, + prevRoundTime_, + result_->roundTime.read(), + mode_ == Mode::proposing, j_); - if (ret == ConsensusState::No) + if (result_->state == ConsensusState::No) return false; // There is consensus, but we need to track if the network moved on // without us. - consensusFail_ = (ret == ConsensusState::MovedOn); - - if (consensusFail_) + if (result_->state == ConsensusState::MovedOn) { - JLOG (j_.error()) << "Unable to reach consensus"; - JLOG (j_.error()) << getJson(true); + JLOG(j_.error()) << "Unable to reach consensus"; + JLOG(j_.error()) << getJson(true); } return true; @@ -1720,75 +1455,132 @@ Consensus::haveConsensus () template void -Consensus::beginAccept (bool synchronous) +Consensus::leaveConsensus() { - if (! ourPosition_ || ! ourSet_) + if (mode_ == Mode::proposing) { - JLOG (j_.fatal()) - << "We don't have a consensus set"; - abort (); - } + if (result_ && !result_->position.isBowOut()) + { + result_->position.bowOut(now_); + impl().propose(result_->position); + } - previousProposers_ = peerProposals_.size(); - previousRoundTime_ = roundTime_; - - if (synchronous) - accept (*ourSet_); - else - { - impl().dispatchAccept(*ourSet_); + mode_ = Mode::observing; + JLOG(j_.info()) << "Bowing out of consensus"; } } - template void -Consensus::accept (TxSet_t const& set) +Consensus::createDisputes(TxSet_t const& o) { + // Cannot create disputes without our stance + assert(result_); - bool validatingOut = impl().accept(set, - ourPosition_->closeTime(), - proposing_, - validating_, - haveCorrectLCL_, - consensusFail_, - prevLedgerID_, - previousLedger_, - closeResolution_, - now_, - roundTime_, - disputes_, - closeTimes_, - closeTime_ - ); + // Only create disputes if this is a new set + if (!result_->compares.emplace(o.id()).second) + return; - // we have accepted a new ledger - bool correct; + // Nothing to dispute if we agree + if (result_->set.id() == o.id()) + return; + + JLOG(j_.debug()) << "createDisputes " << result_->set.id() << " to " + << o.id(); + + auto differences = result_->set.compare(o); + + int dc = 0; + + for (auto& id : differences) { - std::lock_guard _(*lock_); - validating_ = validatingOut; - state_ = State::accepted; - correct = haveCorrectLCL_; + ++dc; + // create disputed transactions (from the ledger that has them) + assert( + (id.second && result_->set.find(id.first) && !o.find(id.first)) || + (!id.second && !result_->set.find(id.first) && o.find(id.first))); + + Tx_t tx = id.second ? *result_->set.find(id.first) : *o.find(id.first); + auto txID = tx.id(); + + if (result_->disputes.find(txID) != result_->disputes.end()) + continue; + + JLOG(j_.debug()) << "Transaction " << txID << " is disputed"; + + typename Result::Dispute_t dtx{tx, result_->set.exists(txID), j_}; + + // Update all of the available peer's votes on the disputed transaction + for (auto& pit : peerProposals_) + { + auto cit(acquired_.find(pit.second.position())); + + if (cit != acquired_.end()) + dtx.setVote(pit.first, cit->second.exists(txID)); + } + impl().relay(dtx.tx()); + + result_->disputes.emplace(txID, std::move(dtx)); } - - impl().endConsensus (correct); + JLOG(j_.debug()) << dc << " differences found"; } - - template void -Consensus::leaveConsensus () +Consensus::updateDisputes( + NodeID_t const& node, + TxSet_t const& other) { - if (ourPosition_ && ! ourPosition_->isBowOut ()) + // Cannot updateDisputes without our stance + assert(result_); + + // Ensure we have created disputes against this set if we haven't seen + // it before + if (result_->compares.find(other.id()) == result_->compares.end()) + createDisputes(other); + + for (auto& it : result_->disputes) { - ourPosition_->bowOut(now_); - if(proposing_) - impl().propose(*ourPosition_); + auto& d = it.second; + d.setVote(node, other.exists(d.tx().id())); } - proposing_ = false; } -} // ripple +template +std::string +Consensus::to_string(Phase p) +{ + switch (p) + { + case Phase::open: + return "open"; + case Phase::establish: + return "establish"; + case Phase::accepted: + return "accepted"; + default: + return "unknown"; + } +} + +template +std::string +Consensus::to_string(Mode m) +{ + switch (m) + { + case Mode::proposing: + return "proposing"; + case Mode::observing: + return "observing"; + case Mode::wrongLedger: + return "wrongLedger"; + case Mode::switchedLedger: + return "switchedLedger"; + default: + return "unknown"; + } +} +} // ripple #endif diff --git a/src/ripple/consensus/ConsensusProposal.h b/src/ripple/consensus/ConsensusProposal.h index 35f9a8705..e39930914 100644 --- a/src/ripple/consensus/ConsensusProposal.h +++ b/src/ripple/consensus/ConsensusProposal.h @@ -1,7 +1,7 @@ //------------------------------------------------------------------------------ /* This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012-2016 Ripple Labs Inc. + Copyright (c) 2012-2017 Ripple Labs Inc. Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above @@ -19,13 +19,12 @@ #ifndef RIPPLE_CONSENSUS_ConsensusProposal_H_INCLUDED #define RIPPLE_CONSENSUS_ConsensusProposal_H_INCLUDED -#include +#include #include #include -#include +#include -namespace ripple -{ +namespace ripple { /** Represents a proposed position taken during a round of consensus. During consensus, peers seek agreement on a set of transactions to @@ -49,10 +48,7 @@ namespace ripple @tparam Position_t Type used to represent the position taken on transactions under consideration during this round of consensus */ -template < - class NodeID_t, - class LedgerID_t, - class Position_t> +template class ConsensusProposal { public: @@ -64,7 +60,6 @@ public: //< Sequence number when a peer wants to bow out and leave consensus static std::uint32_t const seqLeave = 0xffffffff; - /** Constructor @param prevLedger The previous ledger this proposal is building on. @@ -81,33 +76,32 @@ public: NetClock::time_point closeTime, NetClock::time_point now, NodeID_t const& nodeID) - : previousLedger_(prevLedger) - , position_(position) - , closeTime_(closeTime) - , time_(now) - , proposeSeq_(seq) - , nodeID_(nodeID) + : previousLedger_(prevLedger) + , position_(position) + , closeTime_(closeTime) + , time_(now) + , proposeSeq_(seq) + , nodeID_(nodeID) { - } //! Identifying which peer took this position. NodeID_t const& - nodeID () const + nodeID() const { return nodeID_; } //! Get the proposed position. Position_t const& - position () const + position() const { return position_; } //! Get the prior accepted ledger this position is based on. LedgerID_t const& - prevLedger () const + prevLedger() const { return previousLedger_; } @@ -120,21 +114,21 @@ public: @return the sequence number */ std::uint32_t - proposeSeq () const + proposeSeq() const { return proposeSeq_; } //! The current position on the consensus close time. - NetClock::time_point const & - closeTime () const + NetClock::time_point const& + closeTime() const { return closeTime_; } //! Get when this position was taken. - NetClock::time_point const & - seenTime () const + NetClock::time_point const& + seenTime() const { return time_; } @@ -143,49 +137,43 @@ public: consensus round. */ bool - isInitial () const + isInitial() const { return proposeSeq_ == seqJoin; } //! Get whether this node left the consensus process bool - isBowOut () const + isBowOut() const { return proposeSeq_ == seqLeave; } //! Get whether this position is stale relative to the provided cutoff bool - isStale (NetClock::time_point cutoff) const + isStale(NetClock::time_point cutoff) const { return time_ <= cutoff; } /** Update the position during the consensus process. This will increment - the proposal's sequence number. + the proposal's sequence number if it has not already bowed out. @param newPosition The new position taken. @param newCloseTime The new close time. - @param now the time The new position was taken. - - @return `true` if the position was updated or `false` if this node has - already left this consensus round. + @param now the time The new position was taken */ - bool + void changePosition( Position_t const& newPosition, NetClock::time_point newCloseTime, NetClock::time_point now) { - if (proposeSeq_ == seqLeave) - return false; - - position_ = newPosition; - closeTime_ = newCloseTime; - time_ = now; - ++proposeSeq_; - return true; + position_ = newPosition; + closeTime_ = newCloseTime; + time_ = now; + if (proposeSeq_ != seqLeave) + ++proposeSeq_; } /** Leave consensus @@ -197,32 +185,32 @@ public: void bowOut(NetClock::time_point now) { - time_ = now; - proposeSeq_ = seqLeave; + time_ = now; + proposeSeq_ = seqLeave; } //! Get JSON representation for debugging Json::Value - getJson () const + getJson() const { using std::to_string; Json::Value ret = Json::objectValue; - ret[jss::previous_ledger] = to_string (prevLedger()); + ret[jss::previous_ledger] = to_string(prevLedger()); if (!isBowOut()) { - ret[jss::transaction_hash] = to_string (position()); + ret[jss::transaction_hash] = to_string(position()); ret[jss::propose_seq] = proposeSeq(); } - ret[jss::close_time] = to_string(closeTime().time_since_epoch().count()); + ret[jss::close_time] = + to_string(closeTime().time_since_epoch().count()); return ret; } private: - //! Unique identifier of prior ledger this proposal is based on LedgerID_t previousLedger_; @@ -240,23 +228,17 @@ private: //! The identifier of the node taking this position NodeID_t nodeID_; - }; -template +template bool -operator==(ConsensusProposal const & a, - ConsensusProposal const & b) +operator==( + ConsensusProposal const& a, + ConsensusProposal const& b) { - return a.nodeID() == b.nodeID() && - a.proposeSeq() == b.proposeSeq() && - a.prevLedger() == b.prevLedger() && - a.position() == b.position() && - a.closeTime() == b.closeTime() && - a.seenTime() == b.seenTime(); + return a.nodeID() == b.nodeID() && a.proposeSeq() == b.proposeSeq() && + a.prevLedger() == b.prevLedger() && a.position() == b.position() && + a.closeTime() == b.closeTime() && a.seenTime() == b.seenTime(); } - } #endif diff --git a/src/ripple/consensus/DisputedTx.h b/src/ripple/consensus/DisputedTx.h index e4ab7fd69..41e3d8ebe 100644 --- a/src/ripple/consensus/DisputedTx.h +++ b/src/ripple/consensus/DisputedTx.h @@ -17,15 +17,15 @@ */ //============================================================================== -#ifndef RIPPLE_APP_CONSENSUS_DISPUTEDTX_H_INCLUDED -#define RIPPLE_APP_CONSENSUS_DISPUTEDTX_H_INCLUDED +#ifndef RIPPLE_APP_CONSENSUS_IMPL_DISPUTEDTX_H_INCLUDED +#define RIPPLE_APP_CONSENSUS_IMPL_DISPUTEDTX_H_INCLUDED -#include -#include +#include #include #include #include -#include +#include +#include #include namespace ripple { @@ -47,7 +47,8 @@ namespace ripple { template class DisputedTx { - using TxID_t = typename Tx_t::ID; + using TxID_t = typename Tx_t::ID; + public: /** Constructor @@ -55,41 +56,35 @@ public: @param ourVote Our vote on whether tx should be included @param j Journal for debugging */ - DisputedTx (Tx_t const& tx, - bool ourVote, - beast::Journal j) - : yays_ (0) - , nays_ (0) - , ourVote_ (ourVote) - , tx_ (tx) - , j_ (j) + DisputedTx(Tx_t const& tx, bool ourVote, beast::Journal j) + : yays_(0), nays_(0), ourVote_(ourVote), tx_(tx), j_(j) { } //! The unique id/hash of the disputed transaction. - TxID_t - const& ID () const + TxID_t const& + ID() const { return tx_.id(); } //! Our vote on whether the transaction should be included. bool - getOurVote () const + getOurVote() const { return ourVote_; } //! The disputed transaction. - Tx_t - const& tx () const + Tx_t const& + tx() const { return tx_; } //! Change our vote void - setOurVote (bool o) + setOurVote(bool o) { ourVote_ = o; } @@ -100,14 +95,14 @@ public: @param votesYes Whether peer votes to include the disputed transaction. */ void - setVote (NodeID_t const& peer, bool votesYes); + setVote(NodeID_t const& peer, bool votesYes); /** Remove a peer's vote @param peer Identifier of peer. */ void - unVote (NodeID_t const& peer); + unVote(NodeID_t const& peer); /** Update our vote given progression of consensus. @@ -120,49 +115,47 @@ public: @return Whether our vote changed */ bool - updateVote (int percentTime, bool proposing); + updateVote(int percentTime, bool proposing); //! JSON representation of dispute, used for debugging Json::Value - getJson () const; + getJson() const; private: - int yays_; //< Number of yes votes - int nays_; //< Number of no votes + int yays_; //< Number of yes votes + int nays_; //< Number of no votes bool ourVote_; //< Our vote (true is yes) - Tx_t tx_; //< Transaction under dispute + Tx_t tx_; //< Transaction under dispute - hash_map votes_; //< Votes of our peers - beast::Journal j_; //< Debug journal + hash_map votes_; //< Votes of our peers + beast::Journal j_; //< Debug journal }; // Track a peer's yes/no vote on a particular disputed tx_ template -void DisputedTx::setVote (NodeID_t const& peer, bool votesYes) +void +DisputedTx::setVote(NodeID_t const& peer, bool votesYes) { - auto res = votes_.insert (std::make_pair (peer, votesYes)); + auto res = votes_.insert(std::make_pair(peer, votesYes)); // new vote if (res.second) { if (votesYes) { - JLOG (j_.debug()) - << "Peer " << peer << " votes YES on " << tx_.id(); + JLOG(j_.debug()) << "Peer " << peer << " votes YES on " << tx_.id(); ++yays_; } else { - JLOG (j_.debug()) - << "Peer " << peer << " votes NO on " << tx_.id(); + JLOG(j_.debug()) << "Peer " << peer << " votes NO on " << tx_.id(); ++nays_; } } // changes vote to yes else if (votesYes && !res.first->second) { - JLOG (j_.debug()) - << "Peer " << peer << " now votes YES on " << tx_.id(); + JLOG(j_.debug()) << "Peer " << peer << " now votes YES on " << tx_.id(); --nays_; ++yays_; res.first->second = true; @@ -170,8 +163,7 @@ void DisputedTx::setVote (NodeID_t const& peer, bool votesYes) // changes vote to no else if (!votesYes && res.first->second) { - JLOG (j_.debug()) - << "Peer " << peer << " now votes NO on " << tx_.id(); + JLOG(j_.debug()) << "Peer " << peer << " now votes NO on " << tx_.id(); ++nays_; --yays_; res.first->second = false; @@ -180,23 +172,25 @@ void DisputedTx::setVote (NodeID_t const& peer, bool votesYes) // Remove a peer's vote on this disputed transasction template -void DisputedTx::unVote (NodeID_t const& peer) +void +DisputedTx::unVote(NodeID_t const& peer) { - auto it = votes_.find (peer); + auto it = votes_.find(peer); - if (it != votes_.end ()) + if (it != votes_.end()) { if (it->second) --yays_; else --nays_; - votes_.erase (it); + votes_.erase(it); } } template -bool DisputedTx::updateVote (int percentTime, bool proposing) +bool +DisputedTx::updateVote(int percentTime, bool proposing) { if (ourVote_ && (nays_ == 0)) return false; @@ -207,7 +201,7 @@ bool DisputedTx::updateVote (int percentTime, bool proposing) bool newPosition; int weight; - if (proposing) // give ourselves full weight + if (proposing) // give ourselves full weight { // This is basically the percentage of nodes voting 'yes' (including us) weight = (yays_ * 100 + (ourVote_ ? 100 : 0)) / (nays_ + yays_ + 1); @@ -219,7 +213,7 @@ bool DisputedTx::updateVote (int percentTime, bool proposing) // To prevent avalanche stalls, we increase the needed weight slightly // over time. if (percentTime < AV_MID_CONSENSUS_TIME) - newPosition = weight > AV_INIT_CONSENSUS_PCT; + newPosition = weight > AV_INIT_CONSENSUS_PCT; else if (percentTime < AV_LATE_CONSENSUS_TIME) newPosition = weight > AV_MID_CONSENSUS_PCT; else if (percentTime < AV_STUCK_CONSENSUS_TIME) @@ -236,43 +230,43 @@ bool DisputedTx::updateVote (int percentTime, bool proposing) if (newPosition == ourVote_) { - JLOG (j_.info()) - << "No change (" << (ourVote_ ? "YES" : "NO") << ") : weight " - << weight << ", percent " << percentTime; - JLOG (j_.debug()) << getJson (); + JLOG(j_.info()) << "No change (" << (ourVote_ ? "YES" : "NO") + << ") : weight " << weight << ", percent " + << percentTime; + JLOG(j_.debug()) << getJson(); return false; } ourVote_ = newPosition; - JLOG (j_.debug()) - << "We now vote " << (ourVote_ ? "YES" : "NO") - << " on " << tx_.id(); - JLOG (j_.debug()) << getJson (); + JLOG(j_.debug()) << "We now vote " << (ourVote_ ? "YES" : "NO") << " on " + << tx_.id(); + JLOG(j_.debug()) << getJson(); return true; } template -Json::Value DisputedTx::getJson () const +Json::Value +DisputedTx::getJson() const { using std::to_string; - Json::Value ret (Json::objectValue); + Json::Value ret(Json::objectValue); ret["yays"] = yays_; ret["nays"] = nays_; ret["our_vote"] = ourVote_; - if (!votes_.empty ()) + if (!votes_.empty()) { - Json::Value votesj (Json::objectValue); + Json::Value votesj(Json::objectValue); for (auto& vote : votes_) - votesj[to_string (vote.first)] = vote.second; - ret["votes"] = std::move (votesj); + votesj[to_string(vote.first)] = vote.second; + ret["votes"] = std::move(votesj); } return ret; } -} // ripple +} // ripple #endif diff --git a/src/ripple/consensus/LedgerTiming.cpp b/src/ripple/consensus/LedgerTiming.cpp index d2054f644..fe069a2a5 100644 --- a/src/ripple/consensus/LedgerTiming.cpp +++ b/src/ripple/consensus/LedgerTiming.cpp @@ -18,66 +18,64 @@ //============================================================================== #include -#include #include +#include #include #include namespace ripple { bool -shouldCloseLedger ( +shouldCloseLedger( bool anyTransactions, - std::size_t previousProposers, + std::size_t prevProposers, std::size_t proposersClosed, std::size_t proposersValidated, - std::chrono::milliseconds previousTime, - std::chrono::milliseconds currentTime, // Time since last ledger's close time - std::chrono::milliseconds openTime, // Time waiting to close this ledger + std::chrono::milliseconds prevRoundTime, + std::chrono::milliseconds + timeSincePrevClose, // Time since last ledger's close time + std::chrono::milliseconds openTime, // Time waiting to close this ledger std::chrono::seconds idleInterval, beast::Journal j) { using namespace std::chrono_literals; - if ((previousTime < -1s) || (previousTime > 10min) || - (currentTime > 10min)) + if ((prevRoundTime < -1s) || (prevRoundTime > 10min) || (timeSincePrevClose > 10min)) { // These are unexpected cases, we just close the ledger - JLOG (j.warn()) << - "shouldCloseLedger Trans=" << (anyTransactions ? "yes" : "no") << - " Prop: " << previousProposers << "/" << proposersClosed << - " Secs: " << currentTime.count() << " (last: " << - previousTime.count() << ")"; + JLOG(j.warn()) << "shouldCloseLedger Trans=" + << (anyTransactions ? "yes" : "no") + << " Prop: " << prevProposers << "/" << proposersClosed + << " Secs: " << timeSincePrevClose.count() + << " (last: " << prevRoundTime.count() << ")"; return true; } - if ((proposersClosed + proposersValidated) > (previousProposers / 2)) + if ((proposersClosed + proposersValidated) > (prevProposers / 2)) { // If more than half of the network has closed, we close - JLOG (j.trace()) << "Others have closed"; + JLOG(j.trace()) << "Others have closed"; return true; } if (!anyTransactions) { // Only close at the end of the idle interval - return currentTime >= idleInterval; // normal idle + return timeSincePrevClose >= idleInterval; // normal idle } // Preserve minimum ledger open time if (openTime < LEDGER_MIN_CLOSE) { - JLOG (j.debug()) << - "Must wait minimum time before closing"; + JLOG(j.debug()) << "Must wait minimum time before closing"; return false; } // Don't let this ledger close more than twice as fast as the previous // ledger reached consensus so that slower validators can slow down // the network - if (openTime < (previousTime / 2)) + if (openTime < (prevRoundTime / 2)) { - JLOG (j.debug()) << - "Ledger has not been open long enough"; + JLOG(j.debug()) << "Ledger has not been open long enough"; return false; } @@ -86,10 +84,7 @@ shouldCloseLedger ( } bool -checkConsensusReached ( - std::size_t agreeing, - std::size_t total, - bool count_self) +checkConsensusReached(std::size_t agreeing, std::size_t total, bool count_self) { // If we are alone, we have a consensus if (total == 0) @@ -107,8 +102,8 @@ checkConsensusReached ( } ConsensusState -checkConsensus ( - std::size_t previousProposers, +checkConsensus( + std::size_t prevProposers, std::size_t currentProposers, std::size_t currentAgree, std::size_t currentFinished, @@ -117,47 +112,45 @@ checkConsensus ( bool proposing, beast::Journal j) { - JLOG (j.trace()) << - "checkConsensus: prop=" << currentProposers << - "/" << previousProposers << - " agree=" << currentAgree << " validated=" << currentFinished << - " time=" << currentAgreeTime.count() << "/" << previousAgreeTime.count(); + JLOG(j.trace()) << "checkConsensus: prop=" << currentProposers << "/" + << prevProposers << " agree=" << currentAgree + << " validated=" << currentFinished + << " time=" << currentAgreeTime.count() << "/" + << previousAgreeTime.count(); if (currentAgreeTime <= LEDGER_MIN_CONSENSUS) return ConsensusState::No; - if (currentProposers < (previousProposers * 3 / 4)) + if (currentProposers < (prevProposers * 3 / 4)) { // Less than 3/4 of the last ledger's proposers are present; don't // rush: we may need more time. if (currentAgreeTime < (previousAgreeTime + LEDGER_MIN_CONSENSUS)) { - JLOG (j.trace()) << - "too fast, not enough proposers"; + JLOG(j.trace()) << "too fast, not enough proposers"; return ConsensusState::No; } } // Have we, together with the nodes on our UNL list, reached the threshold // to declare consensus? - if (checkConsensusReached (currentAgree, currentProposers, proposing)) + if (checkConsensusReached(currentAgree, currentProposers, proposing)) { - JLOG (j.debug()) << "normal consensus"; + JLOG(j.debug()) << "normal consensus"; return ConsensusState::Yes; } // Have sufficient nodes on our UNL list moved on and reached the threshold // to declare consensus? - if (checkConsensusReached (currentFinished, currentProposers, false)) + if (checkConsensusReached(currentFinished, currentProposers, false)) { - JLOG (j.warn()) << - "We see no consensus, but 80% of nodes have moved on"; + JLOG(j.warn()) << "We see no consensus, but 80% of nodes have moved on"; return ConsensusState::MovedOn; } // no consensus yet - JLOG (j.trace()) << "no consensus"; + JLOG(j.trace()) << "no consensus"; return ConsensusState::No; } -} // ripple +} // ripple diff --git a/src/ripple/consensus/LedgerTiming.h b/src/ripple/consensus/LedgerTiming.h index a9d5fce54..db135031b 100644 --- a/src/ripple/consensus/LedgerTiming.h +++ b/src/ripple/consensus/LedgerTiming.h @@ -20,14 +20,13 @@ #ifndef RIPPLE_APP_LEDGER_LEDGERTIMING_H_INCLUDED #define RIPPLE_APP_LEDGER_LEDGERTIMING_H_INCLUDED -#include -#include #include #include +#include +#include namespace ripple { - //------------------------------------------------------------------------------ // These are protocol parameters used to control the behavior of the system and // they should not be changed arbitrarily. @@ -42,16 +41,10 @@ using namespace std::chrono_literals; @see getNextLedgerTimeResolution */ std::chrono::seconds constexpr ledgerPossibleTimeResolutions[] = - { 10s, 20s, 30s, 60s, 90s, 120s }; + {10s, 20s, 30s, 60s, 90s, 120s}; -#ifndef _MSC_VER //! Initial resolution of ledger close time. auto constexpr ledgerDefaultTimeResolution = ledgerPossibleTimeResolutions[2]; -#else -// HH Remove this workaround of a VS bug when possible -//! Initial resolution of ledger close time. -auto constexpr ledgerDefaultTimeResolution = 30s; -#endif //! How often we increase the close time resolution (in numbers of ledgers) auto constexpr increaseLedgerTimeResolutionEvery = 8; @@ -161,23 +154,25 @@ getNextLedgerTimeResolution( bool previousAgree, std::uint32_t ledgerSeq) { - assert (ledgerSeq); + assert(ledgerSeq); using namespace std::chrono; // Find the current resolution: - auto iter = std::find (std::begin (ledgerPossibleTimeResolutions), - std::end (ledgerPossibleTimeResolutions), previousResolution); - assert (iter != std::end (ledgerPossibleTimeResolutions)); + auto iter = std::find( + std::begin(ledgerPossibleTimeResolutions), + std::end(ledgerPossibleTimeResolutions), + previousResolution); + assert(iter != std::end(ledgerPossibleTimeResolutions)); // This should never happen, but just as a precaution - if (iter == std::end (ledgerPossibleTimeResolutions)) + if (iter == std::end(ledgerPossibleTimeResolutions)) return previousResolution; // If we did not previously agree, we try to decrease the resolution to // improve the chance that we will agree now. if (!previousAgree && ledgerSeq % decreaseLedgerTimeResolutionEvery == 0) { - if (++iter != std::end (ledgerPossibleTimeResolutions)) + if (++iter != std::end(ledgerPossibleTimeResolutions)) return *iter; } @@ -185,7 +180,7 @@ getNextLedgerTimeResolution( // if we can continue to agree. if (previousAgree && ledgerSeq % increaseLedgerTimeResolutionEvery == 0) { - if (iter-- != std::begin (ledgerPossibleTimeResolutions)) + if (iter-- != std::begin(ledgerPossibleTimeResolutions)) return *iter; } @@ -212,7 +207,6 @@ roundCloseTime( return closeTime - (closeTime.time_since_epoch() % closeResolution); } - /** Calculate the effective ledger close time After adjusting the ledger close time based on the current resolution, also @@ -223,16 +217,17 @@ roundCloseTime( @param priorCloseTime The close time of the prior ledger */ template -time_point effectiveCloseTime(time_point closeTime, - typename time_point::duration const resolution, +time_point +effCloseTime( + time_point closeTime, + typename time_point::duration const resolution, time_point priorCloseTime) { if (closeTime == time_point{}) return closeTime; return std::max( - roundCloseTime (closeTime, resolution), - (priorCloseTime + 1s)); + roundCloseTime(closeTime, resolution), (priorCloseTime + 1s)); } /** Determines whether the current ledger should close at this time. @@ -241,30 +236,29 @@ time_point effectiveCloseTime(time_point closeTime, in progress, or when a transaction is received and no close is in progress. @param anyTransactions indicates whether any transactions have been received - @param previousProposers proposers in the last closing + @param prevProposers proposers in the last closing @param proposersClosed proposers who have currently closed this ledger @param proposersValidated proposers who have validated the last closed ledger - @param previousTime time for the previous ledger to reach consensus - @param currentTime time since the previous ledger's - (possibly rounded) close time - @param openTime time waiting to close this ledger + @param prevRoundTime time for the previous ledger to reach consensus + @param timeSincePrevClose time since the previous ledger's (possibly rounded) + close time + @param openTime duration this ledger has been open @param idleInterval the network's desired idle interval @param j journal for logging */ bool -shouldCloseLedger ( +shouldCloseLedger( bool anyTransactions, - std::size_t previousProposers, + std::size_t prevProposers, std::size_t proposersClosed, std::size_t proposersValidated, - std::chrono::milliseconds previousTime, - std::chrono::milliseconds currentTime, // Time since last ledger's close time - std::chrono::milliseconds openTime, // Time waiting to close this ledger + std::chrono::milliseconds prevRoundTime, + std::chrono::milliseconds timeSincePrevClose, + std::chrono::milliseconds openTime, std::chrono::seconds idleInterval, beast::Journal j); - /** Determine if a consensus has been reached This function determines if a consensus has been reached @@ -275,22 +269,18 @@ shouldCloseLedger ( @return True if a consensus has been reached */ bool -checkConsensusReached ( - std::size_t agreeing, - std::size_t total, - bool count_self); +checkConsensusReached(std::size_t agreeing, std::size_t total, bool count_self); /** Whether we have or don't have a consensus */ -enum class ConsensusState -{ - No, //!< We do not have consensus - MovedOn, //!< The network has consensus without us - Yes //!< We have consensus along with the network +enum class ConsensusState { + No, //!< We do not have consensus + MovedOn, //!< The network has consensus without us + Yes //!< We have consensus along with the network }; /** Determine whether the network reached consensus and whether we joined. - @param previousProposers proposers in the last closing (not including us) + @param prevProposers proposers in the last closing (not including us) @param currentProposers proposers in this closing so far (not including us) @param currentAgree proposers who agree with us @param currentFinished proposers who have validated a ledger after this one @@ -302,8 +292,8 @@ enum class ConsensusState @param j journal for logging */ ConsensusState -checkConsensus ( - std::size_t previousProposers, +checkConsensus( + std::size_t prevProposers, std::size_t currentProposers, std::size_t currentAgree, std::size_t currentFinished, @@ -312,6 +302,6 @@ checkConsensus ( bool proposing, beast::Journal j); -} // ripple +} // ripple #endif diff --git a/src/test/consensus/Consensus_test.cpp b/src/test/consensus/Consensus_test.cpp index d061b4a4b..0bffc4bda 100644 --- a/src/test/consensus/Consensus_test.cpp +++ b/src/test/consensus/Consensus_test.cpp @@ -17,22 +17,20 @@ */ //============================================================================== #include +#include #include #include #include -#include #include #include #include - namespace ripple { namespace test { class Consensus_test : public beast::unit_test::suite { public: - void testStandalone() { @@ -41,21 +39,22 @@ public: auto tg = TrustGraph::makeComplete(1); Sim s(tg, topology(tg, fixed{LEDGER_GRANULARITY})); - auto & p = s.peers[0]; + auto& p = s.peers[0]; p.targetLedgers = 1; p.start(); - p.submit(Tx{ 1 }); + p.submit(Tx{1}); s.net.step(); // Inspect that the proper ledger was created - BEAST_EXPECT(p.LCL().seq == 1); - BEAST_EXPECT(p.LCL() == p.lastClosedLedger.id()); + BEAST_EXPECT(p.prevLedgerID().seq == 1); + BEAST_EXPECT(p.prevLedgerID() == p.lastClosedLedger.id()); BEAST_EXPECT(p.lastClosedLedger.id().txs.size() == 1); - BEAST_EXPECT(p.lastClosedLedger.id().txs.find(Tx{ 1 }) - != p.lastClosedLedger.id().txs.end()); - BEAST_EXPECT(p.getLastCloseProposers() == 0); + BEAST_EXPECT( + p.lastClosedLedger.id().txs.find(Tx{1}) != + p.lastClosedLedger.id().txs.end()); + BEAST_EXPECT(p.prevProposers() == 0); } void @@ -65,24 +64,25 @@ public: using namespace std::chrono; auto tg = TrustGraph::makeComplete(5); - Sim sim(tg, + Sim sim( + tg, topology(tg, fixed{round(0.2 * LEDGER_GRANULARITY)})); // everyone submits their own ID as a TX and relay it to peers - for (auto & p : sim.peers) + for (auto& p : sim.peers) p.submit(Tx(p.id)); // Verify all peers have the same LCL and it has all the Txs sim.run(1); - for (auto & p : sim.peers) + for (auto& p : sim.peers) { - auto const &lgrID = p.LCL(); + auto const& lgrID = p.prevLedgerID(); BEAST_EXPECT(lgrID.seq == 1); - BEAST_EXPECT(p.getLastCloseProposers() == sim.peers.size() - 1); - for(std::uint32_t i = 0; i < sim.peers.size(); ++i) - BEAST_EXPECT(lgrID.txs.find(Tx{ i }) != lgrID.txs.end()); + BEAST_EXPECT(p.prevProposers() == sim.peers.size() - 1); + for (std::uint32_t i = 0; i < sim.peers.size(); ++i) + BEAST_EXPECT(lgrID.txs.find(Tx{i}) != lgrID.txs.end()); // Matches peer 0 ledger - BEAST_EXPECT(lgrID.txs == sim.peers[0].LCL().txs); + BEAST_EXPECT(lgrID.txs == sim.peers[0].prevLedgerID().txs); } } @@ -96,70 +96,69 @@ public: // 1. The slow peer is participating in consensus // 2. The slow peer is just observing - for(auto isParticipant : {true, false}) + for (auto isParticipant : {true, false}) { auto tg = TrustGraph::makeComplete(5); - Sim sim(tg, topology(tg,[](PeerID i, PeerID j) - { - auto delayFactor = (i == 0 || j == 0) ? 1.1 : 0.2; - return round(delayFactor* LEDGER_GRANULARITY); - })); + Sim sim(tg, topology(tg, [](PeerID i, PeerID j) { + auto delayFactor = (i == 0 || j == 0) ? 1.1 : 0.2; + return round( + delayFactor * LEDGER_GRANULARITY); + })); - sim.peers[0].proposing = sim.peers[0].validating = isParticipant; + sim.peers[0].proposing_ = sim.peers[0].validating_ = isParticipant; - // All peers submit their own ID as a transaction and relay it to peers - for (auto & p : sim.peers) + // All peers submit their own ID as a transaction and relay it to + // peers + for (auto& p : sim.peers) { - p.submit(Tx{ p.id }); + p.submit(Tx{p.id}); } sim.run(1); - // Verify all peers have same LCL but are missing transaction 0 which - // was not received by all peers before the ledger closed - for (auto & p : sim.peers) + // Verify all peers have same LCL but are missing transaction 0 + // which was not received by all peers before the ledger closed + for (auto& p : sim.peers) { - auto const &lgrID = p.LCL(); + auto const& lgrID = p.prevLedgerID(); BEAST_EXPECT(lgrID.seq == 1); - // If peer 0 is participating - if(isParticipant) + if (isParticipant) { - BEAST_EXPECT(p.getLastCloseProposers() - == sim.peers.size() - 1); - // Peer 0 closes first because it sees a quorum of agreeing positions - // from all other peers in one hop (1->0, 2->0, ..) - // The other peers take an extra timer period before they find that - // Peer 0 agrees with them ( 1->0->1, 2->0->2, ...) - if(p.id != 0) - BEAST_EXPECT(p.getLastConvergeDuration() - > sim.peers[0].getLastConvergeDuration()); + BEAST_EXPECT(p.prevProposers() == sim.peers.size() - 1); + // Peer 0 closes first because it sees a quorum of agreeing + // positions from all other peers in one hop (1->0, 2->0, + // ..) The other peers take an extra timer period before + // they find that Peer 0 agrees with them ( 1->0->1, + // 2->0->2, ...) + if (p.id != 0) + BEAST_EXPECT( + p.prevRoundTime() > sim.peers[0].prevRoundTime()); } - else // peer 0 is not participating + else // peer 0 is not participating { - auto const proposers = p.getLastCloseProposers(); - if(p.id == 0) + auto const proposers = p.prevProposers(); + if (p.id == 0) BEAST_EXPECT(proposers == sim.peers.size() - 1); else BEAST_EXPECT(proposers == sim.peers.size() - 2); // so all peers should have closed together - BEAST_EXPECT(p.getLastConvergeDuration() - == sim.peers[0].getLastConvergeDuration()); + BEAST_EXPECT( + p.prevRoundTime() == sim.peers[0].prevRoundTime()); } - - BEAST_EXPECT(lgrID.txs.find(Tx{ 0 }) == lgrID.txs.end()); - for(std::uint32_t i = 1; i < sim.peers.size(); ++i) - BEAST_EXPECT(lgrID.txs.find(Tx{ i }) != lgrID.txs.end()); + BEAST_EXPECT(lgrID.txs.find(Tx{0}) == lgrID.txs.end()); + for (std::uint32_t i = 1; i < sim.peers.size(); ++i) + BEAST_EXPECT(lgrID.txs.find(Tx{i}) != lgrID.txs.end()); // Matches peer 0 ledger - BEAST_EXPECT(lgrID.txs == sim.peers[0].LCL().txs); + BEAST_EXPECT(lgrID.txs == sim.peers[0].prevLedgerID().txs); } - BEAST_EXPECT(sim.peers[0].openTxs.find(Tx{ 0 }) - != sim.peers[0].openTxs.end()); - } + BEAST_EXPECT( + sim.peers[0].openTxs.find(Tx{0}) != sim.peers[0].openTxs.end()); + } } void @@ -185,32 +184,35 @@ public: // Complicating this matter is that nodes will ignore proposals // with times more than PROPOSE_FRESHNESS =20s in the past. So at - // the minimum granularity, we have at most 3 types of skews (0s,10s,20s). + // the minimum granularity, we have at most 3 types of skews + // (0s,10s,20s). // This test therefore has 6 nodes, with 2 nodes having each type of // skew. Then no majority (1/3 < 1/2) of nodes will agree on an // actual close time. auto tg = TrustGraph::makeComplete(6); - Sim sim(tg, + Sim sim( + tg, topology(tg, fixed{round(0.2 * LEDGER_GRANULARITY)})); - // Run consensus without skew until we have a short close time resolution - while(sim.peers.front().lastClosedLedger.closeTimeResolution() >= - PROPOSE_FRESHNESS) + // Run consensus without skew until we have a short close time + // resolution + while (sim.peers.front().lastClosedLedger.closeTimeResolution() >= + PROPOSE_FRESHNESS) sim.run(1); // Introduce a shift on the time of half the peers - sim.peers[0].clockSkew = PROPOSE_FRESHNESS/2; - sim.peers[1].clockSkew = PROPOSE_FRESHNESS/2; + sim.peers[0].clockSkew = PROPOSE_FRESHNESS / 2; + sim.peers[1].clockSkew = PROPOSE_FRESHNESS / 2; sim.peers[2].clockSkew = PROPOSE_FRESHNESS; sim.peers[3].clockSkew = PROPOSE_FRESHNESS; // Verify all peers have the same LCL and it has all the Txs sim.run(1); - for (auto & p : sim.peers) + for (auto& p : sim.peers) { - BEAST_EXPECT(! p.lastClosedLedger.closeAgree()); + BEAST_EXPECT(!p.lastClosedLedger.closeAgree()); } } @@ -224,9 +226,8 @@ public: // Vary the time it takes to process validations to exercise detecting // the wrong LCL at different phases of consensus - for(auto validationDelay : {0s, LEDGER_MIN_CLOSE}) + for (auto validationDelay : {0s, LEDGER_MIN_CLOSE}) { - // Consider 10 peers: // 0 1 2 3 4 5 6 7 8 9 // @@ -241,19 +242,21 @@ public: // since nodes 2-4 will validate a different ledger. // Nodes 0-1 will acquire the proper ledger from the network and - // resume consensus and eventually generate the dominant network ledger + // resume consensus and eventually generate the dominant network + // ledger std::vector unls; - unls.push_back({2,3,4,5,6,7,8,9}); - unls.push_back({0,1,2,3,4}); - std::vector membership(10,0); + unls.push_back({2, 3, 4, 5, 6, 7, 8, 9}); + unls.push_back({0, 1, 2, 3, 4}); + std::vector membership(10, 0); membership[0] = 1; membership[1] = 1; TrustGraph tg{unls, membership}; - // This topology can fork, which is why we are using it for this test. - BEAST_EXPECT(tg.canFork(minimumConsensusPercentage/100.)); + // This topology can fork, which is why we are using it for this + // test. + BEAST_EXPECT(tg.canFork(minimumConsensusPercentage / 100.)); auto netDelay = round(0.2 * LEDGER_GRANULARITY); Sim sim(tg, topology(tg, fixed{netDelay})); @@ -261,8 +264,9 @@ public: // initial round to set prior state sim.run(1); - // Nodes in smaller UNL have seen tx 0, nodes in other unl have seen tx 1 - for (auto & p : sim.peers) + // Nodes in smaller UNL have seen tx 0, nodes in other unl have seen + // tx 1 + for (auto& p : sim.peers) { p.validationDelay = validationDelay; p.missingLedgerDelay = netDelay; @@ -272,16 +276,29 @@ public: p.openTxs.insert(Tx{1}); } - // Run for 2 additional rounds - // - One round to generate different ledgers - // - One round to detect different prior ledgers (but still generate - // wrong ones) and recover - sim.run(2); + // Run for additional rounds + // With no validation delay, only 2 more rounds are needed. + // 1. Round to generate different ledgers + // 2. Round to detect different prior ledgers (but still generate + // wrong ones) and recover within that round since wrong LCL + // is detected before we close + // + // With a validation delay of LEDGER_MIN_CLOSE, we need 3 more + // rounds. + // 1. Round to generate different ledgers + // 2. Round to detect different prior ledgers (but still generate + // wrong ones) but end up declaring consensus on wrong LCL (but + // with the right transaction set!). This is because we detect + // the wrong LCL after we have closed the ledger, so we declare + // consensus based solely on our peer proposals. But we haven't + // had time to acquire the right LCL + // 3. Round to correct + sim.run(3); bc::flat_map> ledgers; - for (auto & p : sim.peers) + for (auto& p : sim.peers) { - for (auto const & l : p.ledgers) + for (auto const& l : p.ledgers) { ledgers[l.first.seq].insert(l.first); } @@ -289,12 +306,19 @@ public: BEAST_EXPECT(ledgers[0].size() == 1); BEAST_EXPECT(ledgers[1].size() == 1); - BEAST_EXPECT(ledgers[2].size() == 2); - BEAST_EXPECT(ledgers[3].size() == 1); - - + if (validationDelay == 0s) + { + BEAST_EXPECT(ledgers[2].size() == 2); + BEAST_EXPECT(ledgers[3].size() == 1); + BEAST_EXPECT(ledgers[4].size() == 1); + } + else + { + BEAST_EXPECT(ledgers[2].size() == 2); + BEAST_EXPECT(ledgers[3].size() == 2); + BEAST_EXPECT(ledgers[4].size() == 1); + } } - // Additional test engineered to switch LCL during the establish phase. // This was added to trigger a scenario that previously crashed, in which // switchLCL switched from establish to open phase, but still processed @@ -330,7 +354,7 @@ public: // Check all peers recovered for (auto &p : sim.peers) - BEAST_EXPECT(p.LCL() == sim.peers[0].LCL()); + BEAST_EXPECT(p.prevLedgerID() == sim.peers[0].prevLedgerID()); } } @@ -341,43 +365,43 @@ public: using namespace std::chrono; int numPeers = 10; - for(int overlap = 0; overlap <= numPeers; ++overlap) + for (int overlap = 0; overlap <= numPeers; ++overlap) { auto tg = TrustGraph::makeClique(numPeers, overlap); - Sim sim(tg, topology(tg, - fixed{round(0.2 * LEDGER_GRANULARITY)})); + Sim sim( + tg, + topology( + tg, fixed{round(0.2 * LEDGER_GRANULARITY)})); // Initial round to set prior state sim.run(1); - for (auto & p : sim.peers) + for (auto& p : sim.peers) { // Nodes have only seen transactions from their neighbors p.openTxs.insert(Tx{p.id}); - for(auto const link : sim.net.links(&p)) + for (auto const link : sim.net.links(&p)) p.openTxs.insert(Tx{link.to->id}); } sim.run(1); - // See if the network forked bc::flat_set ledgers; - for (auto & p : sim.peers) + for (auto& p : sim.peers) { - ledgers.insert(p.LCL()); + ledgers.insert(p.prevLedgerID()); } // Fork should not happen for 40% or greater overlap // Since the overlapped nodes have a UNL that is the union of the // two cliques, the maximum sized UNL list is the number of peers - if(overlap > 0.4 * numPeers) + if (overlap > 0.4 * numPeers) BEAST_EXPECT(ledgers.size() == 1); - else // Even if we do fork, there shouldn't be more than 3 ledgers - // One for cliqueA, one for cliqueB and one for nodes in both - BEAST_EXPECT(ledgers.size() <= 3); + else // Even if we do fork, there shouldn't be more than 3 ledgers + // One for cliqueA, one for cliqueB and one for nodes in both + BEAST_EXPECT(ledgers.size() <= 3); } } - void simClockSkew() { @@ -396,52 +420,44 @@ public: // Disabled while continuing to understand testt. - - for(auto stagger : {800ms, 1600ms, 3200ms, 30000ms, 45000ms, 300000ms}) + for (auto stagger : {800ms, 1600ms, 3200ms, 30000ms, 45000ms, 300000ms}) { - auto tg = TrustGraph::makeComplete(5); - Sim sim(tg, topology(tg, [](PeerID i, PeerID) - { - return 200ms * (i + 1); - })); - + Sim sim(tg, topology(tg, [](PeerID i, PeerID) { + return 200ms * (i + 1); + })); // all transactions submitted before starting // Initial round to set prior state sim.run(1); - for (auto & p : sim.peers) + for (auto& p : sim.peers) { - p.openTxs.insert(Tx{ 0 }); + p.openTxs.insert(Tx{0}); p.targetLedgers = p.completedLedgers + 1; - } // stagger start of consensus - for (auto & p : sim.peers) + for (auto& p : sim.peers) { p.start(); sim.net.step_for(stagger); } // run until all peers have accepted all transactions - sim.net.step_while([&]() - { - for(auto & p : sim.peers) - { - if(p.LCL().txs.size() != 1) + sim.net.step_while([&]() { + for (auto& p : sim.peers) + { + if (p.prevLedgerID().txs.size() != 1) { return true; } - } - return false; + } + return false; }); } } - - void simScaleFree() { @@ -450,45 +466,46 @@ public: // Generate a quasi-random scale free network and simulate consensus // for a single transaction - int N = 100; // Peers + int N = 100; // Peers int numUNLs = 15; // UNL lists - int minUNLSize = N/4, maxUNLSize = N / 2; + int minUNLSize = N / 4, maxUNLSize = N / 2; double transProb = 0.5; std::mt19937_64 rng; - auto tg = TrustGraph::makeRandomRanked(N, numUNLs, - PowerLawDistribution{1,3}, - std::uniform_int_distribution<>{minUNLSize, maxUNLSize}, - rng); + auto tg = TrustGraph::makeRandomRanked( + N, + numUNLs, + PowerLawDistribution{1, 3}, + std::uniform_int_distribution<>{minUNLSize, maxUNLSize}, + rng); - Sim sim{tg, topology(tg, fixed{round(0.2 * LEDGER_GRANULARITY)})}; + Sim sim{ + tg, + topology(tg, fixed{round(0.2 * LEDGER_GRANULARITY)})}; // Initial round to set prior state sim.run(1); std::uniform_real_distribution<> u{}; - for (auto & p : sim.peers) + for (auto& p : sim.peers) { // 50-50 chance to have seen a transaction - if(u(rng) >= transProb) + if (u(rng) >= transProb) p.openTxs.insert(Tx{0}); - } sim.run(1); - // See if the network forked bc::flat_set ledgers; - for (auto & p : sim.peers) + for (auto& p : sim.peers) { - ledgers.insert(p.LCL()); + ledgers.insert(p.prevLedgerID()); } BEAST_EXPECT(ledgers.size() == 1); - } void @@ -507,5 +524,5 @@ public: }; BEAST_DEFINE_TESTSUITE(Consensus, consensus, ripple); -} // test -} // ripple +} // test +} // ripple diff --git a/src/test/csf/BasicNetwork.h b/src/test/csf/BasicNetwork.h index 141106630..70096f76b 100644 --- a/src/test/csf/BasicNetwork.h +++ b/src/test/csf/BasicNetwork.h @@ -30,11 +30,11 @@ #include #include #include -#include -#include #include #include +#include #include +#include #include #include #include @@ -98,58 +98,54 @@ class BasicNetwork public: using peer_type = Peer; - using clock_type = - beast::manual_clock< - std::chrono::steady_clock>; + using clock_type = beast::manual_clock; - using duration = - typename clock_type::duration; + using duration = typename clock_type::duration; - using time_point = - typename clock_type::time_point; + using time_point = typename clock_type::time_point; private: - struct by_to_tag {}; - struct by_from_tag {}; - struct by_when_tag {}; + struct by_to_tag + { + }; + struct by_from_tag + { + }; + struct by_when_tag + { + }; - using by_to_hook = - boost::intrusive::list_base_hook< - boost::intrusive::link_mode< - boost::intrusive::normal_link>, - boost::intrusive::tag>; + using by_to_hook = boost::intrusive::list_base_hook< + boost::intrusive::link_mode, + boost::intrusive::tag>; - using by_from_hook = - boost::intrusive::list_base_hook< - boost::intrusive::link_mode< - boost::intrusive::normal_link>, - boost::intrusive::tag>; + using by_from_hook = boost::intrusive::list_base_hook< + boost::intrusive::link_mode, + boost::intrusive::tag>; - using by_when_hook = - boost::intrusive::set_base_hook< - boost::intrusive::link_mode< - boost::intrusive::normal_link>>; + using by_when_hook = boost::intrusive::set_base_hook< + boost::intrusive::link_mode>; - struct msg - : by_to_hook, by_from_hook, by_when_hook + struct msg : by_to_hook, by_from_hook, by_when_hook { Peer to; Peer from; time_point when; - msg (msg const&) = delete; - msg& operator= (msg const&) = delete; + msg(msg const&) = delete; + msg& + operator=(msg const&) = delete; virtual ~msg() = default; - virtual void operator()() const = 0; + virtual void + operator()() const = 0; - msg (Peer const& from_, Peer const& to_, - time_point when_) + msg(Peer const& from_, Peer const& to_, time_point when_) : to(to_), from(from_), when(when_) { } bool - operator< (msg const& other) const + operator<(msg const& other) const { return when < other.when; } @@ -162,24 +158,30 @@ private: Handler const h_; public: - msg_impl (msg_impl const&) = delete; - msg_impl& operator= (msg_impl const&) = delete; + msg_impl(msg_impl const&) = delete; + msg_impl& + operator=(msg_impl const&) = delete; - msg_impl (Peer const& from_, Peer const& to_, - time_point when_, Handler&& h) - : msg (from_, to_, when_) - , h_ (std::move(h)) + msg_impl( + Peer const& from_, + Peer const& to_, + time_point when_, + Handler&& h) + : msg(from_, to_, when_), h_(std::move(h)) { } - msg_impl (Peer const& from_, Peer const& to_, - time_point when_, Handler const& h) - : msg (from_, to_, when_) - , h_ (h) + msg_impl( + Peer const& from_, + Peer const& to_, + time_point when_, + Handler const& h) + : msg(from_, to_, when_), h_(h) { } - void operator()() const override + void + operator()() const override { h_(); } @@ -188,19 +190,19 @@ private: class queue_type { private: - using by_to_list = typename - boost::intrusive::make_list, - boost::intrusive::constant_time_size>::type; + using by_to_list = typename boost::intrusive::make_list< + msg, + boost::intrusive::base_hook, + boost::intrusive::constant_time_size>::type; - using by_from_list = typename - boost::intrusive::make_list, - boost::intrusive::constant_time_size>::type; + using by_from_list = typename boost::intrusive::make_list< + msg, + boost::intrusive::base_hook, + boost::intrusive::constant_time_size>::type; - using by_when_set = typename - boost::intrusive::make_multiset>::type; + using by_when_set = typename boost::intrusive::make_multiset< + msg, + boost::intrusive::constant_time_size>::type; qalloc alloc_; by_when_set by_when_; @@ -208,14 +210,13 @@ private: std::unordered_map by_from_; public: - using iterator = - typename by_when_set::iterator; + using iterator = typename by_when_set::iterator; - queue_type (queue_type const&) = delete; - queue_type& operator= (queue_type const&) = delete; + queue_type(queue_type const&) = delete; + queue_type& + operator=(queue_type const&) = delete; - explicit - queue_type (qalloc const& alloc); + explicit queue_type(qalloc const& alloc); ~queue_type(); @@ -230,14 +231,13 @@ private: template typename by_when_set::iterator - emplace (Peer const& from, Peer const& to, - time_point when, Handler&& h); + emplace(Peer const& from, Peer const& to, time_point when, Handler&& h); void - erase (iterator iter); + erase(iterator iter); void - remove (Peer const& from, Peer const& to); + remove(Peer const& from, Peer const& to); }; struct link_type @@ -245,15 +245,13 @@ private: bool inbound; duration delay; - link_type (bool inbound_, duration delay_) - : inbound (inbound_) - , delay (delay_) + link_type(bool inbound_, duration delay_) + : inbound(inbound_), delay(delay_) { } }; - using links_type = - boost::container::flat_map; + using links_type = boost::container::flat_map; class link_transform; @@ -265,8 +263,9 @@ private: std::unordered_map links_; public: - BasicNetwork (BasicNetwork const&) = delete; - BasicNetwork& operator= (BasicNetwork const&) = delete; + BasicNetwork(BasicNetwork const&) = delete; + BasicNetwork& + operator=(BasicNetwork const&) = delete; BasicNetwork(); @@ -308,7 +307,9 @@ public: @return `true` if a new connection was established */ bool - connect (Peer const& from, Peer const& to, + connect( + Peer const& from, + Peer const& to, duration const& delay = std::chrono::seconds{0}); /** Break a link. @@ -324,15 +325,14 @@ public: @return `true` if a connection was broken. */ bool - disconnect (Peer const& peer1, Peer const& peer2); + disconnect(Peer const& peer1, Peer const& peer2); /** Return the range of active links. @return A random access range. */ - boost::transformed_range< - link_transform, links_type> - links (Peer const& from); + boost::transformed_range + links(Peer const& from); /** Send a message to a peer. @@ -354,8 +354,7 @@ public: */ template void - send (Peer const& from, Peer const& to, - Function&& f); + send(Peer const& from, Peer const& to, Function&& f); // Used to cancel timers struct cancel_token; @@ -370,8 +369,7 @@ public: */ template cancel_token - timer (time_point const& when, - Function&& f); + timer(time_point const& when, Function&& f); /** Deliver a timer notification. @@ -383,8 +381,7 @@ public: */ template cancel_token - timer (duration const& delay, - Function&& f); + timer(duration const& delay, Function&& f); /** Cancel a timer. @@ -394,7 +391,7 @@ public: timer() which has not yet been invoked. */ void - cancel (cancel_token const& token); + cancel(cancel_token const& token); /** Perform breadth-first search. @@ -407,7 +404,7 @@ public: */ template void - bfs (Peer const& start, Function&& f); + bfs(Peer const& start, Function&& f); /** Run the network for up to one message. @@ -448,7 +445,7 @@ public: */ template bool - step_while(Function && func); + step_while(Function&& func); /** Run the network until the specified time. @@ -460,7 +457,7 @@ public: @return `true` if any messages remain. */ bool - step_until (time_point const& until); + step_until(time_point const& until); /** Run the network until time has elapsed. @@ -473,24 +470,21 @@ public: */ template bool - step_for (std::chrono::duration< - Period, Rep> const& amount); + step_for(std::chrono::duration const& amount); }; //------------------------------------------------------------------------------ template -BasicNetwork::queue_type::queue_type( - qalloc const& alloc) - : alloc_ (alloc) +BasicNetwork::queue_type::queue_type(qalloc const& alloc) + : alloc_(alloc) { } template BasicNetwork::queue_type::~queue_type() { - for(auto iter = by_when_.begin(); - iter != by_when_.end();) + for (auto iter = by_when_.begin(); iter != by_when_.end();) { auto m = &*iter; ++iter; @@ -500,27 +494,22 @@ BasicNetwork::queue_type::~queue_type() } template -inline -bool +inline bool BasicNetwork::queue_type::empty() const { return by_when_.empty(); } template -inline -auto -BasicNetwork::queue_type::begin() -> - iterator +inline auto +BasicNetwork::queue_type::begin() -> iterator { return by_when_.begin(); } template -inline -auto -BasicNetwork::queue_type::end() -> - iterator +inline auto +BasicNetwork::queue_type::end() -> iterator { return by_when_.end(); } @@ -529,15 +518,14 @@ template template auto BasicNetwork::queue_type::emplace( - Peer const& from, Peer const& to, time_point when, - Handler&& h) -> - typename by_when_set::iterator + Peer const& from, + Peer const& to, + time_point when, + Handler&& h) -> typename by_when_set::iterator { - using msg_type = msg_impl< - std::decay_t>; + using msg_type = msg_impl>; auto const p = alloc_.alloc(1); - auto& m = *new(p) msg_type(from, to, - when, std::forward(h)); + auto& m = *new (p) msg_type(from, to, when, std::forward(h)); if (to) by_to_[to].push_back(m); if (from) @@ -547,8 +535,7 @@ BasicNetwork::queue_type::emplace( template void -BasicNetwork::queue_type::erase( - iterator iter) +BasicNetwork::queue_type::erase(iterator iter) { auto& m = *iter; if (iter->to) @@ -568,13 +555,11 @@ BasicNetwork::queue_type::erase( template void -BasicNetwork::queue_type::remove( - Peer const& from, Peer const& to) +BasicNetwork::queue_type::remove(Peer const& from, Peer const& to) { { auto& list = by_to_[to]; - for(auto iter = list.begin(); - iter != list.end();) + for (auto iter = list.begin(); iter != list.end();) { auto& m = *iter++; if (m.from == from) @@ -583,8 +568,7 @@ BasicNetwork::queue_type::remove( } { auto& list = by_to_[from]; - for(auto iter = list.begin(); - iter != list.end();) + for (auto iter = list.begin(); iter != list.end();) { auto& m = *iter++; if (m.from == to) @@ -603,8 +587,7 @@ private: Peer from_; public: - using argument_type = - typename links_type::value_type; + using argument_type = typename links_type::value_type; class result_type { @@ -612,15 +595,14 @@ public: Peer to; bool inbound; - result_type (result_type const&) = default; + result_type(result_type const&) = default; - result_type (BasicNetwork& net, - Peer const& from, Peer const& to_, - bool inbound_) - : to(to_) - , inbound(inbound_) - , net_(net) - , from_(from) + result_type( + BasicNetwork& net, + Peer const& from, + Peer const& to_, + bool inbound_) + : to(to_), inbound(inbound_), net_(net), from_(from) { } @@ -641,18 +623,14 @@ public: Peer from_; }; - link_transform (BasicNetwork& net, - Peer const& from) - : net_(net) - , from_(from) + link_transform(BasicNetwork& net, Peer const& from) : net_(net), from_(from) { } result_type const operator()(argument_type const& v) const { - return result_type(net_, from_, - v.first, v.second.inbound); + return result_type(net_, from_, v.first, v.second.inbound); } }; @@ -666,14 +644,13 @@ private: public: cancel_token() = delete; - cancel_token (cancel_token const&) = default; - cancel_token& operator= (cancel_token const&) = default; + cancel_token(cancel_token const&) = default; + cancel_token& + operator=(cancel_token const&) = default; private: friend class BasicNetwork; - cancel_token(typename - queue_type::iterator iter) - : iter_ (iter) + cancel_token(typename queue_type::iterator iter) : iter_(iter) { } }; @@ -681,33 +658,27 @@ private: //------------------------------------------------------------------------------ template -BasicNetwork::BasicNetwork() - : queue_ (alloc_) +BasicNetwork::BasicNetwork() : queue_(alloc_) { } template -inline -qalloc const& +inline qalloc const& BasicNetwork::alloc() const { return alloc_; } template -inline -auto -BasicNetwork::clock() const -> - clock_type& +inline auto +BasicNetwork::clock() const -> clock_type& { return clock_; } template -inline -auto -BasicNetwork::now() const -> - time_point +inline auto +BasicNetwork::now() const -> time_point { return clock_.now(); } @@ -715,17 +686,16 @@ BasicNetwork::now() const -> template bool BasicNetwork::connect( - Peer const& from, Peer const& to, - duration const& delay) + Peer const& from, + Peer const& to, + duration const& delay) { if (to == from) return false; using namespace std; - if (! links_[from].emplace(to, - link_type{ false, delay }).second) + if (!links_[from].emplace(to, link_type{false, delay}).second) return false; - auto const result = links_[to].emplace( - from, link_type{ true, delay }); + auto const result = links_[to].emplace(from, link_type{true, delay}); (void)result; assert(result.second); return true; @@ -733,13 +703,11 @@ BasicNetwork::connect( template bool -BasicNetwork::disconnect( - Peer const& peer1, Peer const& peer2) +BasicNetwork::disconnect(Peer const& peer1, Peer const& peer2) { if (links_[peer1].erase(peer2) == 0) return false; - auto const n = - links_[peer2].erase(peer1); + auto const n = links_[peer2].erase(peer1); (void)n; assert(n); queue_.remove(peer1, peer2); @@ -747,64 +715,45 @@ BasicNetwork::disconnect( } template -inline -auto -BasicNetwork::links(Peer const& from) -> - boost::transformed_range< - link_transform, links_type> +inline auto +BasicNetwork::links(Peer const& from) + -> boost::transformed_range { return boost::adaptors::transform( - links_[from], - link_transform{ *this, from }); + links_[from], link_transform{*this, from}); } template template -inline -void -BasicNetwork::send( - Peer const& from, Peer const& to, - Function&& f) +inline void +BasicNetwork::send(Peer const& from, Peer const& to, Function&& f) { using namespace std; - auto const iter = - links_[from].find(to); - queue_.emplace(from, to, - clock_.now() + iter->second.delay, - forward(f)); + auto const iter = links_[from].find(to); + queue_.emplace( + from, to, clock_.now() + iter->second.delay, forward(f)); } template template -inline -auto -BasicNetwork::timer( - time_point const& when, Function&& f) -> - cancel_token +inline auto +BasicNetwork::timer(time_point const& when, Function&& f) -> cancel_token { using namespace std; - return queue_.emplace( - nullptr, nullptr, when, - forward(f)); + return queue_.emplace(nullptr, nullptr, when, forward(f)); } template template -inline -auto -BasicNetwork::timer( - duration const& delay, Function&& f) -> - cancel_token +inline auto +BasicNetwork::timer(duration const& delay, Function&& f) -> cancel_token { - return timer(clock_.now() + delay, - std::forward(f)); + return timer(clock_.now() + delay, std::forward(f)); } template -inline -void -BasicNetwork::cancel( - cancel_token const& token) +inline void +BasicNetwork::cancel(cancel_token const& token) { queue_.erase(token.iter_); } @@ -826,10 +775,10 @@ template bool BasicNetwork::step() { - if (! step_one()) + if (!step_one()) return false; - for(;;) - if (! step_one()) + for (;;) + if (!step_one()) break; return true; } @@ -837,7 +786,7 @@ BasicNetwork::step() template template bool -BasicNetwork::step_while(Function && f) +BasicNetwork::step_while(Function&& f) { bool ran = false; while (f() && step_one()) @@ -847,17 +796,16 @@ BasicNetwork::step_while(Function && f) template bool -BasicNetwork::step_until( - time_point const& until) +BasicNetwork::step_until(time_point const& until) { // VFALCO This routine needs optimizing - if(queue_.empty()) + if (queue_.empty()) { clock_.set(until); return false; } auto iter = queue_.begin(); - if(iter->when > until) + if (iter->when > until) { clock_.set(until); return true; @@ -866,19 +814,15 @@ BasicNetwork::step_until( { step_one(); iter = queue_.begin(); - } - while(iter != queue_.end() && - iter->when <= until); + } while (iter != queue_.end() && iter->when <= until); clock_.set(until); return iter != queue_.end(); } template template -inline -bool -BasicNetwork::step_for( - std::chrono::duration const& amount) +inline bool +BasicNetwork::step_for(std::chrono::duration const& amount) { return step_until(now() + amount); } @@ -886,19 +830,18 @@ BasicNetwork::step_for( template template void -BasicNetwork::bfs( - Peer const& start, Function&& f) +BasicNetwork::bfs(Peer const& start, Function&& f) { std::deque> q; std::unordered_set seen; q.emplace_back(start, 0); seen.insert(start); - while(! q.empty()) + while (!q.empty()) { auto v = q.front(); q.pop_front(); f(v.second, v.first); - for(auto const& link : links_[v.first]) + for (auto const& link : links_[v.first]) { auto const& w = link.first; if (seen.count(w) == 0) @@ -910,8 +853,8 @@ BasicNetwork::bfs( } } -} // csf -} // test -} // ripple +} // csf +} // test +} // ripple #endif diff --git a/src/test/csf/BasicNetwork_test.cpp b/src/test/csf/BasicNetwork_test.cpp index d82912bc4..0973b799a 100644 --- a/src/test/csf/BasicNetwork_test.cpp +++ b/src/test/csf/BasicNetwork_test.cpp @@ -18,15 +18,14 @@ //============================================================================== #include -#include #include #include +#include #include namespace ripple { namespace test { - class BasicNetwork_test : public beast::unit_test::suite { public: @@ -35,28 +34,25 @@ public: int id; std::set set; - Peer (Peer const&) = default; - Peer (Peer&&) = default; + Peer(Peer const&) = default; + Peer(Peer&&) = default; - explicit Peer(int id_) - : id(id_) + explicit Peer(int id_) : id(id_) { } template - void start(Net& net) + void + start(Net& net) { using namespace std::chrono_literals; - auto t = net.timer(1s, - [&]{ set.insert(0); }); + auto t = net.timer(1s, [&] { set.insert(0); }); if (id == 0) { - for(auto const& link : net.links(this)) - net.send(this, link.to, - [&, to = link.to] - { - to->receive(net, this, 1); - }); + for (auto const& link : net.links(this)) + net.send(this, link.to, [&, to = link.to ] { + to->receive(net, this, 1); + }); } else { @@ -65,23 +61,23 @@ public: } template - void receive(Net& net, Peer* from, int m) + void + receive(Net& net, Peer* from, int m) { set.insert(m); ++m; if (m < 5) { - for(auto const& link : net.links(this)) - net.send(this, link.to, - [&, mm = m, to = link.to] - { - to->receive(net, this, mm); - }); + for (auto const& link : net.links(this)) + net.send(this, link.to, [&, mm = m, to = link.to ] { + to->receive(net, this, mm); + }); } } }; - void run() override + void + run() override { using namespace std::chrono_literals; std::vector pv; @@ -89,45 +85,40 @@ public: pv.emplace_back(1); pv.emplace_back(2); csf::BasicNetwork net; - BEAST_EXPECT(! net.connect(&pv[0], &pv[0])); + BEAST_EXPECT(!net.connect(&pv[0], &pv[0])); BEAST_EXPECT(net.connect(&pv[0], &pv[1], 1s)); BEAST_EXPECT(net.connect(&pv[1], &pv[2], 1s)); - BEAST_EXPECT(! net.connect(&pv[0], &pv[1])); + BEAST_EXPECT(!net.connect(&pv[0], &pv[1])); std::size_t diameter = 0; - net.bfs(&pv[0], - [&](auto d, Peer*) - { diameter = std::max(d, diameter); }); + net.bfs( + &pv[0], [&](auto d, Peer*) { diameter = std::max(d, diameter); }); BEAST_EXPECT(diameter == 2); - for(auto& peer : pv) + for (auto& peer : pv) peer.start(net); BEAST_EXPECT(net.step_for(0s)); BEAST_EXPECT(net.step_for(1s)); BEAST_EXPECT(net.step()); - BEAST_EXPECT(! net.step()); - BEAST_EXPECT(! net.step_for(1s)); - net.send(&pv[0], &pv[1], []{}); - net.send(&pv[1], &pv[0], []{}); + BEAST_EXPECT(!net.step()); + BEAST_EXPECT(!net.step_for(1s)); + net.send(&pv[0], &pv[1], [] {}); + net.send(&pv[1], &pv[0], [] {}); BEAST_EXPECT(net.disconnect(&pv[0], &pv[1])); - BEAST_EXPECT(! net.disconnect(&pv[0], &pv[1])); - for(;;) + BEAST_EXPECT(!net.disconnect(&pv[0], &pv[1])); + for (;;) { auto const links = net.links(&pv[1]); - if(links.empty()) + if (links.empty()) break; BEAST_EXPECT(links[0].disconnect()); } - BEAST_EXPECT(pv[0].set == - std::set({0, 2, 4})); - BEAST_EXPECT(pv[1].set == - std::set({1, 3})); - BEAST_EXPECT(pv[2].set == - std::set({2, 4})); - net.timer(0s, []{}); + BEAST_EXPECT(pv[0].set == std::set({0, 2, 4})); + BEAST_EXPECT(pv[1].set == std::set({1, 3})); + BEAST_EXPECT(pv[2].set == std::set({2, 4})); + net.timer(0s, [] {}); } }; BEAST_DEFINE_TESTSUITE(BasicNetwork, test, ripple); -} // test -} // ripple - +} // test +} // ripple diff --git a/src/test/csf/Ledger.h b/src/test/csf/Ledger.h index 1ce5df632..1787ce33c 100644 --- a/src/test/csf/Ledger.h +++ b/src/test/csf/Ledger.h @@ -46,32 +46,32 @@ namespace csf { class Ledger { - public: - struct ID { std::uint32_t seq = 0; TxSetType txs = TxSetType{}; - bool operator==(ID const & o) const + bool + operator==(ID const& o) const { return seq == o.seq && txs == o.txs; } - bool operator!=(ID const & o) const + bool + operator!=(ID const& o) const { return !(*this == o); } - bool operator<(ID const & o) const + bool + operator<(ID const& o) const { return std::tie(seq, txs) < std::tie(o.seq, o.txs); } }; - - auto const & + auto const& id() const { return id_; @@ -113,7 +113,7 @@ public: return parentCloseTime_; } - auto const & + auto const& parentID() const { return parentID_; @@ -127,30 +127,28 @@ public: return res; } - //! Apply the given transactions to this ledger Ledger - close(TxSetType const & txs, + close( + TxSetType const& txs, NetClock::duration closeTimeResolution, - NetClock::time_point const & consensusCloseTime, + NetClock::time_point const& consensusCloseTime, bool closeTimeAgree) const { - Ledger res{ *this }; + Ledger res{*this}; res.id_.txs.insert(txs.begin(), txs.end()); - res.id_ .seq= seq() + 1; + res.id_.seq = seq() + 1; res.closeTimeResolution_ = closeTimeResolution; res.actualCloseTime_ = consensusCloseTime; - res.closeTime_ = effectiveCloseTime(consensusCloseTime, - closeTimeResolution, parentCloseTime_); + res.closeTime_ = effCloseTime( + consensusCloseTime, closeTimeResolution, parentCloseTime_); res.closeTimeAgree_ = closeTimeAgree; res.parentCloseTime_ = closeTime(); res.parentID_ = id(); return res; } - private: - //! Unique identifier of ledger is combination of sequence number and id ID id_; @@ -171,27 +169,24 @@ private: //! Close time unadjusted by closeTimeResolution NetClock::time_point actualCloseTime_; - }; -inline -std::ostream & -operator<<(std::ostream & o, Ledger::ID const & id) +inline std::ostream& +operator<<(std::ostream& o, Ledger::ID const& id) { return o << id.seq << "," << id.txs; } -inline -std::string -to_string(Ledger::ID const & id) +inline std::string +to_string(Ledger::ID const& id) { std::stringstream ss; ss << id; return ss.str(); } -} // csf -} // test -} // ripple +} // csf +} // test +} // ripple -#endif \ No newline at end of file +#endif diff --git a/src/test/csf/Peer.h b/src/test/csf/Peer.h index 628f3600d..f713f487b 100644 --- a/src/test/csf/Peer.h +++ b/src/test/csf/Peer.h @@ -22,15 +22,14 @@ #include #include -#include #include +#include #include namespace ripple { namespace test { namespace csf { - /** Store validations reached by peers */ struct Validation { @@ -45,15 +44,17 @@ class Validations { //< Ledgers seen by peers, saved in order received (which should be order //< created) - bc::flat_map> nodesFromLedger; - bc::flat_map> nodesFromPrevLedger; - bc::flat_map> childLedgers; + bc::flat_map> nodesFromLedger; + bc::flat_map> nodesFromPrevLedger; + bc::flat_map> + childLedgers; + public: void - update(Validation const & v) + update(Validation const& v) { nodesFromLedger[v.ledger].insert(v.id); - if(v.ledger.seq > 0) + if (v.ledger.seq > 0) { nodesFromPrevLedger[v.prevLedger].insert(v.id); childLedgers[v.prevLedger][v.ledger]++; @@ -62,10 +63,10 @@ public: //< The number of peers who have validated this ledger std::size_t - proposersValidated(Ledger::ID const & prevLedger) const + proposersValidated(Ledger::ID const& prevLedger) const { auto it = nodesFromLedger.find(prevLedger); - if(it != nodesFromLedger.end()) + if (it != nodesFromLedger.end()) return it->second.size(); return 0; } @@ -75,35 +76,33 @@ public: as an ancestor. */ std::size_t - proposersFinished(Ledger::ID const & prevLedger) const + proposersFinished(Ledger::ID const& prevLedger) const { auto it = nodesFromPrevLedger.find(prevLedger); - if(it != nodesFromPrevLedger.end()) + if (it != nodesFromPrevLedger.end()) return it->second.size(); return 0; } /** Returns the ledger starting from prevLedger with the most validations. - */ + */ Ledger::ID - getBestLCL(Ledger::ID const & currLedger, - Ledger::ID const & prevLedger) const + getBestLCL(Ledger::ID const& currLedger, Ledger::ID const& prevLedger) const { auto it = childLedgers.find(prevLedger); - if (it != childLedgers.end() && - ! it->second.empty()) + if (it != childLedgers.end() && !it->second.empty()) { std::size_t bestCount = 0; Ledger::ID bestLedger; - for (auto const & b : it->second) + for (auto const& b : it->second) { auto currCount = b.second; - if(currLedger == b.first) + if (currLedger == b.first) currCount++; - if(currCount > bestCount) + if (currCount > bestCount) bestLedger = b.first; - if(currCount == bestCount && currLedger == b.first) + if (currCount == bestCount && currLedger == b.first) bestLedger = b.first; } return bestLedger; @@ -122,11 +121,8 @@ struct Traits using Ledger_t = Ledger; using NodeID_t = PeerID; using TxSet_t = TxSet; - using MissingTxException_t = MissingTx; }; - - /** Represents a single node participating in the consensus process. It implements the Callbacks required by Consensus. */ @@ -144,7 +140,7 @@ struct Peer : public Consensus Ledger lastClosedLedger; //! Handle to network for sending messages - BasicNetwork & net; + BasicNetwork& net; //! UNL of trusted peers UNL unl; @@ -173,12 +169,12 @@ struct Peer : public Consensus //! Delay in acquiring missing ledger from the network std::chrono::milliseconds missingLedgerDelay{0}; - bool validating = true; - bool proposing = true; + bool validating_ = true; + bool proposing_ = true; //! All peers start from the default constructed ledger - Peer(PeerID i, BasicNetwork & n, UNL const & u) - : Consensus( n.clock(), beast::Journal{}) + Peer(PeerID i, BasicNetwork& n, UNL const& u) + : Consensus(n.clock(), beast::Journal{}) , id{i} , net{n} , unl(u) @@ -186,19 +182,8 @@ struct Peer : public Consensus ledgers[lastClosedLedger.id()] = lastClosedLedger; } - - // @return whether we are proposing,validating - // TODO: Bit akward that this is in callbacks, would be nice to extract - std::pair - getMode() - { - // in RCL this hits NetworkOps to decide whether we are proposing - // validating - return{ proposing, validating }; - } - - Ledger const * - acquireLedger(Ledger::ID const & ledgerHash) + Ledger const* + acquireLedger(Ledger::ID const& ledgerHash) { auto it = ledgers.find(ledgerHash); if (it != ledgers.end()) @@ -208,16 +193,16 @@ struct Peer : public Consensus for (auto const& link : net.links(this)) { - auto const & p = *link.to; + auto const& p = *link.to; auto it = p.ledgers.find(ledgerHash); if (it != p.ledgers.end()) { - schedule(missingLedgerDelay, - [this, ledgerHash, ledger = it->second]() - { + schedule( + missingLedgerDelay, + [ this, ledgerHash, ledger = it->second ]() { ledgers.emplace(ledgerHash, ledger); }); - if(missingLedgerDelay == 0ms) + if (missingLedgerDelay == 0ms) return &ledgers[ledgerHash]; break; } @@ -225,23 +210,22 @@ struct Peer : public Consensus return nullptr; } - auto const & - proposals(Ledger::ID const & ledgerHash) + auto const& + proposals(Ledger::ID const& ledgerHash) { return peerPositions_[ledgerHash]; } - TxSet const * - acquireTxSet(TxSet::ID const & setId) + TxSet const* + acquireTxSet(TxSet::ID const& setId) { auto it = txSets.find(setId); - if(it != txSets.end()) + if (it != txSets.end()) return &(it->second); // TODO Get from network/oracle instead! return nullptr; } - bool hasOpenTransactions() const { @@ -249,114 +233,68 @@ struct Peer : public Consensus } std::size_t - proposersValidated(Ledger::ID const & prevLedger) + proposersValidated(Ledger::ID const& prevLedger) { return peerValidations.proposersValidated(prevLedger); } std::size_t - proposersFinished(Ledger::ID const & prevLedger) + proposersFinished(Ledger::ID const& prevLedger) { return peerValidations.proposersFinished(prevLedger); } - void - onStartRound(Ledger const &) {} - - void - onClose(Ledger const &, bool ) {} - - // don't really offload - void - dispatchAccept(TxSet const & f) + Result + onClose(Ledger const& prevLedger, NetClock::time_point closeTime, Mode mode) { - Base::accept(f); + TxSet res{openTxs}; + + return Result{TxSet{openTxs}, + Proposal{prevLedger.id(), + Proposal::seqJoin, + res.id(), + closeTime, + now(), + id}}; } void - share(TxSet const &s) + onForceAccept( + Result const& result, + Ledger const& prevLedger, + NetClock::duration const& closeResolution, + CloseTimes const& rawCloseTimes, + Mode const& mode) { - relay(s); - } - - Ledger::ID - getLCL(Ledger::ID const & currLedger, - Ledger::ID const & priorLedger, - bool haveCorrectLCL) - { - // TODO: Use generic validation code - if(currLedger.seq > 0 && priorLedger.seq > 0) - return peerValidations.getBestLCL(currLedger, priorLedger); - return currLedger; + onAccept(result, prevLedger, closeResolution, rawCloseTimes, mode); } void - propose(Proposal const & pos) + onAccept( + Result const& result, + Ledger const& prevLedger, + NetClock::duration const& closeResolution, + CloseTimes const& rawCloseTimes, + Mode const& mode) { - if(proposing) - relay(pos); - } - - void - relay(DisputedTx const & dispute) - { - relay(dispute.tx()); - } - - std::pair - makeInitialPosition( - Ledger const & prevLedger, - bool isProposing, - bool isCorrectLCL, - NetClock::time_point closeTime, - NetClock::time_point now) - { - TxSet res{ openTxs }; - - return { res, - Proposal{prevLedger.id(), Proposal::seqJoin, res.id(), closeTime, now, id} }; - } - - // Process the accepted transaction set, generating the newly closed ledger - // and clearing out the openTxs that were included. - // TODO: Kinda nasty it takes so many arguments . . . sign of bad coupling - bool - accept(TxSet const& set, - NetClock::time_point consensusCloseTime, - bool proposing_, - bool validating_, - bool haveCorrectLCL_, - bool consensusFail_, - Ledger::ID const & prevLedgerHash_, - Ledger const & previousLedger_, - NetClock::duration closeResolution_, - NetClock::time_point const & now, - std::chrono::milliseconds const & roundTime_, - hash_map> const & disputes_, - std::map closeTimes_, - NetClock::time_point const & closeTime) - { - auto newLedger = previousLedger_.close(set.txs_, closeResolution_, - closeTime, consensusCloseTime != NetClock::time_point{}); + auto newLedger = prevLedger.close( + result.set.txs_, + closeResolution, + rawCloseTimes.self, + result.position.closeTime() != NetClock::time_point{}); ledgers[newLedger.id()] = newLedger; lastClosedLedger = newLedger; - auto it = std::remove_if(openTxs.begin(), openTxs.end(), - [&](Tx const & tx) - { - return set.exists(tx.id()); + auto it = + std::remove_if(openTxs.begin(), openTxs.end(), [&](Tx const& tx) { + return result.set.exists(tx.id()); }); openTxs.erase(it, openTxs.end()); - if(validating) + if (validating_) relay(Validation{id, newLedger.id(), newLedger.parentID()}); - return validating_; - } - void - endConsensus(bool correct) - { // kick off the next round... // in the actual implementation, this passes back through // network ops @@ -364,41 +302,58 @@ struct Peer : public Consensus // startRound sets the LCL state, so we need to call it once after // the last requested round completes // TODO: reconsider this and instead just save LCL generated here? - if(completedLedgers <= targetLedgers) + if (completedLedgers <= targetLedgers) { - startRound(now(), lastClosedLedger.id(), - lastClosedLedger); + startRound( + now(), lastClosedLedger.id(), lastClosedLedger, proposing_); } } + Ledger::ID + getPrevLedger(Ledger::ID const& ledgerID, Ledger const& ledger, Mode mode) + { + // TODO: Use generic validation code + if (mode != Mode::wrongLedger && ledgerID.seq > 0 && + ledger.id().seq > 0) + return peerValidations.getBestLCL(ledgerID, ledger.parentID()); + return ledgerID; + } + + void + propose(Proposal const& pos) + { + if (proposing_) + relay(pos); + } + //------------------------------------------------------------------------- // non-callback helpers void - receive(Proposal const & p) + receive(Proposal const& p) { - if(unl.find(p.nodeID()) == unl.end()) + if (unl.find(p.nodeID()) == unl.end()) return; // TODO: Be sure this is a new proposal!!!!! - auto & dest = peerPositions_[p.prevLedger()]; - if(std::find(dest.begin(), dest.end(), p) != dest.end()) + auto& dest = peerPositions_[p.prevLedger()]; + if (std::find(dest.begin(), dest.end(), p) != dest.end()) return; + dest.push_back(p); peerProposal(now(), p); - } void - receive(TxSet const & txs) + receive(TxSet const& txs) { // save and map complete? auto it = txSets.insert(std::make_pair(txs.id(), txs)); - if(it.second) + if (it.second) gotTxSet(now(), txs); } void - receive(Tx const & tx) + receive(Tx const& tx) { if (openTxs.find(tx.id()) == openTxs.end()) { @@ -409,33 +364,26 @@ struct Peer : public Consensus } void - receive(Validation const & v) + receive(Validation const& v) { - if(unl.find(v.id) != unl.end()) + if (unl.find(v.id) != unl.end()) { - schedule(validationDelay, - [&, v]() - { - peerValidations.update(v); - }); + schedule(validationDelay, [&, v]() { peerValidations.update(v); }); } } template void - relay(T const & t) + relay(T const& t) { - for(auto const& link : net.links(this)) - net.send(this, link.to, - [msg = t, to = link.to] - { - to->receive(msg); - }); + for (auto const& link : net.links(this)) + net.send( + this, link.to, [ msg = t, to = link.to ] { to->receive(msg); }); } // Receive and relay locally submitted transaction void - submit(Tx const & tx) + submit(Tx const& tx) { receive(tx); relay(tx); @@ -446,7 +394,7 @@ struct Peer : public Consensus { Base::timerEntry(now()); // only reschedule if not completed - if(completedLedgers < targetLedgers) + if (completedLedgers < targetLedgers) net.timer(LEDGER_GRANULARITY, [&]() { timerEntry(); }); } void @@ -456,10 +404,9 @@ struct Peer : public Consensus // The ID is the one we have seen the most validations for // In practice, we might not actually have that ledger itself yet, // so there is no gaurantee that bestLCL == lastClosedLedger.id() - auto bestLCL = peerValidations.getBestLCL(lastClosedLedger.id(), - lastClosedLedger.parentID()); - startRound(now(), bestLCL, - lastClosedLedger); + auto bestLCL = peerValidations.getBestLCL( + lastClosedLedger.id(), lastClosedLedger.parentID()); + startRound(now(), bestLCL, lastClosedLedger, proposing_); } NetClock::time_point @@ -470,23 +417,24 @@ struct Peer : public Consensus // any subtractions of two NetClock::time_point in the consensu // code are positive. (e.g. PROPOSE_FRESHNESS) using namespace std::chrono; - return NetClock::time_point(duration_cast - (net.now().time_since_epoch()+ 86400s + clockSkew)); + return NetClock::time_point(duration_cast( + net.now().time_since_epoch() + 86400s + clockSkew)); } // Schedule the provided callback in `when` duration, but if // `when` is 0, call immediately template - void schedule(std::chrono::nanoseconds when, T && what) + void + schedule(std::chrono::nanoseconds when, T&& what) { - if(when == 0ns) + if (when == 0ns) what(); else net.timer(when, std::forward(what)); } }; -} // csf -} // test -} // ripple +} // csf +} // test +} // ripple #endif diff --git a/src/test/csf/Sim.h b/src/test/csf/Sim.h index 13523203a..4515deddf 100644 --- a/src/test/csf/Sim.h +++ b/src/test/csf/Sim.h @@ -20,8 +20,8 @@ #ifndef RIPPLE_TEST_CSF_SIM_H_INCLUDED #define RIPPLE_TEST_CSF_SIM_H_INCLUDED -#include #include +#include namespace ripple { namespace test { @@ -50,19 +50,19 @@ public: */ template - Sim(TrustGraph const & g, Topology const & top) + Sim(TrustGraph const& g, Topology const& top) { peers.reserve(g.numPeers()); - for(int i = 0; i < g.numPeers(); ++i) + for (int i = 0; i < g.numPeers(); ++i) peers.emplace_back(i, net, g.unl(i)); - for(int i = 0; i < peers.size(); ++i) + for (int i = 0; i < peers.size(); ++i) { - for(int j = 0; j < peers.size(); ++j) + for (int j = 0; j < peers.size(); ++j) { - if( i != j) + if (i != j) { - auto d = top(i,j); + auto d = top(i, j); if (d) { net.connect(&peers[i], &peers[j], *d); @@ -81,10 +81,10 @@ public: void run(int ledgers) { - for (auto & p : peers) + for (auto& p : peers) { - if(p.completedLedgers == 0) - p.relay(Validation{p.id, p.LCL(), p.LCL()}); + if (p.completedLedgers == 0) + p.relay(Validation{p.id, p.prevLedgerID(), p.prevLedgerID()}); p.targetLedgers = p.completedLedgers + ledgers; p.start(); } @@ -93,12 +93,10 @@ public: std::vector peers; BasicNetwork net; - }; - -} // csf -} // test -} // ripple +} // csf +} // test +} // ripple #endif diff --git a/src/test/csf/Tx.h b/src/test/csf/Tx.h index e3a540e6c..cb33b6552 100644 --- a/src/test/csf/Tx.h +++ b/src/test/csf/Tx.h @@ -21,9 +21,9 @@ #include #include +#include #include #include -#include namespace ripple { namespace test { @@ -35,7 +35,9 @@ class Tx public: using ID = std::uint32_t; - Tx(ID i) : id_{ i } {} + Tx(ID i) : id_{i} + { + } ID id() const @@ -44,21 +46,19 @@ public: } bool - operator<(Tx const & o) const + operator<(Tx const& o) const { return id_ < o.id_; } bool - operator==(Tx const & o) const + operator==(Tx const& o) const { return id_ == o.id_; } - private: ID id_; - }; //!------------------------------------------------------------------------- @@ -74,37 +74,39 @@ public: using MutableTxSet = TxSet; TxSet() = default; - TxSet(TxSetType const & s) : txs_{ s } {} + TxSet(TxSetType const& s) : txs_{s} + { + } bool - insert(Tx const & t) + insert(Tx const& t) { return txs_.insert(t).second; } bool - erase(Tx::ID const & txId) + erase(Tx::ID const& txId) { - return txs_.erase(Tx{ txId }) > 0; + return txs_.erase(Tx{txId}) > 0; } bool exists(Tx::ID const txId) const { - auto it = txs_.find(Tx{ txId }); + auto it = txs_.find(Tx{txId}); return it != txs_.end(); } - Tx const * + Tx const* find(Tx::ID const& txId) const { - auto it = txs_.find(Tx{ txId }); + auto it = txs_.find(Tx{txId}); if (it != txs_.end()) return &(*it); return nullptr; } - auto const & + auto const& id() const { return txs_; @@ -119,19 +121,14 @@ public: { std::map res; - auto populate_diffs = [&res](auto const & a, auto const & b, bool s) - { - auto populator = [&](auto const & tx) - { - res[tx.id()] = s; - }; + auto populate_diffs = [&res](auto const& a, auto const& b, bool s) { + auto populator = [&](auto const& tx) { res[tx.id()] = s; }; std::set_difference( - a.begin(), a.end(), - b.begin(), b.end(), - boost::make_function_output_iterator( - std::ref(populator) - ) - ); + a.begin(), + a.end(), + b.begin(), + b.end(), + boost::make_function_output_iterator(std::ref(populator))); }; populate_diffs(txs_, other.txs_, true); @@ -143,55 +140,35 @@ public: TxSetType txs_; }; - -/** The RCL consensus process catches missing node SHAMap error - in several points. This exception is meant to represent a similar - case for the unit test. -*/ -class MissingTx : public std::runtime_error -{ -public: - MissingTx() - : std::runtime_error("MissingTx") - {} -}; - - //------------------------------------------------------------------------------ // Helper functions for debug printing -inline -std::ostream& -operator<<(std::ostream & o, const Tx & t) +inline std::ostream& +operator<<(std::ostream& o, const Tx& t) { return o << t.id(); } template -inline -std::ostream& -operator<<(std::ostream & o, boost::container::flat_set const & ts) +inline std::ostream& +operator<<(std::ostream& o, boost::container::flat_set const& ts) { o << "{ "; bool do_comma = false; - for (auto const & t : ts) + for (auto const& t : ts) { if (do_comma) o << ", "; else do_comma = true; o << t; - - } o << " }"; return o; - } -inline -std::string -to_string(TxSetType const & txs) +inline std::string +to_string(TxSetType const& txs) { std::stringstream ss; ss << txs; @@ -199,23 +176,15 @@ to_string(TxSetType const & txs) } template -inline -void -hash_append(Hasher& h, Tx const & tx) +inline void +hash_append(Hasher& h, Tx const& tx) { using beast::hash_append; hash_append(h, tx.id()); } -std::ostream& -operator<<(std::ostream & o, MissingTx const &m) -{ - return o << m.what(); -} - - -} // csf -} // test -} // ripple +} // csf +} // test +} // ripple #endif diff --git a/src/test/csf/UNL.h b/src/test/csf/UNL.h index 554c1c194..4f293a860 100644 --- a/src/test/csf/UNL.h +++ b/src/test/csf/UNL.h @@ -22,10 +22,10 @@ #include #include -#include -#include -#include #include +#include +#include +#include namespace ripple { namespace test { @@ -42,7 +42,7 @@ namespace csf { */ template std::vector -random_weighted_shuffle(std::vector v, std::vector w, G & g) +random_weighted_shuffle(std::vector v, std::vector w, G& g) { using std::swap; @@ -57,7 +57,6 @@ random_weighted_shuffle(std::vector v, std::vector w, G & g) return v; } - /** Power-law distribution with PDF P(x) = (x/xmin)^-a @@ -69,25 +68,22 @@ class PowerLawDistribution double xmin_; double a_; double inv_; - std::uniform_real_distribution uf_{0,1}; + std::uniform_real_distribution uf_{0, 1}; public: - PowerLawDistribution(double xmin, double a) - : xmin_{xmin}, a_{a} - { - inv_ = 1.0/(1.0 - a_); - } + PowerLawDistribution(double xmin, double a) : xmin_{xmin}, a_{a} + { + inv_ = 1.0 / (1.0 - a_); + } template - inline - double - operator()(Generator & g) + inline double + operator()(Generator& g) { // use inverse transform of CDF to sample // CDF is P(X <= x): 1 - (x/xmin)^(1-a) return xmin_ * std::pow(1 - uf_(g), inv_); } - }; //< Unique identifier for each node in the network @@ -110,25 +106,23 @@ class TrustGraph std::vector UNLs_; std::vector assignment_; -public: +public: //< Constructor TrustGraph(std::vector UNLs, std::vector assignment) - : UNLs_{UNLs} - , assignment_{assignment} - {} + : UNLs_{UNLs}, assignment_{assignment} + { + } //< Whether node `i` trusts node `j` - inline - bool + inline bool trusts(PeerID i, PeerID j) const { return unl(i).find(j) != unl(i).end(); } //< Get the UNL for node `i` - inline - UNL const & + inline UNL const& unl(PeerID i) const { return UNLs_[assignment_[i]]; @@ -138,7 +132,6 @@ public: bool canFork(double quorum) const; - auto numPeers() const { @@ -147,7 +140,7 @@ public: //< Save grapviz dot file reprentation of the trust graph void - save_dot(std::string const & fileName); + save_dot(std::string const& fileName); /** Generate a random trust graph based on random ranking of peers @@ -176,28 +169,23 @@ public: */ template - static - TrustGraph - makeRandomRanked(int size, - int numUNLs, - RankPDF rankPDF, - SizePDF unlSizePDF, - Generator & g) + static TrustGraph + makeRandomRanked( + int size, + int numUNLs, + RankPDF rankPDF, + SizePDF unlSizePDF, + Generator& g) { - - // 1. Generate ranks std::vector weights(size); - std::generate(weights.begin(), weights.end(), [&]() - { - return rankPDF(g); - }); + std::generate( + weights.begin(), weights.end(), [&]() { return rankPDF(g); }); // 2. Generate UNLs based on sampling without replacement according // to weights std::vector unls(numUNLs); - std::generate(unls.begin(), unls.end(), [&]() - { + std::generate(unls.begin(), unls.end(), [&]() { std::vector ids(size); std::iota(ids.begin(), ids.end(), 0); auto res = random_weighted_shuffle(ids, weights, g); @@ -206,12 +194,9 @@ public: // 3. Assign membership std::vector assignment(size); - std::uniform_int_distribution u(0, numUNLs-1); - std::generate(assignment.begin(), assignment.end(), - [&]() - { - return u(g); - }); + std::uniform_int_distribution u(0, numUNLs - 1); + std::generate( + assignment.begin(), assignment.end(), [&]() { return u(g); }); return TrustGraph(unls, assignment); } @@ -226,8 +211,7 @@ public: @param size The number of nodes in the trust graph @param overlap The number of nodes trusting both cliques */ - static - TrustGraph + static TrustGraph makeClique(int size, int overlap); /** Generate a complete (fully-connect) trust graph @@ -237,21 +221,17 @@ public: @param size The number of nodes in the trust graph */ - static - TrustGraph + static TrustGraph makeComplete(int size); }; - - //< Make the TrustGraph into a topology with delays given by DelayModel template auto -topology(TrustGraph const & tg, DelayModel const & d) +topology(TrustGraph const& tg, DelayModel const& d) { - return [&](PeerID i, PeerID j) - { - return tg.trusts(i,j) ? boost::make_optional(d(i,j)) : boost::none; + return [&](PeerID i, PeerID j) { + return tg.trusts(i, j) ? boost::make_optional(d(i, j)) : boost::none; }; } @@ -260,18 +240,19 @@ class fixed std::chrono::nanoseconds d_; public: - fixed(std::chrono::nanoseconds const & d) : d_{d} {} + fixed(std::chrono::nanoseconds const& d) : d_{d} + { + } - inline - std::chrono::nanoseconds - operator()(PeerID const & i, PeerID const & j) const + inline std::chrono::nanoseconds + operator()(PeerID const& i, PeerID const& j) const { return d_; } }; -} // csf -} // test -} // ripple +} // csf +} // test +} // ripple #endif diff --git a/src/test/csf/impl/UNL.cpp b/src/test/csf/impl/UNL.cpp index a1d17cdcd..7b9207eb3 100644 --- a/src/test/csf/impl/UNL.cpp +++ b/src/test/csf/impl/UNL.cpp @@ -16,10 +16,10 @@ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ //============================================================================== -#include #include -#include #include +#include +#include namespace ripple { namespace test { @@ -38,8 +38,8 @@ TrustGraph::canFork(double quorum) const for (int i = 0; i < assignment_.size(); ++i) { - auto const & myUNL = UNLs_[assignment_[i]]; - if(myUNL.find(i) == myUNL.end()) + auto const& myUNL = UNLs_[assignment_[i]]; + if (myUNL.find(i) == myUNL.end()) { auto myUNLcopy = myUNL; myUNLcopy.insert(i); @@ -50,21 +50,20 @@ TrustGraph::canFork(double quorum) const // Loop over all pairs of uniqueUNLs for (int i = 0; i < uniqueUNLs.size(); ++i) { - for (int j = (i+1); j < uniqueUNLs.size(); ++j) + for (int j = (i + 1); j < uniqueUNLs.size(); ++j) { - auto const & unlA = uniqueUNLs[i]; - auto const & unlB = uniqueUNLs[j]; + auto const& unlA = uniqueUNLs[i]; + auto const& unlB = uniqueUNLs[j]; - double rhs = 2.0*(1.-quorum) * - std::max(unlA.size(), unlB.size() ); + double rhs = + 2.0 * (1. - quorum) * std::max(unlA.size(), unlB.size()); - int intersectionSize = std::count_if(unlA.begin(), unlA.end(), - [&](PeerID id) - { + int intersectionSize = + std::count_if(unlA.begin(), unlA.end(), [&](PeerID id) { return unlB.find(id) != unlB.end(); }); - if(intersectionSize < rhs) + if (intersectionSize < rhs) return true; } } @@ -80,56 +79,53 @@ TrustGraph::makeClique(int size, int overlap) // Clique A has nodes [0,endA) and Clique B has [startB,numPeers) // Note: Clique B will have an extra peer when numPeers - overlap // is odd - int endA = (size + overlap)/2; - int startB = (size - overlap)/2; + int endA = (size + overlap) / 2; + int startB = (size - overlap) / 2; std::vector unls; unls.emplace_back(bci(0), bci(endA)); unls.emplace_back(bci(startB), bci(size)); unls.emplace_back(bci(0), bci(size)); - std::vector assignment(size,0); + std::vector assignment(size, 0); for (int i = 0; i < size; ++i) { - if(i < startB) + if (i < startB) assignment[i] = 0; - else if(i > endA) + else if (i > endA) assignment[i] = 1; else assignment[i] = 2; } - return TrustGraph(unls, assignment); } TrustGraph TrustGraph::makeComplete(int size) { - UNL all{ boost::counting_iterator( 0 ), - boost::counting_iterator( size ) }; + UNL all{boost::counting_iterator(0), + boost::counting_iterator(size)}; - return TrustGraph(std::vector(1,all), - std::vector(size, 0)); + return TrustGraph(std::vector(1, all), std::vector(size, 0)); } -inline void TrustGraph::save_dot(std::string const & fileName) +inline void +TrustGraph::save_dot(std::string const& fileName) { std::ofstream out(fileName); out << "digraph {\n"; for (int i = 0; i < assignment_.size(); ++i) { - for (auto & j : UNLs_[assignment_[i]]) + for (auto& j : UNLs_[assignment_[i]]) { out << i << " -> " << j << ";\n"; } - } out << "}\n"; - } -} // csf -} // test -} // ripple \ No newline at end of file +} // csf +} // test +} // ripple