diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 4749113e3..b2856ed1c 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -11,6 +11,34 @@ If you are using Red Hat Enterprise Linux 7 or CentOS 7, you can [update using ` # Releases +## Version 0.81.0 + +The `rippled` 0.81.0 release introduces changes that improve the scalability of the XRP Ledger and transitions the recommended validator configuration to a new hosted site, as described in Ripple's [Decentralization Strategy Update](https://ripple.com/dev-blog/decentralization-strategy-update/) post. + +**New and Updated Features** + +- New hosted validator configuration. + + +**Bug Fixes** + +- Optimize queries for account_tx to work around SQLite query planner ([#2312](https://github.com/ripple/rippled/issues/2312)) + + +## Version 0.80.2 + +The `rippled` 0.80.2 release introduces changes that improve the scalability of the XRP Ledger. + +**New and Updated Features** + +This release has no new features. + +**Bug Fixes** + +- Do not dispatch a transaction received from a peer for processing if it has already been dispatched within the past ten seconds. +- Increase the number of transaction handlers that can be in flight in the job queue and decrease the relative cost for peers to share transaction and ledger data. +- Make better use of resources by adjusting the number of threads we initialize, by reverting commit [#68b8ffd](https://github.com/ripple/rippled/commit/68b8ffdb638d07937f841f7217edeb25efdb3b5d). + ## Version 0.80.1 The `rippled` 0.80.1 release provides several enhancements in support of published validator lists and corrects several bugs. diff --git a/doc/validators-example.txt b/doc/validators-example.txt index 57f2f9a9e..e18d9c3fb 100644 --- a/doc/validators-example.txt +++ b/doc/validators-example.txt @@ -45,10 +45,11 @@ # 02dd8b7075f64d77d9d2bdb88da364f29fcd975f9ea6f21894abcc7564efda8054 # -# Public keys of the validators that this rippled instance trusts. -[validators] -n949f75evCHwgyP4fPVgaHqNHxUVN15PsJEZ3B3HnXPcPjcZAoy7 -n9MD5h24qrQqiyBC8aeqqCWvpiBiYQ3jxSr91uiDvmrkyHRdYLUj -n9L81uNCaPgtUJfaHh89gmdvXKAmSt5Gdsw2g1iPWaPkAHW5Nm4C -n9KiYM9CgngLvtRCQHZwgC2gjpdaZcCcbt3VboxiNFcKuwFVujzS -n9LdgEtkmGB9E2h3K4Vp7iGUaKuq23Zr32ehxiU8FWY7xoxbWTSA +# The default validator list publishers that the rippled instance +# trusts + +[validator_list_sites] +https://vl.ripple.com + +[validator_list_keys] +ED2677ABFFD1B33AC6FBC3062B71F1E8397C1505E1C42C64D11AD1B28FF73F4734 diff --git a/src/ripple/app/main/Application.cpp b/src/ripple/app/main/Application.cpp index 40b3311b5..f1bf083ca 100644 --- a/src/ripple/app/main/Application.cpp +++ b/src/ripple/app/main/Application.cpp @@ -1042,9 +1042,7 @@ private: bool ApplicationImp::setup() { // VFALCO NOTE: 0 means use heuristics to determine the thread count. - m_jobQueue->setThreadCount (config_->WORKERS, config_->standalone(), - config_->exists (SECTION_VALIDATOR_TOKEN) || - config_->exists (SECTION_VALIDATION_SEED)); + m_jobQueue->setThreadCount (config_->WORKERS, config_->standalone()); // We want to intercept and wait for CTRL-C to terminate the process m_signals.add (SIGINT); diff --git a/src/ripple/app/misc/HashRouter.cpp b/src/ripple/app/misc/HashRouter.cpp index 8e4071260..42a846a41 100644 --- a/src/ripple/app/misc/HashRouter.cpp +++ b/src/ripple/app/misc/HashRouter.cpp @@ -71,6 +71,18 @@ bool HashRouter::addSuppressionPeer (uint256 const& key, PeerShortID peer, int& return result.second; } +bool HashRouter::shouldProcess (uint256 const& key, PeerShortID peer, int& flags, + Stopwatch::time_point now, std::chrono::seconds interval) +{ + std::lock_guard lock (mutex_); + + auto result = emplace(key); + auto& s = result.first; + s.addPeer (peer); + flags = s.getFlags (); + return s.shouldProcess (now, interval); +} + int HashRouter::getFlags (uint256 const& key) { std::lock_guard lock (mutex_); diff --git a/src/ripple/app/misc/HashRouter.h b/src/ripple/app/misc/HashRouter.h index a40d3558b..aa27b255b 100644 --- a/src/ripple/app/misc/HashRouter.h +++ b/src/ripple/app/misc/HashRouter.h @@ -118,12 +118,21 @@ private: return ++recoveries_ % limit != 0; } + bool shouldProcess(Stopwatch::time_point now, std::chrono::seconds interval) + { + if (processed_ && ((*processed_ + interval) > now)) + return false; + processed_.emplace (now); + return true; + } + private: int flags_ = 0; std::set peers_; // This could be generalized to a map, if more // than one flag needs to expire independently. boost::optional relayed_; + boost::optional processed_; std::uint32_t recoveries_ = 0; }; @@ -161,6 +170,10 @@ public: bool addSuppressionPeer (uint256 const& key, PeerShortID peer, int& flags); + // Add a peer suppression and return whether the entry should be processed + bool shouldProcess (uint256 const& key, PeerShortID peer, + int& flags, Stopwatch::time_point now, std::chrono::seconds interval); + /** Set the flags on a hash. @return `true` if the flags were changed. `false` if unchanged. diff --git a/src/ripple/app/misc/impl/AccountTxPaging.cpp b/src/ripple/app/misc/impl/AccountTxPaging.cpp index 6d4bad2ef..6ce0ed901 100644 --- a/src/ripple/app/misc/impl/AccountTxPaging.cpp +++ b/src/ripple/app/misc/impl/AccountTxPaging.cpp @@ -142,19 +142,27 @@ accountTxPage ( } else if (forward && (findLedger != 0)) { + auto b58acct = idCache.toBase58(account); sql = boost::str (boost::format( - prefix + - (R"( - AccountTransactions.LedgerSeq BETWEEN '%u' AND '%u' OR - ( AccountTransactions.LedgerSeq = '%u' AND - AccountTransactions.TxnSeq >= '%u' ) + (R"(SELECT AccountTransactions.LedgerSeq,AccountTransactions.TxnSeq, + Status,RawTxn,TxnMeta + FROM AccountTransactions, Transactions WHERE + (AccountTransactions.TransID = Transactions.TransID AND + AccountTransactions.Account = '%s' AND + AccountTransactions.LedgerSeq BETWEEN '%u' AND '%u') + OR + (AccountTransactions.TransID = Transactions.TransID AND + AccountTransactions.Account = '%s' AND + AccountTransactions.LedgerSeq = '%u' AND + AccountTransactions.TxnSeq >= '%u') ORDER BY AccountTransactions.LedgerSeq ASC, AccountTransactions.TxnSeq ASC LIMIT %u; )")) - % idCache.toBase58(account) + % b58acct % (findLedger + 1) % maxLedger + % b58acct % findLedger % findSeq % queryLimit); @@ -174,17 +182,27 @@ accountTxPage ( } else if (!forward && (findLedger != 0)) { + auto b58acct = idCache.toBase58(account); sql = boost::str (boost::format( - prefix + - (R"(AccountTransactions.LedgerSeq BETWEEN '%u' AND '%u' OR - (AccountTransactions.LedgerSeq = '%u' AND - AccountTransactions.TxnSeq <= '%u') - ORDER BY AccountTransactions.LedgerSeq DESC, - AccountTransactions.TxnSeq DESC - LIMIT %u;)")) - % idCache.toBase58(account) + (R"(SELECT AccountTransactions.LedgerSeq,AccountTransactions.TxnSeq, + Status,RawTxn,TxnMeta + FROM AccountTransactions, Transactions WHERE + (AccountTransactions.TransID = Transactions.TransID AND + AccountTransactions.Account = '%s' AND + AccountTransactions.LedgerSeq BETWEEN '%u' AND '%u') + OR + (AccountTransactions.TransID = Transactions.TransID AND + AccountTransactions.Account = '%s' AND + AccountTransactions.LedgerSeq = '%u' AND + AccountTransactions.TxnSeq <= '%u') + ORDER BY AccountTransactions.LedgerSeq DESC, + AccountTransactions.TxnSeq DESC + LIMIT %u; + )")) + % b58acct % minLedger % (findLedger - 1) + % b58acct % findLedger % findSeq % queryLimit); diff --git a/src/ripple/core/JobQueue.h b/src/ripple/core/JobQueue.h index b5c9228e8..303a5542b 100644 --- a/src/ripple/core/JobQueue.h +++ b/src/ripple/core/JobQueue.h @@ -180,8 +180,7 @@ public: /** Set the number of thread serving the job queue to precisely this number. */ - void setThreadCount (int c, bool const standaloneMode, - bool const validator=true); + void setThreadCount (int c, bool const standaloneMode); /** Return a scoped LoadEvent. */ diff --git a/src/ripple/core/impl/JobQueue.cpp b/src/ripple/core/impl/JobQueue.cpp index 414624bbc..1ff74e342 100644 --- a/src/ripple/core/impl/JobQueue.cpp +++ b/src/ripple/core/impl/JobQueue.cpp @@ -153,8 +153,7 @@ JobQueue::getJobCountGE (JobType t) const } void -JobQueue::setThreadCount (int c, bool const standaloneMode, - bool const validator) +JobQueue::setThreadCount (int c, bool const standaloneMode) { if (standaloneMode) { @@ -163,13 +162,9 @@ JobQueue::setThreadCount (int c, bool const standaloneMode, else if (c == 0) { c = static_cast(std::thread::hardware_concurrency()); - if (validator) - c = 2 + std::min(c, 4); // I/O will bottleneck - else - c *= 2; // Tested to improve stability under high RPC load. + c = 2 + std::min(c, 4); // I/O will bottleneck JLOG (m_journal.info()) << "Auto-tuning to " << c << - " validation/transaction/proposal threads for " << - (validator ? "" : "non-") << "validator."; + " validation/transaction/proposal threads."; } else { diff --git a/src/ripple/overlay/impl/PeerImp.cpp b/src/ripple/overlay/impl/PeerImp.cpp index e05736309..0cdc63465 100644 --- a/src/ripple/overlay/impl/PeerImp.cpp +++ b/src/ripple/overlay/impl/PeerImp.cpp @@ -1055,8 +1055,9 @@ PeerImp::onMessage (std::shared_ptr const& m) int flags; - if (! app_.getHashRouter ().addSuppressionPeer ( - txID, id_, flags)) + constexpr std::chrono::seconds tx_interval = 10s; + if (! app_.getHashRouter ().shouldProcess ( + txID, id_, flags, clock_type::now(), tx_interval)) { // we have seen this transaction recently if (flags & SF_BAD) @@ -1064,8 +1065,9 @@ PeerImp::onMessage (std::shared_ptr const& m) fee_ = Resource::feeInvalidSignature; JLOG(p_journal_.debug()) << "Ignoring known bad tx " << txID; - return; } + + return; } JLOG(p_journal_.debug()) << "Got tx " << txID; @@ -1088,7 +1090,9 @@ PeerImp::onMessage (std::shared_ptr const& m) } } - if (app_.getJobQueue().getJobCount(jtTRANSACTION) > 100) + // The maximum number of transactions to have in the job queue. + constexpr int max_transactions = 250; + if (app_.getJobQueue().getJobCount(jtTRANSACTION) > max_transactions) { JLOG(p_journal_.info()) << "Transaction queue is full"; } diff --git a/src/ripple/resource/impl/Fees.cpp b/src/ripple/resource/impl/Fees.cpp index b1d03526b..0ca4e4b41 100644 --- a/src/ripple/resource/impl/Fees.cpp +++ b/src/ripple/resource/impl/Fees.cpp @@ -23,31 +23,24 @@ namespace ripple { namespace Resource { -Charge const feeInvalidRequest ( 10, "malformed request" ); -Charge const feeRequestNoReply ( 1, "unsatisfiable request" ); -Charge const feeInvalidSignature ( 100, "invalid signature" ); -Charge const feeUnwantedData ( 15, "useless data" ); -Charge const feeBadData ( 20, "invalid data" ); +Charge const feeInvalidRequest ( 100, "malformed request" ); +Charge const feeRequestNoReply ( 10, "unsatisfiable request" ); +Charge const feeInvalidSignature ( 1000, "invalid signature" ); +Charge const feeUnwantedData ( 150, "useless data" ); +Charge const feeBadData ( 200, "invalid data" ); -Charge const feeInvalidRPC ( 10, "malformed RPC" ); -Charge const feeReferenceRPC ( 2, "reference RPC" ); -Charge const feeExceptionRPC ( 10, "exceptioned RPC" ); -Charge const feeLightRPC ( 5, "light RPC" ); // DAVID: Check the cost -Charge const feeLowBurdenRPC ( 20, "low RPC" ); -Charge const feeMediumBurdenRPC ( 40, "medium RPC" ); -Charge const feeHighBurdenRPC ( 300, "heavy RPC" ); +Charge const feeInvalidRPC ( 100, "malformed RPC" ); +Charge const feeReferenceRPC ( 20, "reference RPC" ); +Charge const feeExceptionRPC ( 100, "exceptioned RPC" ); +Charge const feeMediumBurdenRPC ( 400, "medium RPC" ); +Charge const feeHighBurdenRPC ( 3000, "heavy RPC" ); -Charge const feeLightPeer (1, "trivial peer request" ); -Charge const feeLowBurdenPeer (2, "simple peer request" ); -Charge const feeMediumBurdenPeer (50, "moderate peer request" ); -Charge const feeHighBurdenPeer (250, "heavy peer request" ); +Charge const feeLightPeer ( 1, "trivial peer request" ); +Charge const feeMediumBurdenPeer ( 250, "moderate peer request" ); +Charge const feeHighBurdenPeer ( 2000, "heavy peer request" ); -Charge const feeNewTrustedNote ( 10, "trusted note" ); -Charge const feeNewValidTx ( 10, "valid tx" ); -Charge const feeSatisfiedRequest ( 10, "needed data" ); - -Charge const feeWarning ( 200, "received warning" ); -Charge const feeDrop ( 300, "dropped" ); +Charge const feeWarning ( 2000, "received warning" ); +Charge const feeDrop ( 3000, "dropped" ); } } diff --git a/src/ripple/resource/impl/Tuning.h b/src/ripple/resource/impl/Tuning.h index 894bd8d09..0752b8ad1 100644 --- a/src/ripple/resource/impl/Tuning.h +++ b/src/ripple/resource/impl/Tuning.h @@ -29,17 +29,17 @@ namespace Resource { enum { // Balance at which a warning is issued - warningThreshold = 500 + warningThreshold = 5000 // Balance at which the consumer is disconnected - ,dropThreshold = 1500 + ,dropThreshold = 15000 // The number of seconds in the exponential decay window // (This should be a power of two) ,decayWindowSeconds = 32 // The minimum balance required in order to include a load source in gossip - ,minimumGossipBalance = 100 + ,minimumGossipBalance = 1000 }; // The number of seconds until an inactive table item is removed