diff --git a/Builds/VisualStudio2013/RippleD.vcxproj b/Builds/VisualStudio2013/RippleD.vcxproj index e7c35e9ba..4e9d4be32 100644 --- a/Builds/VisualStudio2013/RippleD.vcxproj +++ b/Builds/VisualStudio2013/RippleD.vcxproj @@ -1806,8 +1806,6 @@ - - diff --git a/Builds/VisualStudio2013/RippleD.vcxproj.filters b/Builds/VisualStudio2013/RippleD.vcxproj.filters index 2801473b9..8732e7b06 100644 --- a/Builds/VisualStudio2013/RippleD.vcxproj.filters +++ b/Builds/VisualStudio2013/RippleD.vcxproj.filters @@ -2778,9 +2778,6 @@ protobuf\vsprojects - - ripple\algorithm\api - ripple\algorithm\api diff --git a/src/beast/beast/container/detail/aged_ordered_container.h b/src/beast/beast/container/detail/aged_ordered_container.h index 8fc672f1b..38830aa37 100644 --- a/src/beast/beast/container/detail/aged_ordered_container.h +++ b/src/beast/beast/container/detail/aged_ordered_container.h @@ -1503,8 +1503,8 @@ operator[] (Key const& key) element* const p (new_element ( std::piecewise_construct, std::forward_as_tuple (key), std::forward_as_tuple ())); - chronological.list.push_back (*p); m_cont.insert_commit (*p, d); + chronological.list.push_back (*p); return p->value.second; } return result.first->value.second; @@ -1526,8 +1526,8 @@ operator[] (Key&& key) std::piecewise_construct, std::forward_as_tuple (std::move (key)), std::forward_as_tuple ())); - chronological.list.push_back (*p); m_cont.insert_commit (*p, d); + chronological.list.push_back (*p); return p->value.second; } return result.first->value.second; @@ -1564,8 +1564,8 @@ insert (value_type const& value) -> if (result.second) { element* const p (new_element (value)); - chronological.list.push_back (*p); auto const iter (m_cont.insert_commit (*p, d)); + chronological.list.push_back (*p); return std::make_pair (iterator (iter), true); } return std::make_pair (iterator (result.first), false); @@ -1605,8 +1605,8 @@ insert (value_type&& value) -> if (result.second) { element* const p (new_element (std::move (value))); - chronological.list.push_back (*p); auto const iter (m_cont.insert_commit (*p, d)); + chronological.list.push_back (*p); return std::make_pair (iterator (iter), true); } return std::make_pair (iterator (result.first), false); @@ -1648,8 +1648,8 @@ insert (const_iterator hint, value_type const& value) -> if (result.second) { element* const p (new_element (value)); - chronological.list.push_back (*p); auto const iter (m_cont.insert_commit (*p, d)); + chronological.list.push_back (*p); return iterator (iter); } return iterator (result.first); @@ -1671,8 +1671,8 @@ insert (const_iterator hint, value_type&& value) -> if (result.second) { element* const p (new_element (std::move (value))); - chronological.list.push_back (*p); auto const iter (m_cont.insert_commit (*p, d)); + chronological.list.push_back (*p); return iterator (iter); } return iterator (result.first); @@ -1697,8 +1697,8 @@ emplace (Args&&... args) -> std::cref (m_config.key_compare()), d)); if (result.second) { - chronological.list.push_back (*p); auto const iter (m_cont.insert_commit (*p, d)); + chronological.list.push_back (*p); return std::make_pair (iterator (iter), true); } delete_element (p); @@ -1743,8 +1743,8 @@ emplace_hint (const_iterator hint, Args&&... args) -> extract (p->value), std::cref (m_config.key_compare()), d)); if (result.second) { - chronological.list.push_back (*p); auto const iter (m_cont.insert_commit (*p, d)); + chronological.list.push_back (*p); return std::make_pair (iterator (iter), true); } delete_element (p); diff --git a/src/beast/beast/container/detail/aged_unordered_container.h b/src/beast/beast/container/detail/aged_unordered_container.h index ca2355df6..b1c44f7c9 100644 --- a/src/beast/beast/container/detail/aged_unordered_container.h +++ b/src/beast/beast/container/detail/aged_unordered_container.h @@ -2057,8 +2057,8 @@ operator[] (Key const& key) std::piecewise_construct, std::forward_as_tuple (key), std::forward_as_tuple ())); - chronological.list.push_back (*p); m_cont.insert_commit (*p, d); + chronological.list.push_back (*p); return p->value.second; } return result.first->value.second; @@ -2083,8 +2083,8 @@ operator[] (Key&& key) std::piecewise_construct, std::forward_as_tuple (std::move (key)), std::forward_as_tuple ())); - chronological.list.push_back (*p); m_cont.insert_commit (*p, d); + chronological.list.push_back (*p); return p->value.second; } return result.first->value.second; @@ -2126,8 +2126,8 @@ insert (value_type const& value) -> if (result.second) { element* const p (new_element (value)); - chronological.list.push_back (*p); auto const iter (m_cont.insert_commit (*p, d)); + chronological.list.push_back (*p); return std::make_pair (iterator (iter), true); } return std::make_pair (iterator (result.first), false); @@ -2170,8 +2170,8 @@ insert (value_type&& value) -> if (result.second) { element* const p (new_element (std::move (value))); - chronological.list.push_back (*p); auto const iter (m_cont.insert_commit (*p, d)); + chronological.list.push_back (*p); return std::make_pair (iterator (iter), true); } return std::make_pair (iterator (result.first), false); @@ -2195,6 +2195,32 @@ insert (value_type&& value) -> return iterator (iter); } +#if 1 // Use insert() instead of insert_check() insert_commit() +// set, map +template +template +auto +aged_unordered_container :: +emplace (Args&&... args) -> + typename std::enable_if >::type +{ + maybe_rehash (1); + // VFALCO NOTE Its unfortunate that we need to + // construct element here + element* const p (new_element (std::forward (args)...)); + auto const result (m_cont.insert (*p)); + if (result.second) + { + chronological.list.push_back (*p); + return std::make_pair (iterator (result.first), true); + } + delete_element (p); + return std::make_pair (iterator (result.first), false); +} +#else // As original, use insert_check() / insert_commit () pair. // set, map template @@ -2217,13 +2243,14 @@ emplace (Args&&... args) -> std::cref (m_config.key_value_equal()), d)); if (result.second) { - chronological.list.push_back (*p); auto const iter (m_cont.insert_commit (*p, d)); + chronological.list.push_back (*p); return std::make_pair (iterator (iter), true); } delete_element (p); return std::make_pair (iterator (result.first), false); } +#endif // 0 // multiset, multimap template std::cref (m_config.key_value_equal()), d)); if (result.second) { - chronological.list.push_back (*p); auto const iter (m_cont.insert_commit (*p, d)); + chronological.list.push_back (*p); return std::make_pair (iterator (iter), true); } delete_element (p); @@ -2458,8 +2485,8 @@ insert_unchecked (value_type const& value) -> if (result.second) { element* const p (new_element (value)); - chronological.list.push_back (*p); auto const iter (m_cont.insert_commit (*p, d)); + chronological.list.push_back (*p); return std::make_pair (iterator (iter), true); } return std::make_pair (iterator (result.first), false); diff --git a/src/ripple/algorithm/api/CycledSet.h b/src/ripple/algorithm/api/CycledSet.h deleted file mode 100644 index 5b9df948c..000000000 --- a/src/ripple/algorithm/api/CycledSet.h +++ /dev/null @@ -1,116 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_TYPES_CYCLEDSET_H_INCLUDED -#define RIPPLE_TYPES_CYCLEDSET_H_INCLUDED - -#include -#include - -namespace ripple { - -/** Cycled set of unique keys. - This provides a system of remembering a set of keys, with aging. Two - containers are kept. When one container fills, the other is cleared - and a swap is performed. A key is considered present if it is in either - container. -*/ -template , - class KeyEqual = std::equal_to , - class Allocator = std::allocator > -class CycledSet -{ -private: - // HH This unordered_set can't be changed from boost until gcc allows for - // stateful hash functions (or until rippled eliminates stateful hash - // functions). - typedef boost::unordered_set< - Key, Hash, KeyEqual, Allocator> ContainerType; - typedef typename ContainerType::iterator iterator; - -public: - typedef typename ContainerType::key_type key_type; - typedef typename ContainerType::value_type value_type; - typedef typename ContainerType::size_type size_type; - typedef typename ContainerType::difference_type difference_type; - typedef typename ContainerType::hasher hasher; - typedef typename ContainerType::key_equal key_equal; - typedef typename ContainerType::allocator_type allocator_type; - typedef typename ContainerType::reference reference; - typedef typename ContainerType::const_reference const_reference; - typedef typename ContainerType::pointer pointer; - typedef typename ContainerType::const_pointer const_pointer; - - explicit CycledSet ( - size_type item_max = 0, // 0 means no limit - Hash hash = Hash(), - KeyEqual equal = KeyEqual(), - Allocator alloc = Allocator()) - : m_max (item_max) - , m_hash (hash) - , m_equal (equal) - , m_alloc (alloc) - , m_front (m_max, hash, equal, alloc) - , m_back (m_max, hash, equal, alloc) - { - } - - // Returns `true` if the next real insert would swap - bool full() const - { - return (m_max != 0) && m_front.size() >= m_max; - } - - // Adds the key to the front if its not in either map - bool insert (key_type const& key) - { - if (full()) - cycle (); - if (m_back.find (key) != m_back.end()) - return false; - std::pair result ( - m_front.insert (key)); - if (result.second) - return true; - return false; - } - - void cycle () - { - std::swap (m_front, m_back); - m_front.clear (); - -#if BOOST_VERSION > 105400 - m_front.reserve (m_max); -#endif - } - -private: - size_type m_max; - hasher m_hash; - key_equal m_equal; - allocator_type m_alloc; - ContainerType m_front; - ContainerType m_back; -}; - -} - -#endif diff --git a/src/ripple/module/app/ledger/LedgerMaster.cpp b/src/ripple/module/app/ledger/LedgerMaster.cpp index 460f9e1c1..dc4f1efe2 100644 --- a/src/ripple/module/app/ledger/LedgerMaster.cpp +++ b/src/ripple/module/app/ledger/LedgerMaster.cpp @@ -642,7 +642,7 @@ public: // { if (isCurrent) - getApp ().getValidators ().ledgerClosed (ledger->getHash()); + getApp ().getValidators ().on_ledger_closed (ledger->getHash()); } // //-------------------------------------------------------------------------- diff --git a/src/ripple/module/app/tx/TransactionMeta.h b/src/ripple/module/app/tx/TransactionMeta.h index 767e59e12..c5cfc5da9 100644 --- a/src/ripple/module/app/tx/TransactionMeta.h +++ b/src/ripple/module/app/tx/TransactionMeta.h @@ -107,7 +107,7 @@ public: bool hasDeliveredAmount () const { - return bool(mDelivered); + return static_cast (mDelivered); } static bool thread (STObject& node, uint256 const& prevTxID, std::uint32_t prevLgrID); diff --git a/src/ripple/overlay/impl/PeerImp.h b/src/ripple/overlay/impl/PeerImp.h index e7f1cee0f..107fca09f 100644 --- a/src/ripple/overlay/impl/PeerImp.h +++ b/src/ripple/overlay/impl/PeerImp.h @@ -170,7 +170,7 @@ public: boost::optional http_message_; boost::optional http_parser_; message_stream message_stream_; - + boost::asio::streambuf write_buffer_; bool write_pending_; @@ -1127,7 +1127,7 @@ private: Validators::ReceivedValidation rv; rv.ledgerHash = sv.getLedgerHash (); rv.publicKey = sv.getSignerPublic(); - getApp ().getValidators ().receiveValidation (rv); + getApp ().getValidators ().on_receive_validation (rv); } // //---------------------------------------------------------------------- diff --git a/src/ripple/unity/peerfinder.cpp b/src/ripple/unity/peerfinder.cpp index 41efb758f..b12b20470 100644 --- a/src/ripple/unity/peerfinder.cpp +++ b/src/ripple/unity/peerfinder.cpp @@ -21,7 +21,6 @@ #include -#include #include #include diff --git a/src/ripple/unity/validators.cpp b/src/ripple/unity/validators.cpp index 2884a4eea..5b687f993 100644 --- a/src/ripple/unity/validators.cpp +++ b/src/ripple/unity/validators.cpp @@ -32,7 +32,6 @@ #include #include -#include #include // for unit test #include diff --git a/src/ripple/validators/api/Manager.h b/src/ripple/validators/api/Manager.h index 82242579f..d4c1b4120 100644 --- a/src/ripple/validators/api/Manager.h +++ b/src/ripple/validators/api/Manager.h @@ -43,7 +43,7 @@ public: @param journal Where to send log output. */ static Manager* New ( - beast::Stoppable& stoppableParent, + beast::Stoppable& stoppableParent, beast::File const& pathToDbFileOrDirectory, beast::Journal journal); @@ -92,11 +92,11 @@ public: //virtual bool isPublicKeyTrusted (RipplePublicKey const& publicKey) = 0; - /** Called when a validation with a proper signature is received. */ - virtual void receiveValidation (ReceivedValidation const& rv) = 0; - - /** Called when a ledger is closed. */ - virtual void ledgerClosed (RippleLedgerHash const& ledgerHash) = 0; + /** Callback to call when a properly signed validation is received. */ + virtual void on_receive_validation (ReceivedValidation const& rv) = 0; + + /** Callback to call when a ledger is closed. */ + virtual void on_ledger_closed (RippleLedgerHash const& ledgerHash) = 0; }; } diff --git a/src/ripple/validators/impl/Count.h b/src/ripple/validators/impl/Count.h index 7ef025ba9..fa33a6eb2 100644 --- a/src/ripple/validators/impl/Count.h +++ b/src/ripple/validators/impl/Count.h @@ -67,7 +67,7 @@ struct Count } /** Output to PropertyStream. */ - void onWrite (beast::PropertyStream::Map& map) + void onWrite (beast::PropertyStream::Map& map) const { map["received"] = received; map["expected"] = expected; @@ -81,14 +81,6 @@ struct Count std::size_t closed; // Number of validations with closed ledgers }; -inline Count operator+ (Count const& lhs, Count const& rhs) -{ - return Count ( - lhs.received + rhs.received, - lhs.expected + rhs.expected, - lhs.closed + rhs.closed); -} - } } diff --git a/src/ripple/validators/impl/Logic.h b/src/ripple/validators/impl/Logic.h index 30d3cea15..4bbe938fc 100644 --- a/src/ripple/validators/impl/Logic.h +++ b/src/ripple/validators/impl/Logic.h @@ -20,11 +20,99 @@ #ifndef RIPPLE_VALIDATORS_LOGIC_H_INCLUDED #define RIPPLE_VALIDATORS_LOGIC_H_INCLUDED +#include +#include #include namespace ripple { namespace Validators { +// Forward declare unit test so it can be a friend to LRUCache. +class Logic_test; + +namespace detail +{ +// The LRUCache class (ab)uses an aged_unordered_set so it can hold on +// to a limited number of values. When the container gets too full the +// LRUCache expires the oldest values. +// +// An aged_unordered_set gives us the functionality we want by keeping the +// chronological list. We don't care about the actual time of entry, only +// the time ordering. So we hook the aged_unordered_set up to a maunual_clock +// (which we never bother to increment). +// +// The implementation could potentially be changed to be time-based, rather +// than count-based, by hooking up a beast::basic_second_clock in place of the +// manual_clock and deleting a range of expired entries on insert. +// +template , + class KeyEqual = std::equal_to , + class Allocator = std::allocator > +class LRUCache +{ +private: + typedef std::chrono::seconds Duration; + typedef beast::manual_clock Clock; + typedef beast::aged_unordered_set < + Key, Duration, Hash, KeyEqual, Allocator> ContainerType; + +public: + LRUCache () = delete; + + LRUCache (LRUCache const& lhs) = delete; + + explicit LRUCache ( + size_t item_max, + Hash hash = Hash(), + KeyEqual equal = KeyEqual(), + Allocator alloc = Allocator()) + : m_clock () + , m_cache (m_clock, hash, equal, alloc) + , m_item_max (item_max) + { + m_cache.reserve (m_item_max + 1); + } + + LRUCache& operator= (LRUCache const& lhs) = delete; + + // Add the entry. Remove the oldest entry if we went over our limit. + // Returns true on insertion (the entry was not already in the cache). + bool insert (Key const& key) + { + auto const insertRet (m_cache.insert (key)); + if (insertRet.second == false) + { + // key is re-referenced. Mark it as MRU. + m_cache.touch (insertRet.first); + } + else if (m_cache.size () > m_item_max) + { + // Added key and cache is too big. Erase oldest element. + m_cache.erase (m_cache.chronological.begin ()); + } + return insertRet.second; + } + + size_t size () + { + return m_cache.size(); + } + + Key const* oldest () + { + return m_cache.empty() ? nullptr : &(*m_cache.chronological.begin()); + } + +private: + Clock m_clock; + ContainerType m_cache; + const size_t m_item_max; +}; +} // namespace detail + +//------------------------------------------------------------------------------ + // Encapsulates the logic for creating the chosen validators. // This is a separate class to facilitate the unit tests. // @@ -44,6 +132,7 @@ public: beast::SharedPtr fetchSource; }; +private: typedef beast::SharedData SharedState; SharedState m_state; @@ -72,27 +161,28 @@ public: // Filters duplicate validations // - typedef CycledSet SeenValidations; - SeenValidations m_seenValidations; + typedef detail::LRUCache RecentValidations; + RecentValidations m_recentValidations; // Filters duplicate ledger hashes // - typedef CycledSet SeenLedgerHashes; - SeenLedgerHashes m_seenLedgerHashes; + typedef detail::LRUCache RecentLedgerHashes; + RecentLedgerHashes m_recentLedgerHashes; //-------------------------------------------------------------------------- +public: explicit Logic (Store& store, beast::Journal journal = beast::Journal ()) : m_store (store) , m_journal (journal) , m_ledgerID (0) , m_rebuildChosenList (false) - , m_seenValidations (seenValidationsCacheSize) - , m_seenLedgerHashes (seenLedgersCacheSize) + , m_recentValidations (recentValidationsCacheSize) + , m_recentLedgerHashes (recentLedgersCacheSize) { m_sources.reserve (16); } @@ -233,6 +323,18 @@ public: return numRemoved; } + /** Return reference to m_sources for Mangager::PropertyStream. */ + SourceTable const& getSources () + { + return m_sources; + } + + /** Return reference to m_validators for Manager::PropertyStream. */ + ValidatorTable const& getValidators () + { + return m_validators; + } + //-------------------------------------------------------------------------- // // Chosen @@ -275,12 +377,10 @@ public: } } - /** Returns the current Chosen list. - This can be called from any thread at any time. - */ - ChosenList::Ptr getChosen () + /** Returns number of elements in the current Chosen list. */ + std::uint32_t getChosenSize() { - return m_chosenList; + return m_chosenList ? m_chosenList->size() : 0; } //-------------------------------------------------------------------------- @@ -393,7 +493,7 @@ public: { std::size_t n (0); beast::Time const currentTime (beast::Time::getCurrentTime ()); - + for (SourceTable::iterator iter = m_sources.begin (); (n == 0) && iter != m_sources.end (); ++iter) { @@ -420,7 +520,7 @@ public: } //-------------------------------------------------------------------------- - // + // // Ripple interface // //-------------------------------------------------------------------------- @@ -434,10 +534,10 @@ public: if (iter != m_validators.end ()) { // Filter duplicates (defensive programming) - if (! m_seenValidations.insert (rv)) + if (! m_recentValidations.insert (rv)) return; - iter->second.receiveValidation (rv.ledgerHash); + iter->second.on_validation (rv.ledgerHash); m_journal.trace << "New trusted validation for " << rv.ledgerHash << @@ -456,7 +556,7 @@ public: void ledgerClosed (RippleLedgerHash const& ledgerHash) { // Filter duplicates (defensive programming) - if (! m_seenLedgerHashes.insert (ledgerHash)) + if (! m_recentLedgerHashes.insert (ledgerHash)) return; ++m_ledgerID; @@ -466,10 +566,7 @@ public: for (ValidatorTable::iterator iter (m_validators.begin()); iter != m_validators.end(); ++iter) - { - Validator& v (iter->second); - v.ledgerClosed (ledgerHash); - } + iter->second.on_ledger (ledgerHash); } // Returns `true` if the public key hash is contained in the Chosen List. diff --git a/src/ripple/validators/impl/Manager.cpp b/src/ripple/validators/impl/Manager.cpp index 2d46abd66..666e9e715 100644 --- a/src/ripple/validators/impl/Manager.cpp +++ b/src/ripple/validators/impl/Manager.cpp @@ -93,7 +93,7 @@ * Measurements of constructive/destructive behavior is calculated in units of percentage of ledgers for which the behavior is measured. - + What we want from the unique node list: - Some number of trusted roots (known by domain) probably organizations whose job is to provide a list of validators @@ -151,8 +151,8 @@ public: bool m_checkSources; ManagerImp ( - Stoppable& parent, - beast::File const& pathToDbFileOrDirectory, + Stoppable& parent, + beast::File const& pathToDbFileOrDirectory, beast::Journal journal) : Stoppable ("Validators::Manager", parent) , Thread ("Validators") @@ -232,14 +232,14 @@ public: //-------------------------------------------------------------------------- - void receiveValidation (ReceivedValidation const& rv) + void on_receive_validation (ReceivedValidation const& rv) { if (! isStopping()) m_queue.dispatch (m_context.wrap (std::bind ( &Logic::receiveValidation, &m_logic, rv))); } - void ledgerClosed (RippleLedgerHash const& ledgerHash) + void on_ledger_closed (RippleLedgerHash const& ledgerHash) { if (! isStopping()) m_queue.dispatch (m_context.wrap (std::bind ( @@ -283,24 +283,21 @@ public: { Context::Scope scope (m_context); - map ["trusted"] = std::uint32_t ( - m_logic.m_chosenList ? - m_logic.m_chosenList->size() : 0); - + map ["trusted"] = m_logic.getChosenSize(); { beast::PropertyStream::Set items ("sources", map); - for (Logic::SourceTable::const_iterator iter (m_logic.m_sources.begin()); - iter != m_logic.m_sources.end(); ++iter) - items.add (iter->source->to_string()); + for (auto const& entry : m_logic.getSources()) + { + items.add (entry.source->to_string()); + } } { beast::PropertyStream::Set items ("validators", map); - for (Logic::ValidatorTable::iterator iter (m_logic.m_validators.begin()); - iter != m_logic.m_validators.end(); ++iter) + for (auto const& entry : m_logic.getValidators()) { - RipplePublicKey const& publicKey (iter->first); - Validator const& validator (iter->second); + RipplePublicKey const& publicKey (entry.first); + Validator const& validator (entry.second); beast::PropertyStream::Map item (items); item["public_key"] = publicKey.to_string(); validator.count().onWrite (item); @@ -317,7 +314,7 @@ public: void init () { beast::Error error (m_store.open (m_databaseFile)); - + if (! error) { m_logic.load (); @@ -383,7 +380,7 @@ Manager::Manager () } Validators::Manager* Validators::Manager::New ( - beast::Stoppable& parent, + beast::Stoppable& parent, beast::File const& pathToDbFileOrDirectory, beast::Journal journal) { diff --git a/src/ripple/validators/impl/Tests.cpp b/src/ripple/validators/impl/Tests.cpp index c34f12694..a5ab9b5be 100644 --- a/src/ripple/validators/impl/Tests.cpp +++ b/src/ripple/validators/impl/Tests.cpp @@ -184,6 +184,127 @@ public: } } + void testLRUCache () + { + detail::LRUCache testCache {3}; + expect (testCache.size () == 0, "Wrong initial size"); + + struct TestValues + { + char const* const value; + bool const insertResult; + }; + { + std::array const v1 { + {{"A", true}, {"B", true}, {"C", true}}}; + for (auto const& v : v1) + { + expect (testCache.insert (v.value) == v.insertResult, + "Failed first insert tests"); + } + expect (testCache.size() == 3, "Unexpected intermediate size"); + expect (*testCache.oldest() == "A", "Unexpected oldest member"); + } + { + std::array const v2 { + {{"A", false}, {"D", true}, {"C", false}}}; + for (auto const& v : v2) + { + expect (testCache.insert (v.value) == v.insertResult, + "Failed second insert tests"); + } + expect (testCache.size() == 3, "Unexpected final size"); + expect (*testCache.oldest() == "A", + "Unexpected oldest member"); + } + } + + void testValidator () + { + int receivedCount = 0; + int expectedCount = 0; + int closedCount = 0; + + // Lambda as local function + auto updateCounts = [&](bool received, bool validated) + { + bool const sent = received || validated; + + receivedCount += sent && !validated ? 1 : 0; + expectedCount += sent && !received ? 1 : 0; + closedCount += validated && received ? 1 : 0; + }; + + auto checkCounts = [&] (Count const& count) + { +// std::cout << "Received actual: " << count.received << " expected: " << receivedCount << std::endl; +// std::cout << "Expected actual: " << count.expected << " expected: " << expectedCount << std::endl; +// std::cout << "Closed actual: " << count.closed << " expected: " << closedCount << std::endl; + expect (count.received == receivedCount, "Bad received count"); + expect (count.expected == expectedCount, "Bad expected count"); + expect (count.closed == closedCount, "Bad closed count"); + }; + + Validator validator; + std::uint64_t i = 1; + + // Received before closed + for (; i <= ledgersPerValidator; ++i) + { + RippleLedgerHash const hash {i}; + + bool const received = (i % 13 != 0); + bool const validated = (i % 7 != 0); + updateCounts (received, validated); + + if (received) + validator.on_validation (hash); + + if (validated) + validator.on_ledger (hash); + } + checkCounts (validator.count ()); + + // Closed before received + for (; i <= ledgersPerValidator * 2; ++i) + { + RippleLedgerHash const hash {i}; + + bool const received = (i % 11 != 0); + bool const validated = (i % 17 != 0); + updateCounts (received, validated); + + if (validated) + validator.on_ledger (hash); + + if (received) + validator.on_validation (hash); + } + checkCounts (validator.count ()); + + { + // Repeated receives + RippleLedgerHash const hash {++i}; + receivedCount += 1; + for (auto j = 0; j < 100; ++j) + { + validator.on_validation (hash); + } + } + checkCounts (validator.count ()); + + { + // Repeated closes + RippleLedgerHash const hash {++i}; + expectedCount += 1; + for (auto j = 0; j < 100; ++j) + { + validator.on_ledger (hash); + } + } + checkCounts (validator.count ()); + } + void testLogic () { //TestStore store; @@ -206,7 +327,7 @@ public: logic.fetch_one (); - ChosenList::Ptr list (logic.getChosen ()); +// auto chosenSize (logic.getChosenSize ()); pass (); } @@ -214,6 +335,8 @@ public: void run () { + testLRUCache (); + testValidator (); testLogic (); } }; diff --git a/src/ripple/validators/impl/Tuning.h b/src/ripple/validators/impl/Tuning.h index 3e8982eb9..4343b5e48 100644 --- a/src/ripple/validators/impl/Tuning.h +++ b/src/ripple/validators/impl/Tuning.h @@ -20,10 +20,6 @@ #ifndef RIPPLE_VALIDATORS_TUNING_H_INCLUDED #define RIPPLE_VALIDATORS_TUNING_H_INCLUDED -#include - -#include - namespace ripple { namespace Validators { @@ -43,115 +39,16 @@ enum #endif // This tunes the preallocated arrays - ,expectedNumberOfResults = 1000 + ,expectedNumberOfResults = 1000 - // NUmber of entries in the seen validations cache - ,seenValidationsCacheSize = 1000 + // Number of entries in the recent validations cache + ,recentValidationsCacheSize = 1000 - // Number of entries in the seen ledgers cache - ,seenLedgersCacheSize = 1000 // about half an hour at 2/sec + // Number of entries in the recent ledgers cache + ,recentLedgersCacheSize = 1000 // about half an hour at 2/sec // Number of closed Ledger entries per Validator - ,ledgersPerValidator = 100 // this shouldn't be too large -}; - -//------------------------------------------------------------------------------ - -/** Cycled associative map of unique keys. */ -template , - class Allocator = std::allocator > > -class CycledMap -{ -private: - typedef hash_map ContainerType; - typedef typename ContainerType::iterator iterator; - -public: - typedef typename ContainerType::key_type key_type; - typedef typename ContainerType::value_type value_type; - typedef typename ContainerType::size_type size_type; - typedef typename ContainerType::difference_type difference_type; - typedef typename ContainerType::hasher hasher; - typedef typename ContainerType::key_equal key_equal; - typedef typename ContainerType::allocator_type allocator_type; - typedef typename ContainerType::reference reference; - typedef typename ContainerType::const_reference const_reference; - typedef typename ContainerType::pointer pointer; - typedef typename ContainerType::const_pointer const_pointer; - - explicit CycledMap ( - size_type item_max, - Hash hash = Hash(), - KeyEqual equal = KeyEqual(), - Allocator alloc = Allocator()) - : m_max (item_max) - , m_hash (hash) - , m_equal (equal) - , m_alloc (alloc) - , m_front (m_max, hash, equal, alloc) - , m_back (m_max, hash, equal, alloc) - { - } - - Info& front() - { return m_front_info; } - - Info const & front() const - { return m_front_info; } - - Info& back () - { return m_back_info; } - - Info const& back () const - { return m_back_info; } - - /** Returns `true` if the next real insert would swap. */ - bool full() const - { - return m_front.size() >= m_max; - } - - /** Insert the value if it doesn't already exist. */ - std::pair insert (value_type const& value) - { - if (full()) - cycle (); - iterator iter (m_back.find (value.first)); - if (iter != m_back.end()) - return std::make_pair ( - std::ref (iter->second), - std::ref (m_back_info)); - std::pair result ( - m_front.insert (value)); - return std::make_pair ( - std::ref (result.first->second), - std::ref (m_front_info)); - } - - void cycle () - { - std::swap (m_front, m_back); - m_front.clear (); -#if BOOST_VERSION > 105400 - m_front.reserve (m_max); -#endif - std::swap (m_front_info, m_back_info); - m_front_info.clear(); - } - -private: - size_type m_max; - hasher m_hash; - key_equal m_equal; - allocator_type m_alloc; - ContainerType m_front; - ContainerType m_back; - Info m_front_info; - Info m_back_info; + ,ledgersPerValidator = 100 // this shouldn't be too large }; } diff --git a/src/ripple/validators/impl/Validator.h b/src/ripple/validators/impl/Validator.h index 2f416d716..8bb0399f6 100644 --- a/src/ripple/validators/impl/Validator.h +++ b/src/ripple/validators/impl/Validator.h @@ -20,6 +20,11 @@ #ifndef RIPPLE_VALIDATORS_VALIDATOR_H_INCLUDED #define RIPPLE_VALIDATORS_VALIDATOR_H_INCLUDED +#include +#include +#include +#include + namespace ripple { namespace Validators { @@ -27,83 +32,132 @@ namespace Validators { class Validator { private: - /** State of a ledger. */ - struct Ledger + // State of a ledger. + struct Entry { - Ledger() : closed (false), received (false) - { } - - bool closed; // `true` if the ledger was closed - bool received; // `true` if we got a validation + bool closed = false; // `true` if the ledger was closed + bool received = false; // `true` if we got a validation }; - /** Number of sources that reference this validator. */ - int m_refCount; + // Holds the Entry of all recent ledgers for this validator. +#if 1 + typedef beast::aged_unordered_map , + std::hash, + RippleLedgerHash::key_equal> Table; +#else + typedef beast::aged_map > Table; +#endif - /** Holds the state of all recent ledgers for this validator. */ - /** @{ */ - typedef CycledMap , - RippleLedgerHash::key_equal> LedgerMap; - LedgerMap m_ledgers; - /** @} */ + int refs_; // Number of sources that reference this validator. + Table table_; + Count count_; public: - Validator () - : m_refCount (0) - , m_ledgers (ledgersPerValidator) + Validator() + : refs_ (0) + , table_ (get_seconds_clock ()) { } /** Increment the number of references to this validator. */ - void addRef () - { ++m_refCount; } + void + addRef() + { + ++refs_; + } /** Decrement the number of references to this validator. When the reference count reaches zero, the validator will be removed and no longer tracked. */ - bool release () - { return (--m_refCount) == 0; } + bool + release() + { + return (--refs_) == 0; + } + + size_t + size () const + { + return table_.size (); + } /** Returns the composite performance statistics. */ - Count count () const - { return m_ledgers.front() + m_ledgers.back(); } + Count const& + count () const + { + return count_; + } /** Called upon receipt of a validation. */ - void receiveValidation (RippleLedgerHash const& ledgerHash) + void + on_validation (RippleLedgerHash const& ledgerHash) { - std::pair result (m_ledgers.insert ( - std::make_pair (ledgerHash, Ledger()))); - Ledger& ledger (result.first); - Count& count (result.second); - ledger.received = true; - if (ledger.closed) + auto const result (table_.insert ( + std::make_pair (ledgerHash, Entry()))); + auto& entry (result.first->second); + if (entry.received) + return; + entry.received = true; + if (entry.closed) { - --count.expected; - ++count.closed; + --count_.expected; + ++count_.closed; + //table_.erase (result.first); } else { - ++count.received; + ++count_.received; } + //expire(); } /** Called when a ledger is closed. */ - void ledgerClosed (RippleLedgerHash const& ledgerHash) + void + on_ledger (RippleLedgerHash const& ledgerHash) { - std::pair result (m_ledgers.insert ( - std::make_pair (ledgerHash, Ledger()))); - Ledger& ledger (result.first); - Count& count (result.second); - ledger.closed = true; - if (ledger.received) + auto const result (table_.insert ( + std::make_pair (ledgerHash, Entry()))); + auto& entry (result.first->second); + if (entry.closed) + return; + entry.closed = true; + if (entry.received) { - --count.received; - ++count.closed; + --count_.received; + ++count_.closed; + //table_.erase (result.first); } else { - ++count.expected; + ++count_.expected; + } + //expire(); + } + + /** Prunes old entries. */ + void + expire() + { + beast::expire (table_, std::chrono::minutes(5)); + } + +private: + void dump () + { + std::cout << "Validator: " << this << std::endl; + std::cout << "Size: " << table_.size() << std::endl; + std::cout << "end at: " << &(*table_.end()) << std::endl; + for (auto const& ledgerKeyAndState : table_) + { + std::cout << "keyAndState at: " << &ledgerKeyAndState.first << std::endl; + std::cout << " Hash: " << ledgerKeyAndState.first << std::endl; + std::cout << " closed: " << ledgerKeyAndState.second.closed << + " received: " << ledgerKeyAndState.second.received << + std::endl; } } };