From 75f3c52d5305b3c8af3c0b526da4ffcd542115dd Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Fri, 4 Oct 2013 03:09:23 -0700 Subject: [PATCH] Validators work --- Builds/VisualStudio2012/RippleD.vcxproj | 3 +- .../VisualStudio2012/RippleD.vcxproj.filters | 9 +- src/BeastConfig.h | 4 +- src/ripple/validators/api/Source.h | 33 +- src/ripple/validators/api/Types.h | 17 +- src/ripple/validators/impl/Count.h | 85 +++++ src/ripple/validators/impl/Logic.h | 341 ++++++++---------- src/ripple/validators/impl/Manager.cpp | 56 +-- src/ripple/validators/impl/Source.cpp | 9 +- src/ripple/validators/impl/SourceDesc.h | 4 +- src/ripple/validators/impl/SourceFile.cpp | 4 +- src/ripple/validators/impl/SourceStrings.cpp | 10 +- src/ripple/validators/impl/SourceURL.cpp | 13 +- src/ripple/validators/impl/StoreSqdb.cpp | 146 +++++--- src/ripple/validators/impl/StoreSqdb.h | 4 +- src/ripple/validators/impl/Tests.cpp | 16 +- src/ripple/validators/impl/Tuning.h | 140 +++++-- src/ripple/validators/impl/Utilities.cpp | 20 +- src/ripple/validators/impl/Utilities.h | 20 +- .../impl/{Ledger.h => Validation.h} | 33 +- src/ripple/validators/impl/Validator.h | 216 ++++------- src/ripple/validators/ripple_validators.cpp | 5 +- src/ripple_app/peers/Peer.cpp | 1 - 23 files changed, 665 insertions(+), 524 deletions(-) create mode 100644 src/ripple/validators/impl/Count.h rename src/ripple/validators/impl/{Ledger.h => Validation.h} (57%) diff --git a/Builds/VisualStudio2012/RippleD.vcxproj b/Builds/VisualStudio2012/RippleD.vcxproj index 07bb7b27c4..23888ca8b3 100644 --- a/Builds/VisualStudio2012/RippleD.vcxproj +++ b/Builds/VisualStudio2012/RippleD.vcxproj @@ -1694,7 +1694,7 @@ - + @@ -1704,6 +1704,7 @@ + diff --git a/Builds/VisualStudio2012/RippleD.vcxproj.filters b/Builds/VisualStudio2012/RippleD.vcxproj.filters index 28219bdabf..2524b6d48e 100644 --- a/Builds/VisualStudio2012/RippleD.vcxproj.filters +++ b/Builds/VisualStudio2012/RippleD.vcxproj.filters @@ -2184,15 +2184,18 @@ [1] Ripple\peerfinder\api - - [1] Ripple\validators\impl - [1] Ripple\validators\impl [1] Ripple\validators\impl + + [1] Ripple\validators\impl + + + [1] Ripple\validators\impl + diff --git a/src/BeastConfig.h b/src/BeastConfig.h index e5e47c58bc..7c6fd19203 100644 --- a/src/BeastConfig.h +++ b/src/BeastConfig.h @@ -159,8 +159,8 @@ // Here temporarily to turn off new Validations code while it // is being written. // -#ifndef RIPPLE_USE_NEW_VALIDATORS -#define RIPPLE_USE_NEW_VALIDATORS 0 +#ifndef RIPPLE_USE_VALIDATORS +#define RIPPLE_USE_VALIDATORS 0 #endif // Turning this on will use the new PeerFinder logic to establish connections diff --git a/src/ripple/validators/api/Source.h b/src/ripple/validators/api/Source.h index 1c1e7bb444..bead5a8001 100644 --- a/src/ripple/validators/api/Source.h +++ b/src/ripple/validators/api/Source.h @@ -28,7 +28,7 @@ class Source : public SharedObject { public: /** A Source's descriptor for a Validator. */ - struct Info + struct Item { /** The unique key for this validator. */ RipplePublicKey publicKey; @@ -55,25 +55,26 @@ public: /** A string that is used to recreate the source from the database entry. */ virtual String createParam () = 0; - /** Fetch the most recent list from the Source. - This call will block. - */ - struct Result - { - Result (); - void swapWith (Result& other); - - bool success; - String message; - Time expirationTime; - std::vector list; - }; - /** Cancel any pending fetch. The default implementation does nothing. */ virtual void cancel () { } - virtual void fetch (Result& result, Journal journal) = 0; + + /** Fetch results. + This call will block + */ + /** @{ */ + struct Results + { + Results (); + + bool success; + String message; + Time expirationTime; + std::vector list; + }; + virtual void fetch (Results& results, Journal journal) = 0; + /** @} */ }; } diff --git a/src/ripple/validators/api/Types.h b/src/ripple/validators/api/Types.h index 835f9c6359..99e03cf43b 100644 --- a/src/ripple/validators/api/Types.h +++ b/src/ripple/validators/api/Types.h @@ -25,9 +25,20 @@ namespace Validators { struct ReceivedValidation { - RippleLedgerHash ledgerHash; - RipplePublicKey publicKey; - RipplePublicKeyHash publicKeyHash; + ReceivedValidation () + { + } + + ReceivedValidation ( + LedgerHash const& ledgerHash_, + RipplePublicKey const& publicKey_) + : ledgerHash (ledgerHash_) + , publicKey (publicKey_) + { + } + + RippleLedgerHash ledgerHash; + RipplePublicKey publicKey; }; } diff --git a/src/ripple/validators/impl/Count.h b/src/ripple/validators/impl/Count.h new file mode 100644 index 0000000000..0a9c9f6028 --- /dev/null +++ b/src/ripple/validators/impl/Count.h @@ -0,0 +1,85 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_VALIDATORS_COUNT_H_INCLUDED +#define RIPPLE_VALIDATORS_COUNT_H_INCLUDED + +namespace ripple { +namespace Validators { + +/** Measures Validator performance statistics. */ +struct Count +{ + Count() + : received (0) + , expected (0) + , closed (0) + { + } + + Count (std::size_t received_, + std::size_t expected_, + std::size_t closed_) + : received (received_) + , expected (expected_) + , closed (closed_) + { + } + + /** Reset the statistics. */ + void clear () + { + *this = Count(); + } + + /** Returns the percentage of ledger participation. */ + int percent () const + { + int const total (closed + expected); + if (total > 0) + return (closed * 100) / total; + return 0; + } + + /** Returns the percentage of orphaned validations. */ + int percent_orphaned () const + { + int const total (received + closed); + if (total > 0) + return (received * 100) / total; + return 0; + } + + std::size_t received; // Count of validations without a closed ledger + std::size_t expected; // Count of closed ledgers without a validation + std::size_t closed; // Number of validations with closed ledgers +}; + +inline Count operator+ (Count const& lhs, Count const& rhs) +{ + return Count ( + lhs.received + rhs.received, + lhs.expected + rhs.expected, + lhs.closed + rhs.closed); +} + +} +} + +#endif diff --git a/src/ripple/validators/impl/Logic.h b/src/ripple/validators/impl/Logic.h index 3693f4c177..5e54411f50 100644 --- a/src/ripple/validators/impl/Logic.h +++ b/src/ripple/validators/impl/Logic.h @@ -32,26 +32,52 @@ public: struct State { State () - { - //sources.reserve (64); - } + : stopping (false) + { } - ValidatorMap validators; - SourcesType sources; + /** True if we are stopping. */ + bool stopping; + + /** The source we are currently fetching. */ + SharedPtr fetchSource; }; typedef SharedData SharedState; - Store& m_store; - Journal m_journal; - bool m_rebuildChosenList; - ChosenList::Ptr m_chosenList; SharedState m_state; - // Used to filter duplicate ledger hashes + Store& m_store; + Journal m_journal; + + // The chosen set of trusted validators (formerly the "UNL") // - typedef AgedHistory > SeenLedgerHashes; + bool m_rebuildChosenList; + ChosenList::Ptr m_chosenList; + + // Holds the list of sources + // + typedef std::vector SourceTable; + SourceTable m_sources; + + // Holds the internal list of trusted validators + // + typedef boost::unordered_map < + RipplePublicKey, Validator, + RipplePublicKey::hasher> ValidatorTable; + ValidatorTable m_validators; + + // Filters duplicate validations + // + typedef CycledSet SeenValidations; + SeenValidations m_seenValidations; + + // Filters duplicate ledger hashes + // + typedef CycledSet SeenLedgerHashes; SeenLedgerHashes m_seenLedgerHashes; //---------------------------------------------------------------------- @@ -60,21 +86,39 @@ public: : m_store (store) , m_journal (journal) , m_rebuildChosenList (false) + , m_seenValidations (seenValidationsCacheSize) + , m_seenLedgerHashes (seenLedgersCacheSize) { + m_sources.reserve (16); } + /** Stop the logic. + This will cancel the current fetch and set the stopping flag + to `true` to prevent further fetches. + Thread safety: + Safe to call from any thread. + */ + void stop () + { + SharedState::Access state (m_state); + state->stopping = true; + if (state->fetchSource != nullptr) + state->fetchSource->cancel (); + } + + //---------------------------------------------------------------------- + void load () { - // load data from m_store + // load data from the database } // Returns `true` if a Source with the same unique ID already exists // bool findSourceByID (String id) { - SharedState::Access state (m_state); - for (SourcesType::const_iterator iter (state->sources.begin()); - iter != state->sources.end(); ++iter) + for (SourceTable::const_iterator iter (m_sources.begin()); + iter != m_sources.end(); ++iter) if (iter->source->uniqueID() == id) return true; return false; @@ -93,14 +137,12 @@ public: m_journal.info << "Addding static " << source->name(); - Source::Result result; - source->fetch (result, m_journal); + Source::Results results; + source->fetch (results, m_journal); - if (result.success) + if (results.success) { - SharedState::Access state (m_state); - std::size_t const numAdded ( - merge (result.list, source, state)); + std::size_t const numAdded (merge (results.list, source)); m_journal.info << "Added " << numAdded << " trusted validators from " << source->name(); } @@ -124,30 +166,28 @@ public: m_journal.info << "Adding " << source->name(); { - SharedState::Access state (m_state); - state->sources.resize (state->sources.size() + 1); - SourceDesc& desc (state->sources.back()); + m_sources.resize (m_sources.size() + 1); + SourceDesc& desc (m_sources.back()); desc.source = source; m_store.insert (desc); - merge (desc.result.list, desc.source, state); + merge (desc.results.list, desc.source); } } // Add each entry in the list to the map, incrementing the // reference count if it already exists, and updating fields. // - std::size_t merge (std::vector const& list, - Source* source, SharedState::Access& state) + std::size_t merge (std::vector const& list, Source* source) { std::size_t numAdded (0); for (std::size_t i = 0; i < list.size (); ++i) { - Source::Info const& info (list [i]); - std::pair result ( - state->validators.emplace (info.publicKey, Validator ())); - Validator& validatorInfo (result.first->second); - ++validatorInfo.refCount; - if (result.second) + Source::Item const& item (list [i]); + std::pair results ( + m_validators.emplace (item.publicKey, Validator ())); + Validator& validatorInfo (results.first->second); + validatorInfo.addRef(); + if (results.second) { // This is a new one ++numAdded; @@ -161,21 +201,21 @@ public: // Decrement the reference count of each item in the list // in the map. // - std::size_t remove (std::vector const& list, - Source* source, SharedState::Access& state) + std::size_t remove (std::vector const& list, + Source* source) { std::size_t numRemoved (0); for (std::size_t i = 0; i < list.size (); ++i) { - Source::Info const& info (list [i]); - ValidatorMap::iterator iter (state->validators.find (info.publicKey)); - bassert (iter != state->validators.end ()); + Source::Item const& item (list [i]); + ValidatorTable::iterator iter (m_validators.find (item.publicKey)); + bassert (iter != m_validators.end ()); Validator& validatorInfo (iter->second); - if (--validatorInfo.refCount == 0) + if (validatorInfo.release()) { // Last reference removed ++numRemoved; - state->validators.erase (iter); + m_validators.erase (iter); dirtyChosen (); } } @@ -191,14 +231,13 @@ public: /** Rebuild the Chosen List. */ void buildChosen () { - SharedState::ConstAccess state (m_state); - ChosenList::Ptr list (new ChosenList (state->validators.size ())); + ChosenList::Ptr list (new ChosenList (m_validators.size ())); - for (ValidatorMap::const_iterator iter = state->validators.begin (); - iter != state->validators.end (); ++iter) + for (ValidatorTable::const_iterator iter = m_validators.begin (); + iter != m_validators.end (); ++iter) { - ChosenList::Info info; - list->insert (iter->first, info); + ChosenList::Info item; + list->insert (iter->first, item); } // This is thread safe @@ -241,33 +280,46 @@ public: /** Perform a fetch on the source. */ void fetch (SourceDesc& desc) { - Source* const source (desc.source); + SharedPtr const& source (desc.source); + Source::Results results; - Source::Result result; - source->fetch (result, m_journal); + { + { + SharedState::Access state (m_state); + if (state->stopping) + return; + state->fetchSource = source; + } - // Reset fetch timer for the source. + source->fetch (results, m_journal); + + { + SharedState::Access state (m_state); + if (state->stopping) + return; + state->fetchSource = nullptr; + } + } + + // Reset fetch timer for the source-> desc.whenToFetch = Time::getCurrentTime () + RelativeTime (secondsBetweenFetches); - if (result.success) + if (results.success) { - SharedState::Access state (m_state); - // Count the number fetched std::size_t const numFetched ( - result.list.size()); + results.list.size()); - // Add the new source info to the map + // Add the new source item to the map std::size_t const numAdded ( - merge (result.list, source, state)); + merge (results.list, source)); // Swap lists - desc.result.swapWith (result); + std::swap (desc.results, results); - // Remove the old source info from the map - std::size_t const numRemoved ( - remove (result.list, source, state)); + // Remove the old source item from the map + std::size_t const numRemoved (remove (results.list, source)); // Report if (numAdded > numRemoved) @@ -307,17 +359,16 @@ public: ++desc.numberOfFailures; desc.status = SourceDesc::statusFailed; - // Record the failure in the Store m_store.update (desc); } } /** Expire a source's list of validators. */ - void expire (SourceDesc& desc, SharedState::Access& state) + void expire (SourceDesc& desc) { // Decrement reference count on each validator - remove (desc.result.list, desc.source, state); + remove (desc.results.list, desc.source); m_store.update (desc); } @@ -330,9 +381,8 @@ public: std::size_t n (0); Time const currentTime (Time::getCurrentTime ()); - SharedState::Access state (m_state); - for (SourcesType::iterator iter = state->sources.begin (); - (n == 0) && iter != state->sources.end (); ++iter) + for (SourceTable::iterator iter = m_sources.begin (); + (n == 0) && iter != m_sources.end (); ++iter) { SourceDesc& desc (*iter); @@ -349,7 +399,7 @@ public: if (desc.expirationTime.isNotNull () && desc.expirationTime <= currentTime) { - expire (desc, state); + expire (desc); } } @@ -364,89 +414,40 @@ public: // Return the current ChosenList as JSON Json::Value rpcPrint (Json::Value const& args) { - Json::Value result (Json::objectValue); + Json::Value results (Json::objectValue); -#if 0 - Json::Value entries (Json::arrayValue); - ChosenList::Ptr list (m_chosenList); - if (list != nullptr) - { - for (ChosenList::ValidatorMap::const_iterator iter (list->map().begin()); - iter != list->map().end(); ++iter) - { - Json::Value entry (Json::objectValue); - ChosenList::ValidatorMap::key_type const& key (iter->first); - entry ["key"] = key.to_string(); - //ChosenList::ValidatorMap::mapped_type const& value (iter->second); - //entry ["value"] = value.to_string(); - entries.append (entry); - } - } - result ["chosen_list"] = entries; - - { - SharedState::ConstAccess state (m_state); - std::size_t count (0); - result ["validators"] = state->validators.size(); - for (ValidatorMap::const_iterator iter (state->validators.begin()); - iter != state->validators.end(); ++iter) - count += iter->second.map.size(); - result ["signatures"] = count; - } -#else Json::Value entries (Json::arrayValue); { - SharedState::ConstAccess state (m_state); - result ["count"] = int(state->validators.size()); - for (ValidatorMap::const_iterator iter (state->validators.begin()); - iter != state->validators.end(); ++iter) + results ["count"] = int(m_validators.size()); + for (ValidatorTable::const_iterator iter (m_validators.begin()); + iter != m_validators.end(); ++iter) { Validator const& v (iter->second); Json::Value entry (Json::objectValue); + Count const count (v.count ()); - std::size_t const closed ( - v.count->closed + v.count.back().closed); - - std::size_t const seen ( - v.count->seen + v.count.back().seen); - - std::size_t const missing ( - v.count->missing + v.count.back().missing); - - std::size_t const orphans ( - v.count->orphans + v.count.back().orphans); - - entry ["public"] = iter->first.to_string(); - entry ["closed"] = int(closed); - entry ["seen"] = int(seen); - entry ["missing"] = int(missing); - entry ["orphans"] = int(orphans); - - if (closed > 0) - { - int const percent ( - ((seen - missing) * 100) / closed); - entry ["percent"] = percent; - } + entry ["public"] = iter->first.to_string(); + entry ["received"] = int(count.received); + entry ["expected"] = int(count.expected); + entry ["closed"] = int(count.closed); + entry ["percent"] = count.percent(); entries.append (entry); } } - result ["validators"] = entries; + results ["validators"] = entries; -#endif - return result; + return results; } // Returns the list of sources Json::Value rpcSources (Json::Value const& arg) { - Json::Value result (Json::objectValue); + Json::Value results (Json::objectValue); Json::Value entries (Json::arrayValue); - SharedState::ConstAccess state (m_state); - for (SourcesType::const_iterator iter (state->sources.begin()); - iter != state->sources.end(); ++iter) + for (SourceTable::const_iterator iter (m_sources.begin()); + iter != m_sources.end(); ++iter) { Json::Value entry (Json::objectValue); SourceDesc const& desc (*iter); @@ -454,20 +455,20 @@ public: entry ["param"] = desc.source->createParam(); Json::Value results (Json::arrayValue); - for (int i = 0; i < desc.result.list.size(); ++i) + for (int i = 0; i < desc.results.list.size(); ++i) { Json::Value info (Json::objectValue); info ["key"] = "publicKey"; - info ["label"] = desc.result.list[i].label; + info ["label"] = desc.results.list[i].label; results.append (info); } - entry ["result"] = results; + entry ["results"] = results; entries.append (entry); } - result ["sources"] = entries; + results ["sources"] = entries; - return result; + return results; } //---------------------------------------------------------------------- @@ -475,74 +476,32 @@ public: // Ripple interface // - // VFALCO NOTE We cannot make any assumptions about the quality of the - // information being passed into the logic. Specifically, - // we can expect to see duplicate ledgerClose, and duplicate - // receiveValidation. Therefore, we must program defensively - // to prevent undefined behavior - // Called when we receive a signed validation // - // Used to filter duplicate public keys - // - typedef AgedCache > SeenPublicKeys; - SeenPublicKeys m_seenPublicKeys; - void receiveValidation (ReceivedValidation const& rv) { - // Filter duplicates - { - std::pair result ( - m_seenPublicKeys->emplace (rv.publicKey)); - if (m_seenPublicKeys->size() > maxSizeBeforeSwap) - { - m_seenPublicKeys.swap(); - m_seenPublicKeys->clear(); - } - if (! result.second) - return; - } - - SharedState::Access state (m_state); -#if 1 // Accept validation from the trusted list - ValidatorMap::iterator iter (state->validators.find (rv.publicKey)); - if (iter != state->validators.end ()) + ValidatorTable::iterator iter (m_validators.find (rv.publicKey)); + if (iter != m_validators.end ()) { - Validator& v (iter->second); - v.receiveValidation (rv.ledgerHash); + // Filter duplicates (defensive programming) + if (! m_seenValidations.insert (rv)) + return; + + iter->second.receiveValidation (rv.ledgerHash); } -#else - // Accept any validation (for testing) - std::pair result ( - state->validators.emplace (rv.publicKey, Validator())); - Validator& v (result.first->second); - v.receiveValidation (rv.ledgerHash); -#endif } // Called when a ledger is closed // void ledgerClosed (RippleLedgerHash const& ledgerHash) { - // Filter duplicates - { - std::pair result ( - m_seenLedgerHashes->emplace (ledgerHash)); - if (m_seenLedgerHashes->size() > maxSizeBeforeSwap) - { - m_seenLedgerHashes.swap(); - m_seenLedgerHashes->clear(); - } - if (! result.second) - return; - } + // Filter duplicates (defensive programming) + if (! m_seenLedgerHashes.insert (ledgerHash)) + return; - SharedState::Access state (m_state); - for (ValidatorMap::iterator iter (state->validators.begin()); - iter != state->validators.end(); ++iter) + for (ValidatorTable::iterator iter (m_validators.begin()); + iter != m_validators.end(); ++iter) { Validator& v (iter->second); v.ledgerClosed (ledgerHash); diff --git a/src/ripple/validators/impl/Manager.cpp b/src/ripple/validators/impl/Manager.cpp index b4c3571bb3..c0a1d04b92 100644 --- a/src/ripple/validators/impl/Manager.cpp +++ b/src/ripple/validators/impl/Manager.cpp @@ -176,6 +176,9 @@ public: } //-------------------------------------------------------------------------- + // + // Manager + // void addStrings (String name, std::vector const& strings) { @@ -203,47 +206,43 @@ public: addStaticSource (SourceFile::New (file)); } - void addURL (URL const& url) - { - addSource (SourceURL::New (url)); - } - - //-------------------------------------------------------------------------- - - void addSource (Source* source) - { -#if RIPPLE_USE_NEW_VALIDATORS - m_queue.dispatch (bind (&Logic::add, &m_logic, source)); -#else - delete source; -#endif - } - void addStaticSource (Source* source) { -#if RIPPLE_USE_NEW_VALIDATORS +#if RIPPLE_USE_VALIDATORS m_queue.dispatch (bind (&Logic::addStatic, &m_logic, source)); #else delete source; #endif } - // VFALCO NOTE we should just do this on the callers thread? - // + void addURL (URL const& url) + { + addSource (SourceURL::New (url)); + } + + void addSource (Source* source) + { +#if RIPPLE_USE_VALIDATORS + m_queue.dispatch (bind (&Logic::add, &m_logic, source)); +#else + delete source; +#endif + } + + //-------------------------------------------------------------------------- + void receiveValidation (ReceivedValidation const& rv) { -#if RIPPLE_USE_NEW_VALIDATORS +#if RIPPLE_USE_VALIDATORS if (! isStopping()) m_queue.dispatch (bind ( &Logic::receiveValidation, &m_logic, rv)); #endif } - // VFALCO NOTE we should just do this on the callers thread? - // void ledgerClosed (RippleLedgerHash const& ledgerHash) { -#if RIPPLE_USE_NEW_VALIDATORS +#if RIPPLE_USE_VALIDATORS if (! isStopping()) m_queue.dispatch (bind ( &Logic::ledgerClosed, &m_logic, ledgerHash)); @@ -257,7 +256,7 @@ public: void onPrepare () { -#if RIPPLE_USE_NEW_VALIDATORS +#if RIPPLE_USE_VALIDATORS m_journal.info << "Validators preparing"; addRPCHandlers(); @@ -266,7 +265,7 @@ public: void onStart () { -#if RIPPLE_USE_NEW_VALIDATORS +#if RIPPLE_USE_VALIDATORS m_journal.info << "Validators starting"; // Do this late so the sources have a chance to be added. @@ -278,7 +277,11 @@ public: void onStop () { +#if RIPPLE_USE_VALIDATORS m_journal.info << "Validators stopping"; +#endif + + m_logic.stop (); if (this->Thread::isThreadRunning()) { @@ -292,6 +295,9 @@ public: } //-------------------------------------------------------------------------- + // + // ManagerImp + // void init () { diff --git a/src/ripple/validators/impl/Source.cpp b/src/ripple/validators/impl/Source.cpp index ba808df367..7ef24cb0ba 100644 --- a/src/ripple/validators/impl/Source.cpp +++ b/src/ripple/validators/impl/Source.cpp @@ -20,18 +20,11 @@ namespace ripple { namespace Validators { -Source::Result::Result () +Source::Results::Results () : success (false) , message ("uninitialized") { } -void Source::Result::swapWith (Result& other) -{ - std::swap (success, other.success); - std::swap (message, other.message); - list.swap (other.list); -} - } } diff --git a/src/ripple/validators/impl/SourceDesc.h b/src/ripple/validators/impl/SourceDesc.h index 938d56d705..9ce1fbc779 100644 --- a/src/ripple/validators/impl/SourceDesc.h +++ b/src/ripple/validators/impl/SourceDesc.h @@ -39,7 +39,7 @@ struct SourceDesc int numberOfFailures; // The result of the last fetch - Source::Result result; + Source::Results results; //------------------------------------------------------------------ @@ -63,8 +63,6 @@ struct SourceDesc } }; -typedef std::vector SourcesType; - } } diff --git a/src/ripple/validators/impl/SourceFile.cpp b/src/ripple/validators/impl/SourceFile.cpp index 67465e55ba..a0811536f8 100644 --- a/src/ripple/validators/impl/SourceFile.cpp +++ b/src/ripple/validators/impl/SourceFile.cpp @@ -49,7 +49,7 @@ public: return m_file.getFullPathName (); } - void fetch (Result& result, Journal journal) + void fetch (Results& results, Journal journal) { int64 const fileSize (m_file.getSize ()); @@ -66,7 +66,7 @@ public: if (amountRead == fileSize) { - Utilities::ParseResultLine lineFunction (result, journal); + Utilities::ParseResultLine lineFunction (results, journal); Utilities::processLines (buffer.begin(), buffer.end(), lineFunction); } } diff --git a/src/ripple/validators/impl/SourceStrings.cpp b/src/ripple/validators/impl/SourceStrings.cpp index 9020f12e79..171062c9bb 100644 --- a/src/ripple/validators/impl/SourceStrings.cpp +++ b/src/ripple/validators/impl/SourceStrings.cpp @@ -51,18 +51,18 @@ public: return String::empty; } - void fetch (Result& result, Journal journal) + void fetch (Results& results, Journal journal) { - result.list.reserve (m_strings.size ()); + results.list.reserve (m_strings.size ()); for (int i = 0; i < m_strings.size (); ++i) { std::string const s (m_strings [i].toStdString ()); - Utilities::parseResultLine (result, s); + Utilities::parseResultLine (results, s); } - result.success = result.list.size () > 0; - result.expirationTime = Time::getCurrentTime () + RelativeTime::hours (24); + results.success = results.list.size () > 0; + results.expirationTime = Time::getCurrentTime () + RelativeTime::hours (24); } private: diff --git a/src/ripple/validators/impl/SourceURL.cpp b/src/ripple/validators/impl/SourceURL.cpp index d19c7097ce..e3a190ebec 100644 --- a/src/ripple/validators/impl/SourceURL.cpp +++ b/src/ripple/validators/impl/SourceURL.cpp @@ -27,6 +27,7 @@ class SourceURLImp public: explicit SourceURLImp (URL const& url) : m_url (url) + , m_client (HTTPClientBase::New ()) { } @@ -49,15 +50,18 @@ public: return m_url.full(); } - void fetch (Result& result, Journal journal) + void cancel () { - ScopedPointer client (HTTPClientBase::New ()); + m_client->cancel (); + } - HTTPClientBase::result_type httpResult (client->get (m_url)); + void fetch (Results& results, Journal journal) + { + HTTPClientBase::result_type httpResult (m_client->get (m_url)); if (httpResult.first == 0) { - Utilities::ParseResultLine lineFunction (result, journal); + Utilities::ParseResultLine lineFunction (results, journal); std::string const s (httpResult.second->body().to_string()); Utilities::processLines (s.begin(), s.end(), lineFunction); } @@ -71,6 +75,7 @@ public: private: URL m_url; + ScopedPointer m_client; }; //------------------------------------------------------------------------------ diff --git a/src/ripple/validators/impl/StoreSqdb.cpp b/src/ripple/validators/impl/StoreSqdb.cpp index 07010cef5b..7e55bbaa49 100644 --- a/src/ripple/validators/impl/StoreSqdb.cpp +++ b/src/ripple/validators/impl/StoreSqdb.cpp @@ -64,7 +64,7 @@ void StoreSqdb::insert (SourceDesc& desc) String const expirationTime (Utilities::timeToString (desc.expirationTime)); sqdb::statement st = (m_session.prepare << - "INSERT INTO ValidatorsSource ( " + "INSERT INTO Validators_Source ( " " sourceID, " " createParam, " " lastFetchTime, " @@ -106,7 +106,7 @@ void StoreSqdb::update (SourceDesc& desc, bool updateFetchResults) sqdb::transaction tr (m_session); m_session.once (error) << - "UPDATE ValidatorsSource SET " + "UPDATE Validators_Source SET " " lastFetchTime = ?, " " expirationTime = ? " "WHERE " @@ -120,7 +120,7 @@ void StoreSqdb::update (SourceDesc& desc, bool updateFetchResults) { // Delete the previous data set m_session.once (error) << - "DELETE FROM ValidatorsSourceInfo WHERE " + "DELETE FROM Validators_SourceItem WHERE " " sourceID = ?; " ,sqdb::use (sourceID) ; @@ -132,7 +132,7 @@ void StoreSqdb::update (SourceDesc& desc, bool updateFetchResults) String label; sqdb::statement st = (m_session.prepare << - "INSERT INTO ValidatorsSourceInfo ( " + "INSERT INTO Validators_SourceItem ( " " sourceID, " " publicKey, " " label " @@ -144,11 +144,11 @@ void StoreSqdb::update (SourceDesc& desc, bool updateFetchResults) ,sqdb::use (label) ); - std::vector & list (desc.result.list); + std::vector & list (desc.results.list); for (std::size_t i = 0; ! error && i < list.size(); ++i) { - Source::Info& info (list [i]); - publicKeyString = info.publicKey.to_string (); + Source::Item& item (list [i]); + publicKeyString = item.publicKey.to_string (); label = list[i].label; st.execute_and_fetch (error); } @@ -197,7 +197,7 @@ bool StoreSqdb::select (SourceDesc& desc) "SELECT " " lastFetchTime, " " expirationTime " - "FROM ValidatorsSource WHERE " + "FROM Validators_Source WHERE " " sourceID = ? " ,sqdb::into (lastFetchTime) ,sqdb::into (expirationTime) @@ -245,7 +245,7 @@ void StoreSqdb::selectList (SourceDesc& desc) m_session.once (error) << "SELECT " " COUNT(*) " - "FROM ValidatorsSourceInfo WHERE " + "FROM Validators_SourceItem WHERE " " sourceID = ? " ,sqdb::into (count) ,sqdb::use (sourceID) @@ -259,10 +259,10 @@ void StoreSqdb::selectList (SourceDesc& desc) } // Precondition: the list must be empty. - bassert (desc.result.list.size() == 0); + bassert (desc.results.list.size() == 0); // Pre-allocate some storage - desc.result.list.reserve (count); + desc.results.list.reserve (count); // Prepare the select { @@ -272,7 +272,7 @@ void StoreSqdb::selectList (SourceDesc& desc) "SELECT " " publicKey, " " label " - "FROM ValidatorsSourceInfo WHERE " + "FROM Validators_SourceItem WHERE " " sourceID = ? " ,sqdb::into (publicKeyString) ,sqdb::into (label) @@ -284,7 +284,7 @@ void StoreSqdb::selectList (SourceDesc& desc) { do { - Source::Info info; + Source::Item info; std::pair result ( RipplePublicKey::from_string (publicKeyString)); if (result.second) @@ -292,7 +292,7 @@ void StoreSqdb::selectList (SourceDesc& desc) bassert (result.first.to_string() == publicKeyString); info.publicKey = result.first; info.label = label; - desc.result.list.push_back (info); + desc.results.list.push_back (info); } else { @@ -304,7 +304,7 @@ void StoreSqdb::selectList (SourceDesc& desc) if (! error) { - m_journal.info << "Loaded " << desc.result.list.size() << + m_journal.info << "Loaded " << desc.results.list.size() << " trusted validators for " << desc.source->name (); } } @@ -346,21 +346,25 @@ Error StoreSqdb::update () if (! error && version != currentSchemaVersion) { - m_journal.info << "Updating old database version " << version; + m_journal.info << + "Update database to version " << currentSchemaVersion << + " from version " << version; } // Update database based on version - if (! error && version < 1) + if (! error && version < 2) { - // Delete all the old data since its likely wrong - m_session.once (error) << - "DELETE FROM ValidatorsSource"; + if (! error) + m_session.once (error) << + "DROP TABLE IF EXISTS ValidatorsSource"; if (! error) - { m_session.once (error) << - "DELETE FROM ValidatorsSourceInfo"; - } + "DROP TABLE IF EXISTS ValidatorsSourceInfo"; + + if (! error) + m_session.once (error) << + "DROP INDEX IF EXISTS ValidatorsSourceInfoIndex"; } // Update the version to the current version @@ -393,6 +397,8 @@ Error StoreSqdb::update () return error; } +//-------------------------------------------------------------------------- + Error StoreSqdb::init () { Error error; @@ -405,42 +411,6 @@ Error StoreSqdb::init () "PRAGMA encoding=\"UTF-8\""; } - if (! error) - { - m_session.once (error) << - "CREATE TABLE IF NOT EXISTS ValidatorsSource ( " - " id INTEGER PRIMARY KEY AUTOINCREMENT, " - " sourceID TEXT UNIQUE, " - " createParam TEXT NOT NULL, " - " lastFetchTime TEXT NOT NULL, " - " expirationTime TEXT NOT NULL " - ");" - ; - } - - if (! error) - { - m_session.once (error) << - "CREATE TABLE IF NOT EXISTS ValidatorsSourceInfo ( " - " id INTEGER PRIMARY KEY AUTOINCREMENT, " - " sourceID TEXT NOT NULL, " - " publicKey TEXT NOT NULL, " - " label TEXT NOT NULL " - ");" - ; - } - - if (! error) - { - m_session.once (error) << - "CREATE INDEX IF NOT EXISTS " - " ValidatorsSourceInfoIndex ON ValidatorsSourceInfo " - " ( " - " sourceID " - " ); " - ; - } - if (! error) { // This table maps component names like "Validators" to their @@ -458,6 +428,64 @@ Error StoreSqdb::init () ; } + if (! error) + { + m_session.once (error) << + "CREATE TABLE IF NOT EXISTS Validators_Source ( " + " id INTEGER PRIMARY KEY AUTOINCREMENT, " + " sourceID TEXT UNIQUE, " + " createParam TEXT NOT NULL, " + " lastFetchTime TEXT NOT NULL, " + " expirationTime TEXT NOT NULL " + ");" + ; + } + + if (! error) + { + m_session.once (error) << + "CREATE TABLE IF NOT EXISTS Validators_SourceItem ( " + " id INTEGER PRIMARY KEY AUTOINCREMENT, " + " sourceID TEXT NOT NULL, " + " publicKey TEXT NOT NULL, " + " label TEXT NOT NULL " + ");" + ; + } + + if (! error) + { + m_session.once (error) << + "CREATE INDEX IF NOT EXISTS " + " Validators_SourceItem_Index ON Validators_SourceItem " + " ( " + " sourceID " + " ); " + ; + } + +#if 0 + if (! error) + { + m_session.once (error) << + "CREATE TABLE IF NOT EXISTS ValidatorsValidator ( " + " id INTEGER PRIMARY KEY AUTOINCREMENT, " + " publicKey TEXT UNIQUE " + ");" + ; + } + + if (! error) + { + m_session.once (error) << + "CREATE TABLE IF NOT EXISTS ValidatorsValidatorStats ( " + " id INTEGER PRIMARY KEY AUTOINCREMENT, " + " publicKey TEXT UNIQUE " + ");" + ; + } +#endif + if (! error) { error = tr.commit(); diff --git a/src/ripple/validators/impl/StoreSqdb.h b/src/ripple/validators/impl/StoreSqdb.h index 377dd29acc..ab0671fb50 100644 --- a/src/ripple/validators/impl/StoreSqdb.h +++ b/src/ripple/validators/impl/StoreSqdb.h @@ -32,7 +32,7 @@ public: enum { // This affects the format of the data! - currentSchemaVersion = 1 + currentSchemaVersion = 2 }; explicit StoreSqdb (Journal journal = Journal()); @@ -45,6 +45,8 @@ public: void update (SourceDesc& desc, bool updateFetchResults); + void remove (RipplePublicKey const& publicKey); + private: void report (Error const& error, char const* fileName, int lineNumber); diff --git a/src/ripple/validators/impl/Tests.cpp b/src/ripple/validators/impl/Tests.cpp index e1d0551bfa..7de4a606b4 100644 --- a/src/ripple/validators/impl/Tests.cpp +++ b/src/ripple/validators/impl/Tests.cpp @@ -121,18 +121,18 @@ public: return String::empty; } - void fetch (Result& result, Journal) + void fetch (Results& results, Journal) { - result.success = true; - result.message = String::empty; - result.list.reserve (numberOfTestValidators); + results.success = true; + results.message = String::empty; + results.list.reserve (numberOfTestValidators); for (uint32 i = m_start ; i < m_end; ++i) { - Info info; - info.publicKey = RipplePublicKey::createFromInteger (i); - info.label = String::fromNumber (i); - result.list.push_back (info); + Item item;; + item.publicKey = RipplePublicKey::createFromInteger (i); + item.label = String::fromNumber (i); + results.list.push_back (item); } } diff --git a/src/ripple/validators/impl/Tuning.h b/src/ripple/validators/impl/Tuning.h index 78483b7009..372f4d60e7 100644 --- a/src/ripple/validators/impl/Tuning.h +++ b/src/ripple/validators/impl/Tuning.h @@ -20,6 +20,8 @@ #ifndef RIPPLE_VALIDATORS_TUNING_H_INCLUDED #define RIPPLE_VALIDATORS_TUNING_H_INCLUDED +#include + namespace ripple { namespace Validators { @@ -41,22 +43,31 @@ enum // This tunes the preallocated arrays ,expectedNumberOfResults = 1000 - // How many elements in the aged history before we swap containers - ,maxSizeBeforeSwap = 100 + // NUmber of entries in the seen validations cache + ,seenValidationsCacheSize = 1000 + + // Number of entries in the seen ledgers cache + ,seenLedgersCacheSize = 1000 // about half an hour at 2/sec + + // Number of closed Ledger entries per Validator + ,ledgersPerValidator = 100 // this shouldn't be too large }; //------------------------------------------------------------------------------ -/** Associative container of unique keys. */ +/** Cycled associative map of unique keys. */ template + class Info, // per-container info + class Hash = typename Key::hasher, class KeyEqual = std::equal_to , - class Allocator = std::allocator > > + class Allocator = std::allocator > class CycledMap { private: - typedef boost::unordered_map ContainerType; + typedef boost::unordered_map < + Key, T, Hash, KeyEqual, Allocator> ContainerType; + typedef typename ContainerType::iterator iterator; public: typedef typename ContainerType::key_type key_type; @@ -64,34 +75,97 @@ public: typedef typename ContainerType::size_type size_type; typedef typename ContainerType::difference_type difference_type; typedef typename ContainerType::hasher hasher; + typedef typename ContainerType::key_equal key_equal; typedef typename ContainerType::allocator_type allocator_type; typedef typename ContainerType::reference reference; typedef typename ContainerType::const_reference const_reference; typedef typename ContainerType::pointer pointer; typedef typename ContainerType::const_pointer const_pointer; + explicit CycledMap ( + size_type item_max, + Hash hash = Hash(), + KeyEqual equal = KeyEqual(), + Allocator alloc = Allocator()) + : m_max (item_max) + , m_hash (hash) + , m_equal (equal) + , m_alloc (alloc) + , m_front (m_max, hash, equal, alloc) + , m_back (m_max, hash, equal, alloc) + { + } + + Info& front() + { return m_front_info; } + + Info const & front() const + { return m_front_info; } + + Info& back () + { return m_back_info; } + + Info const& back () const + { return m_back_info; } + + /** Returns `true` if the next real insert would swap. */ + bool full() const + { + return m_front.size() >= m_max; + } + + /** Insert the value if it doesn't already exist. */ + std::pair insert (value_type const& value) + { + if (full()) + cycle (); + iterator iter (m_back.find (value.first)); + if (iter != m_back.end()) + return std::make_pair ( + boost::ref (iter->second), + boost::ref (m_back_info)); + std::pair result ( + m_front.insert (value)); + return std::make_pair ( + boost::ref (result.first->second), + boost::ref (m_front_info)); + } + void cycle () { - m_front.clear (); std::swap (m_front, m_back); + m_front.clear (); +#if BOOST_VERSION > 105400 + m_front.reserve (m_max); +#endif + std::swap (m_front_info, m_back_info); + m_front_info.clear(); } private: + size_type m_max; + hasher m_hash; + key_equal m_equal; + allocator_type m_alloc; ContainerType m_front; ContainerType m_back; + Info m_front_info; + Info m_back_info; }; //------------------------------------------------------------------------------ -/** Associative container of unique keys. */ +/** Cycled set of unique keys. */ template + class Hash = typename Key::hasher, class KeyEqual = std::equal_to , class Allocator = std::allocator > class CycledSet { private: - typedef boost::unordered_set ContainerType; + typedef boost::unordered_set < + Key, Hash, KeyEqual, Allocator> ContainerType; + typedef typename ContainerType::iterator iterator; public: typedef typename ContainerType::key_type key_type; @@ -115,23 +189,47 @@ public: , m_hash (hash) , m_equal (equal) , m_alloc (alloc) - , m_front (hash, equal, alloc) - , m_back (hash, equal, alloc) + , m_front (m_max, hash, equal, alloc) + , m_back (m_max, hash, equal, alloc) { - m_front.reserve (m_max); - m_back.reserve (m_max); } + // Returns `true` if the next real insert would swap + bool full() const + { + return m_front.size() >= m_max; + } + + // Adds the key to the front if its not in either map + bool insert (key_type const& key) + { + if (full()) + cycle (); + if (m_back.find (key) != m_back.end()) + return false; + std::pair result ( + m_front.insert (key)); + if (result.second) + return true; + return false; + } + +#if 0 + bool find (key_type const& key) + { + if (m_front.find (key) != m_front.end()) + return true; + return m_back.find (key) != m_back.end(); + } +#endif + void cycle () { - m_front.clear (); std::swap (m_front, m_back); - } - - bool insert (value_type const& value) - { - std::size_t const hash (m_hash (value)); - + m_front.clear (); +#if BOOST_VERSION > 105400 + m_front.reserve (m_max); +#endif } private: diff --git a/src/ripple/validators/impl/Utilities.cpp b/src/ripple/validators/impl/Utilities.cpp index 25e128d5ae..41e3f6e623 100644 --- a/src/ripple/validators/impl/Utilities.cpp +++ b/src/ripple/validators/impl/Utilities.cpp @@ -75,7 +75,7 @@ struct Utilities::Helpers //------------------------------------------------------------------------------ bool Utilities::parseInfoLine ( - Source::Info& info, + Source::Item& item, std::string const& line, Journal journal) { @@ -88,14 +88,14 @@ bool Utilities::parseInfoLine ( std::string const encodedKey (match [1]); std::string const commentText (match [2]); - std::pair result ( + std::pair results ( RipplePublicKey::from_string (encodedKey)); - if (result.second) + if (results.second) { // We got a public key. - info.label = commentText; - info.publicKey = result.first; + item.label = commentText; + item.publicKey = results.first; success = true; } else @@ -120,17 +120,17 @@ bool Utilities::parseInfoLine ( //------------------------------------------------------------------------------ void Utilities::parseResultLine ( - Source::Result& result, + Source::Results& results, std::string const& line, Journal journal) { - Source::Info info; + Source::Item item; - bool success = parseInfoLine (info, line, journal); + bool success = parseInfoLine (item, line, journal); if (success) { - result.list.push_back (info); - result.success = true; + results.list.push_back (item); + results.success = true; } } diff --git a/src/ripple/validators/impl/Utilities.h b/src/ripple/validators/impl/Utilities.h index a0ac2c05e1..5035815960 100644 --- a/src/ripple/validators/impl/Utilities.h +++ b/src/ripple/validators/impl/Utilities.h @@ -29,12 +29,12 @@ class Utilities public: typedef std::vector Strings; - /** A suitable LineFunction for parsing items into a fetch result. */ + /** A suitable LineFunction for parsing items into a fetch results. */ class ParseResultLine { public: - ParseResultLine (Source::Result& result, Journal journal) - : m_result (&result) + ParseResultLine (Source::Results& results, Journal journal) + : m_result (&results) , m_journal (journal) { } @@ -46,7 +46,7 @@ public: } private: - Source::Result* m_result; + Source::Results* m_result; Journal m_journal; }; @@ -111,13 +111,13 @@ public: } } - /** Parse a string into the Source::Result. + /** Parse a string into the Source::Results. Invalid or comment lines will be skipped. - Lines containing validator info will be added to the Result object. - Metadata lines will update the corresponding Result fields. + Lines containing validator info will be added to the Results object. + Metadata lines will update the corresponding Results fields. */ static void parseResultLine ( - Source::Result& result, + Source::Results& results, std::string const& line, Journal journal = Journal()); @@ -131,11 +131,11 @@ public: struct Helpers; - /** Parse a string into a Source::Info. + /** Parse a string into a Source::Item. @return `true` on success. */ static bool parseInfoLine ( - Source::Info& info, std::string const& line, Journal journal); + Source::Item& item, std::string const& line, Journal journal); }; } diff --git a/src/ripple/validators/impl/Ledger.h b/src/ripple/validators/impl/Validation.h similarity index 57% rename from src/ripple/validators/impl/Ledger.h rename to src/ripple/validators/impl/Validation.h index 2609b85d28..6f2724ffad 100644 --- a/src/ripple/validators/impl/Ledger.h +++ b/src/ripple/validators/impl/Validation.h @@ -17,25 +17,40 @@ */ //============================================================================== -#ifndef RIPPLE_VALIDATORS_LEDGER_H_INCLUDED -#define RIPPLE_VALIDATORS_LEDGER_H_INCLUDED +#ifndef RIPPLE_VALIDATORS_VALIDATION_INCLUDED +#define RIPPLE_VALIDATORS_VALIDATION_INCLUDED namespace ripple { namespace Validators { -// Stored each time a ledger is closed -// -struct Ledger +/** Hash function for ReceivedValidation. */ +class ReceivedValidationHash { - Ledger() : when (RelativeTime::fromStartup()) +public: + std::size_t operator() (ReceivedValidation const& key) const { + return m_ledger_hasher (key.ledgerHash) + + m_key_hasher (key.publicKey); } - RelativeTime when; +private: + RippleLedgerHash::hasher m_ledger_hasher; + RipplePublicKey::hasher m_key_hasher; }; -typedef AgedHistory > LedgerMap; +//------------------------------------------------------------------------------ + +/** KeyEqual function for ReceivedValidation. */ +class ReceivedValidationKeyEqual +{ +public: + bool operator() (ReceivedValidation const& lhs, + ReceivedValidation const& rhs) const + { + return lhs.ledgerHash == rhs.ledgerHash && + lhs.publicKey == rhs.publicKey; + } +}; } } diff --git a/src/ripple/validators/impl/Validator.h b/src/ripple/validators/impl/Validator.h index 2b77cf3426..eb66857e69 100644 --- a/src/ripple/validators/impl/Validator.h +++ b/src/ripple/validators/impl/Validator.h @@ -23,155 +23,91 @@ namespace ripple { namespace Validators { -// Stored for each distinguishable Validator in the trusted list. -// These are kept in an associative container or multi-index container. -// +/** Tracks statistics on a validator. */ class Validator { -public: - Validator () : refCount (0) +private: + /** State of a ledger. */ + struct Ledger { - } + Ledger() : closed (false), received (false) + { } - void receiveValidation (RippleLedgerHash const& ledgerHash) - { - typedef LedgerMap::container_type::iterator iterator; - - ++count->seen; - - // If we already have it in the expected list, close it out - // - iterator iter (expected->find (ledgerHash)); - if (iter != expected->end()) - { - expected->erase (iter); - expected.back().erase (ledgerHash); - return; - } - else if ((iter = expected.back().find(ledgerHash)) != - expected.back().end()) - { - expected.back().erase (iter); - return; - } - - // Ledger hasn't closed yet so put it in the received list - // - std::pair result ( - received->emplace (ledgerHash, Ledger())); - bassert (result.second); - if (received->size() >= maxSizeBeforeSwap) - swap(); - } - - void ledgerClosed (RippleLedgerHash const& ledgerHash) - { - typedef LedgerMap::container_type::iterator iterator; - - ++count->closed; - - // If the Validator already gave us the ledger - // then count it and remove it from both tables. - // - iterator iter (received->find (ledgerHash)); - if (iter != received->end()) - { - received->erase (iter); - received.back().erase (ledgerHash); - return; - } - else if ((iter = received.back().find (ledgerHash)) != - received.back().end()) - { - received.back().erase (iter); - return; - } - - // We haven't seen this ledger hash from the - // validator yet so put it on the expected list - // - std::pair result ( - expected->emplace (ledgerHash, Ledger ())); - bassert (result.second); - if (expected->size() >= maxSizeBeforeSwap) - swap(); - } - - void swap() - { - // Count anything in the old expected list as missing - count->missing += expected.back().size(); - - // Count anything in the old received list as orphaned - count->orphans += received.back().size(); - - // Rotate and clear - count.swap(); - expected.swap(); - received.swap(); - count->clear(); - expected->clear(); - received->clear(); - } - - struct Count - { - Count() - : closed (0) - , seen (0) - , missing (0) - , orphans (0) - { - } - - void clear () - { - *this = Count(); - } - - // How many LedgerMap we've seen - std::size_t closed; - - // How many validation's we've seen - std::size_t seen; - - // Estimate of validation's that were missed - std::size_t missing; - - // Estimate of validations not belonging to any ledger - std::size_t orphans; + bool closed; // `true` if the ledger was closed + bool received; // `true` if we got a validation }; - int refCount; + /** Number of sources that reference this validator. */ + int m_refCount; - AgedHistory count; - LedgerMap received; - LedgerMap expected; + /** Holds the state of all recent ledgers for this validator. */ + /** @{ */ + typedef CycledMap LedgerMap; + LedgerMap m_ledgers; + /** @} */ + +public: + Validator () + : m_refCount (0) + , m_ledgers (ledgersPerValidator) + { + } + + /** Increment the number of references to this validator. */ + void addRef () + { ++m_refCount; } + + /** Decrement the number of references to this validator. + When the reference count reaches zero, the validator will + be removed and no longer tracked. + */ + bool release () + { return (--m_refCount) == 0; } + + /** Returns the composite performance statistics. */ + Count count () const + { return m_ledgers.front() + m_ledgers.back(); } + + /** Called upon receipt of a validation. */ + void receiveValidation (RippleLedgerHash const& ledgerHash) + { + std::pair result (m_ledgers.insert ( + std::make_pair (ledgerHash, Ledger()))); + Ledger& ledger (result.first); + Count& count (result.second); + ledger.received = true; + if (ledger.closed) + { + --count.expected; + ++count.closed; + } + else + { + ++count.received; + } + } + + /** Called when a ledger is closed. */ + void ledgerClosed (RippleLedgerHash const& ledgerHash) + { + std::pair result (m_ledgers.insert ( + std::make_pair (ledgerHash, Ledger()))); + Ledger& ledger (result.first); + Count& count (result.second); + ledger.closed = true; + if (ledger.received) + { + --count.received; + ++count.closed; + } + else + { + ++count.expected; + } + } }; -//------------------------------------------------------------------------------ - -typedef boost::unordered_map < - RipplePublicKey, Validator, - RipplePublicKey::hasher> ValidatorMap; - -// The master in-memory database of Validator, indexed by all the -// possible things that we need to care about, and even some that we don't. -// -/* -typedef boost::multi_index_container < - Validator, boost::multi_index::indexed_by < - - boost::multi_index::hashed_unique < - BOOST_MULTI_INDEX_MEMBER(Logic::Validator,UniqueID,uniqueID)>, - - boost::multi_index::hashed_unique < - BOOST_MULTI_INDEX_MEMBER(Logic::Validator,IPEndpoint,endpoint), - Connectible::HashAddress> - > -> ValidationsMap; -*/ - } } diff --git a/src/ripple/validators/ripple_validators.cpp b/src/ripple/validators/ripple_validators.cpp index c286505bfb..ba2a104d98 100644 --- a/src/ripple/validators/ripple_validators.cpp +++ b/src/ripple/validators/ripple_validators.cpp @@ -42,14 +42,15 @@ using namespace beast; # include "impl/Tuning.h" # include "impl/ChosenList.h" +# include "impl/Count.h" # include "impl/SourceFile.h" # include "impl/SourceStrings.h" # include "impl/SourceURL.h" -# include "impl/Utilities.h" # include "impl/SourceDesc.h" # include "impl/Store.h" # include "impl/StoreSqdb.h" -# include "impl/Ledger.h" +# include "impl/Utilities.h" +# include "impl/Validation.h" # include "impl/Validator.h" #include "impl/Logic.h" diff --git a/src/ripple_app/peers/Peer.cpp b/src/ripple_app/peers/Peer.cpp index 9e4665a8a3..47b946b0e9 100644 --- a/src/ripple_app/peers/Peer.cpp +++ b/src/ripple_app/peers/Peer.cpp @@ -1483,7 +1483,6 @@ static void checkValidation (Job&, SerializedValidation::pointer val, bool isTru Validators::ReceivedValidation rv; rv.ledgerHash = sv.getLedgerHash (); rv.publicKey = sv.getSignerPublic(); - rv.publicKeyHash = sv.getSignerPublic(); getApp ().getValidators ().receiveValidation (rv); } //