From 2f6af906f4262ce382bcf1ea9c6a67c2dc921f30 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Tue, 18 Nov 2014 15:43:33 -0800 Subject: [PATCH] Validators work (RIPD-703): This replaces the experimental validators module with foundational code to implement a new system for tracking validators, validations and the UNL. The code is turned off by default, in BeastConfig.h * Remove obsolete public Manager interfaces * Remove obsolete database methods * Remove obsolete ChosenList concept * Remove obsolete code * Add missing includes * Tidy up STValidation.h * Move factory function to Validators::make_Manager * Add Connection object for tracking STValidations --- Builds/VisualStudio2013/RippleD.vcxproj | 45 +- .../VisualStudio2013/RippleD.vcxproj.filters | 58 +- src/BeastConfig.h | 7 + src/ripple/app/ledger/LedgerMaster.cpp | 14 +- src/ripple/app/main/Application.cpp | 26 +- src/ripple/overlay/impl/PeerImp.cpp | 13 +- src/ripple/overlay/impl/PeerImp.h | 5 + src/ripple/protocol/STValidation.h | 9 +- src/ripple/types/IdentifierType.h | 2 + src/ripple/types/RippleLedgerHash.h | 2 + src/ripple/unity/validators.cpp | 7 +- .../{impl/SourceURL.h => Connection.h} | 15 +- src/ripple/validators/Manager.h | 66 +- src/ripple/validators/Source.h | 91 --- src/ripple/validators/impl/ChosenList.h | 81 --- .../impl/{Source.cpp => ConnectionImp.cpp} | 13 +- src/ripple/validators/impl/ConnectionImp.h | 207 +++++++ src/ripple/validators/impl/Count.h | 87 --- src/ripple/validators/impl/Logic.cpp | 148 +++++ src/ripple/validators/impl/Logic.h | 586 ++---------------- src/ripple/validators/impl/Manager.cpp | 307 ++++----- src/ripple/validators/impl/SourceDesc.h | 71 --- src/ripple/validators/impl/SourceFile.cpp | 94 --- src/ripple/validators/impl/SourceFile.h | 38 -- src/ripple/validators/impl/SourceStrings.cpp | 80 --- src/ripple/validators/impl/SourceStrings.h | 39 -- src/ripple/validators/impl/SourceURL.cpp | 105 ---- src/ripple/validators/impl/Store.h | 16 +- src/ripple/validators/impl/StoreSqdb.cpp | 537 +--------------- src/ripple/validators/impl/StoreSqdb.h | 44 +- src/ripple/validators/impl/Tests.cpp | 252 +------- src/ripple/validators/impl/Tuning.h | 28 - src/ripple/validators/impl/Utilities.cpp | 140 ----- src/ripple/validators/impl/Utilities.h | 136 ---- src/ripple/validators/impl/Validation.h | 58 -- src/ripple/validators/impl/Validator.h | 151 ----- .../validators/{Types.h => make_Manager.h} | 33 +- .../validators/tests/Validators.test.cpp | 95 +-- 38 files changed, 666 insertions(+), 3040 deletions(-) rename src/ripple/validators/{impl/SourceURL.h => Connection.h} (74%) delete mode 100644 src/ripple/validators/Source.h delete mode 100644 src/ripple/validators/impl/ChosenList.h rename src/ripple/validators/impl/{Source.cpp => ConnectionImp.cpp} (85%) create mode 100644 src/ripple/validators/impl/ConnectionImp.h delete mode 100644 src/ripple/validators/impl/Count.h create mode 100644 src/ripple/validators/impl/Logic.cpp delete mode 100644 src/ripple/validators/impl/SourceDesc.h delete mode 100644 src/ripple/validators/impl/SourceFile.cpp delete mode 100644 src/ripple/validators/impl/SourceFile.h delete mode 100644 src/ripple/validators/impl/SourceStrings.cpp delete mode 100644 src/ripple/validators/impl/SourceStrings.h delete mode 100644 src/ripple/validators/impl/SourceURL.cpp delete mode 100644 src/ripple/validators/impl/Utilities.cpp delete mode 100644 src/ripple/validators/impl/Utilities.h delete mode 100644 src/ripple/validators/impl/Validation.h delete mode 100644 src/ripple/validators/impl/Validator.h rename src/ripple/validators/{Types.h => make_Manager.h} (68%) diff --git a/Builds/VisualStudio2013/RippleD.vcxproj b/Builds/VisualStudio2013/RippleD.vcxproj index 98376d276..224400896 100644 --- a/Builds/VisualStudio2013/RippleD.vcxproj +++ b/Builds/VisualStudio2013/RippleD.vcxproj @@ -156,6 +156,8 @@ True + + @@ -3299,35 +3301,21 @@ - + - + + True + + + + True + True - - True - - - - - True - - - - - True - - - - - True - - - @@ -3340,24 +3328,13 @@ - - True - - - - - - + - - True - - True diff --git a/Builds/VisualStudio2013/RippleD.vcxproj.filters b/Builds/VisualStudio2013/RippleD.vcxproj.filters index 31252e9dc..1cb10beeb 100644 --- a/Builds/VisualStudio2013/RippleD.vcxproj.filters +++ b/Builds/VisualStudio2013/RippleD.vcxproj.filters @@ -615,6 +615,9 @@ beast\asio\tests + + beast\asio + beast @@ -4485,42 +4488,24 @@ ripple\unity - + + ripple\validators + + + ripple\validators\impl + + ripple\validators\impl - + ripple\validators\impl - + ripple\validators\impl ripple\validators\impl - - ripple\validators\impl - - - ripple\validators\impl - - - ripple\validators\impl - - - ripple\validators\impl - - - ripple\validators\impl - - - ripple\validators\impl - - - ripple\validators\impl - - - ripple\validators\impl - ripple\validators\impl @@ -4536,30 +4521,15 @@ ripple\validators\impl - - ripple\validators\impl - - - ripple\validators\impl - - - ripple\validators\impl - - - ripple\validators\impl - - + ripple\validators - + ripple\validators ripple\validators\tests - - ripple\validators - ripple\websocket\autosocket diff --git a/src/BeastConfig.h b/src/BeastConfig.h index 365a711fe..1f7d5c6b4 100644 --- a/src/BeastConfig.h +++ b/src/BeastConfig.h @@ -197,4 +197,11 @@ #define RIPPLE_SINGLE_IO_SERVICE_THREAD 0 #endif +/** Config: RIPPLE_HOOK_VALIDATORS + Activates code for handling validations and validators (work in progress). +*/ +#ifndef RIPPLE_HOOK_VALIDATORS +#define RIPPLE_HOOK_VALIDATORS 0 +#endif + #endif diff --git a/src/ripple/app/ledger/LedgerMaster.cpp b/src/ripple/app/ledger/LedgerMaster.cpp index bf9af059a..663eed7a0 100644 --- a/src/ripple/app/ledger/LedgerMaster.cpp +++ b/src/ripple/app/ledger/LedgerMaster.cpp @@ -200,6 +200,11 @@ public: mValidLedgerSeq = l->getLedgerSeq(); getApp().getOPs().updateLocalTx (l); getApp().getSHAMapStore().onLedgerClosed (getValidatedLedger()); + + #if RIPPLE_HOOK_VALIDATORS + getApp().getValidators().onLedgerClosed (l->getLedgerSeq(), + l->getHash(), l->getParentHash()); + #endif } void setPubLedger(Ledger::ref l) @@ -666,15 +671,6 @@ public: } } } - - //-------------------------------------------------------------------------- - // - { - if (isCurrent) - getApp ().getValidators ().on_ledger_closed (ledger->getHash()); - } - // - //-------------------------------------------------------------------------- } void failedSave(std::uint32_t seq, uint256 const& hash) diff --git a/src/ripple/app/main/Application.cpp b/src/ripple/app/main/Application.cpp index 870734fd7..3ceeea62c 100644 --- a/src/ripple/app/main/Application.cpp +++ b/src/ripple/app/main/Application.cpp @@ -37,7 +37,7 @@ #include #include #include -#include +#include #include #include #include @@ -318,10 +318,8 @@ public: , m_sntpClient (SNTPClient::New (*this)) - , m_validators (add (Validators::Manager::New ( - *this, - getConfig ().getModuleDatabasePath (), - m_logs.journal("Validators")))) + , m_validators (Validators::make_Manager(*this, get_io_service(), + getConfig ().getModuleDatabasePath (), m_logs.journal("UVL"))) , m_amendmentTable (make_AmendmentTable (weeks(2), MAJORITY_FRACTION, m_logs.journal("AmendmentTable"))) @@ -365,6 +363,7 @@ public: // VFALCO HACK m_nodeStoreScheduler.setJobQueue (*m_jobQueue); + add (*m_validators); add (m_ledgerMaster->getPropertySource ()); add (*serverHandler_); } @@ -787,28 +786,13 @@ public: } } - //-------------------------------------------------------------------------- - - // Initialize the Validators object with Config information. - void prepareValidators () - { - m_validators->addStrings ("rippled.cfg", getConfig().validators); - - if (! getConfig().getValidatorsURL().empty()) - m_validators->addURL (getConfig().getValidatorsURL()); - - if (getConfig().getValidatorsFile() != beast::File::nonexistent ()) - m_validators->addFile (getConfig().getValidatorsFile()); - } - //-------------------------------------------------------------------------- // // Stoppable // - void onPrepare () + void onPrepare() override { - prepareValidators (); } void onStart () diff --git a/src/ripple/overlay/impl/PeerImp.cpp b/src/ripple/overlay/impl/PeerImp.cpp index 0b2850822..af575bb62 100644 --- a/src/ripple/overlay/impl/PeerImp.cpp +++ b/src/ripple/overlay/impl/PeerImp.cpp @@ -62,6 +62,7 @@ PeerImp::PeerImp (id_t id, endpoint_type remote_endpoint, , usage_(consumer) , slot_ (slot) , http_message_(std::move(request)) + , validatorsConnection_(getApp().getValidators().newConnection(id)) { } @@ -1784,15 +1785,9 @@ PeerImp::checkValidation (Job&, STValidation::pointer val, return; } - //---------------------------------------------------------------------- - { - STValidation const& sv (*val); - Validators::ReceivedValidation rv; - rv.ledgerHash = sv.getLedgerHash (); - rv.publicKey = sv.getSignerPublic(); - getApp ().getValidators ().on_receive_validation (rv); - } - //---------------------------------------------------------------------- + #if RIPPLE_HOOK_VALIDATORS + validatorsConnection_->onValidation(*val); + #endif std::set peers; if (getApp().getOPs ().recvValidation (val, std::to_string(id())) && diff --git a/src/ripple/overlay/impl/PeerImp.h b/src/ripple/overlay/impl/PeerImp.h index 063146178..715ef1afc 100644 --- a/src/ripple/overlay/impl/PeerImp.h +++ b/src/ripple/overlay/impl/PeerImp.h @@ -31,6 +31,7 @@ #include #include #include // VFALCO REMOVE +#include #include #include #include @@ -152,6 +153,8 @@ private: std::unique_ptr load_event_; + std::unique_ptr validatorsConnection_; + //-------------------------------------------------------------------------- public: @@ -467,6 +470,7 @@ PeerImp::PeerImp (id_t id, endpoint_type remote_endpoint, , m_inbound (true) , state_ (State::connected) , slot_ (slot) + , validatorsConnection_(getApp().getValidators().newConnection(id)) { read_buffer_.commit(boost::asio::buffer_copy(read_buffer_.prepare( boost::asio::buffer_size(buffers)), buffers)); @@ -497,6 +501,7 @@ PeerImp::PeerImp (std::unique_ptr&& ssl_bundle, , hello_ (std::move(hello)) , usage_ (usage) , slot_ (std::move(slot)) + , validatorsConnection_(getApp().getValidators().newConnection(id)) { read_buffer_.commit (boost::asio::buffer_copy(read_buffer_.prepare( boost::asio::buffer_size(buffers)), buffers)); diff --git a/src/ripple/protocol/STValidation.h b/src/ripple/protocol/STValidation.h index 7ffbf698a..be77e4e41 100644 --- a/src/ripple/protocol/STValidation.h +++ b/src/ripple/protocol/STValidation.h @@ -17,8 +17,13 @@ */ //============================================================================== -#ifndef RIPPLE_SERIALIZEDVALIDATION_H -#define RIPPLE_SERIALIZEDVALIDATION_H +#ifndef RIPPLE_PROTOCOL_STVALIDATION_H_INCLUDED +#define RIPPLE_PROTOCOL_STVALIDATION_H_INCLUDED + +#include +#include +#include +#include namespace ripple { diff --git a/src/ripple/types/IdentifierType.h b/src/ripple/types/IdentifierType.h index 84744a371..4188e442c 100644 --- a/src/ripple/types/IdentifierType.h +++ b/src/ripple/types/IdentifierType.h @@ -20,6 +20,8 @@ #ifndef RIPPLE_TYPES_IDENTIFIERTYPE_H_INCLUDED #define RIPPLE_TYPES_IDENTIFIERTYPE_H_INCLUDED +#include +#include #include #include #include diff --git a/src/ripple/types/RippleLedgerHash.h b/src/ripple/types/RippleLedgerHash.h index ffb02b2b7..cecb75a89 100644 --- a/src/ripple/types/RippleLedgerHash.h +++ b/src/ripple/types/RippleLedgerHash.h @@ -20,6 +20,8 @@ #ifndef RIPPLE_TYPES_RIPPLELEDGERHASH_H_INCLUDED #define RIPPLE_TYPES_RIPPLELEDGERHASH_H_INCLUDED +#include +#include #include namespace ripple { diff --git a/src/ripple/unity/validators.cpp b/src/ripple/unity/validators.cpp index c74836806..9b4abad7c 100644 --- a/src/ripple/unity/validators.cpp +++ b/src/ripple/unity/validators.cpp @@ -19,13 +19,10 @@ #include +#include +#include #include -#include -#include -#include -#include #include #include -#include #include diff --git a/src/ripple/validators/impl/SourceURL.h b/src/ripple/validators/Connection.h similarity index 74% rename from src/ripple/validators/impl/SourceURL.h rename to src/ripple/validators/Connection.h index fa040716c..3632448bb 100644 --- a/src/ripple/validators/impl/SourceURL.h +++ b/src/ripple/validators/Connection.h @@ -17,17 +17,22 @@ */ //============================================================================== -#ifndef RIPPLE_VALIDATORS_SOURCEURL_H_INCLUDED -#define RIPPLE_VALIDATORS_SOURCEURL_H_INCLUDED +#ifndef RIPPLE_VALIDATORS_CONNECTION_H_INCLUDED +#define RIPPLE_VALIDATORS_CONNECTION_H_INCLUDED + +#include namespace ripple { namespace Validators { -/** Provides validators from a trusted URI (e.g. HTTPS) */ -class SourceURL : public Source +/** Represents validator concerns on a protocol connection. */ +class Connection { public: - static SourceURL* New (beast::URL const& url); + virtual ~Connection() = default; + + /** Called when a signed validation is received on the connection. */ + virtual void onValidation (STValidation const& v) = 0; }; } diff --git a/src/ripple/validators/Manager.h b/src/ripple/validators/Manager.h index 67904d51e..7ddaf03b4 100644 --- a/src/ripple/validators/Manager.h +++ b/src/ripple/validators/Manager.h @@ -20,8 +20,9 @@ #ifndef RIPPLE_VALIDATORS_MANAGER_H_INCLUDED #define RIPPLE_VALIDATORS_MANAGER_H_INCLUDED -#include -#include +#include +#include +#include #include #include #include @@ -41,64 +42,23 @@ protected: Manager(); public: - /** Create a new Manager object. - @param parent The parent Stoppable. - @param pathToDbFileOrDirectory The directory where our database is stored - @param journal Where to send log output. - */ - static Manager* New ( - beast::Stoppable& stoppableParent, - beast::File const& pathToDbFileOrDirectory, - beast::Journal journal); - /** Destroy the object. Any pending source fetch operations are aborted. This will block until any pending database I/O has completed and the thread has stopped. */ - virtual ~Manager () { } + virtual ~Manager() = default; - /** Add a static source of validators. - The validators added using these methods will always be chosen when - constructing the UNL regardless of statistics. The fetch operation - is performed asynchronously, so this call returns immediately. A - failed fetch (depending on the source) is not retried. The caller - loses ownership of any dynamic objects. - Thread safety: - Can be called from any thread. - */ - /** @{ */ - virtual void addStrings (std::string const& name, - std::vector const& strings) = 0; - virtual void addFile (beast::File const& file) = 0; - virtual void addStaticSource (Validators::Source* source) = 0; - /** @} */ + /** Create a new Connection. */ + virtual + std::unique_ptr + newConnection (int id) = 0; - /** Add a live source of validators from a trusted URL. - The URL will be contacted periodically to update the list. The fetch - operation is performed asynchronously, this call doesn't block. - Thread safety: - Can be called from any thread. - */ - virtual void addURL (beast::URL const& url) = 0; - - /** Add a live source of validators. - The caller loses ownership of the object. The fetch is performed - asynchronously, this call doesn't block. - Thread safety: - Can be called from any thread. - */ - virtual void addSource (Validators::Source* source) = 0; - - //-------------------------------------------------------------------------- - - //virtual bool isPublicKeyTrusted (RipplePublicKey const& publicKey) = 0; - - /** Callback to call when a properly signed validation is received. */ - virtual void on_receive_validation (ReceivedValidation const& rv) = 0; - - /** Callback to call when a ledger is closed. */ - virtual void on_ledger_closed (RippleLedgerHash const& ledgerHash) = 0; + /** Called when a ledger is built. */ + virtual + void + onLedgerClosed (LedgerIndex index, + LedgerHash const& hash, LedgerHash const& parent) = 0; }; } diff --git a/src/ripple/validators/Source.h b/src/ripple/validators/Source.h deleted file mode 100644 index cee124c38..000000000 --- a/src/ripple/validators/Source.h +++ /dev/null @@ -1,91 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_VALIDATORS_SOURCE_H_INCLUDED -#define RIPPLE_VALIDATORS_SOURCE_H_INCLUDED - -#include -#include -#include -#include - -namespace ripple { -namespace Validators { - -/** A source of validator descriptors. */ -class Source : public beast::SharedObject -{ -public: - /** A Source's descriptor for a Validator. */ - struct Item - { - /** The unique key for this validator. */ - RipplePublicKey publicKey; - - /** Optional human readable comment describing the validator. */ - std::string label; - }; - - /** Destroy the Source. - This can be called from any thread. If the Source is busy - fetching, the destructor must block until the operation is either - canceled or complete. - */ - virtual ~Source () { } - - /** The name of the source, used in diagnostic output. */ - virtual std::string to_string () const = 0; - - /** An identifier that uniquely describes the source. - This is used for identification in the database. - */ - virtual std::string uniqueID () const = 0; - - /** A string that is used to recreate the source from the database entry. */ - virtual std::string createParam () = 0; - - /** Cancel any pending fetch. - The default implementation does nothing. - */ - virtual void cancel () { } - - /** Fetch results. - This call will block - */ - /** @{ */ - struct Results - { - Results (); - - bool success; - std::string message; - // VFALCO TODO Replace with chrono - beast::Time expirationTime; - std::vector list; - }; - virtual void fetch (Results& results, beast::Journal journal) = 0; - /** @} */ -}; - -std::ostream& operator<< (std::ostream& os, Source const& v); - -} -} - -#endif diff --git a/src/ripple/validators/impl/ChosenList.h b/src/ripple/validators/impl/ChosenList.h deleted file mode 100644 index d7087851d..000000000 --- a/src/ripple/validators/impl/ChosenList.h +++ /dev/null @@ -1,81 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_VALIDATORS_CHOSENLIST_H_INCLUDED -#define RIPPLE_VALIDATORS_CHOSENLIST_H_INCLUDED - -#include -#include - -namespace ripple { -namespace Validators { - -class ChosenList : public beast::SharedObject -{ -public: - typedef beast::SharedPtr Ptr; - - struct Info - { - Info () - { - } - }; - - typedef hardened_hash_map MapType; - - ChosenList (std::size_t expectedSize = 0) - { - // Available only in recent boost versions? - //m_map.reserve (expectedSize); - } - - MapType const& map() const - { - return m_map; - } - - std::size_t size () const noexcept - { - return m_map.size (); - } - - void insert (RipplePublicKey const& key, Info const& info) noexcept - { - m_map [key] = info; - } - - bool containsPublicKey (RipplePublicKey const& publicKey) const noexcept - { - return m_map.find (publicKey) != m_map.cend (); - } - - bool containsPublicKeyHash (RipplePublicKeyHash const& publicKeyHash) const noexcept - { - return false; - } - -private: - MapType m_map; -}; - -} -} - -#endif diff --git a/src/ripple/validators/impl/Source.cpp b/src/ripple/validators/impl/ConnectionImp.cpp similarity index 85% rename from src/ripple/validators/impl/Source.cpp rename to src/ripple/validators/impl/ConnectionImp.cpp index 7a7358e3d..716d1c8a8 100644 --- a/src/ripple/validators/impl/Source.cpp +++ b/src/ripple/validators/impl/ConnectionImp.cpp @@ -17,20 +17,13 @@ */ //============================================================================== +#include + namespace ripple { namespace Validators { -Source::Results::Results () - : success (false) - , message ("uninitialized") -{ -} -std::ostream& operator<< (std::ostream& os, Source const& v) -{ - os << v.to_string(); - return os; -} + } } diff --git a/src/ripple/validators/impl/ConnectionImp.h b/src/ripple/validators/impl/ConnectionImp.h new file mode 100644 index 000000000..2156567f5 --- /dev/null +++ b/src/ripple/validators/impl/ConnectionImp.h @@ -0,0 +1,207 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_VALIDATORS_CONNECTIONIMP_H_INCLUDED +#define RIPPLE_VALIDATORS_CONNECTIONIMP_H_INCLUDED + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace ripple { +namespace Validators { + +class ConnectionImp + : public Connection +{ +private: + // Metadata on a validation source + struct Source + { + // New sources are just at the threshold + double score = 0.8; + + // returns `true` if the score is high + // enough to count as available + bool + available() const + { + return score >= 0.8; + } + + // returns `true` if the score is so low we + // have no expectation of seeing the validator again + bool + gone() const + { + return score <= 0.2; + } + + // returns `true` if became unavailable + bool + onHit() + { + bool const prev = available(); + score = 0.90 * score + 0.10; + if (! prev && available()) + return true; + return false; + } + + // returns `true` if became available + bool + onMiss() + { + bool const prev = available(); + score = 0.90 * score; + if (prev && ! available()) + return true; + return false; + } + }; + + using Item = std::pair; + + Logic& logic_; + beast::WrappedSink sink_; + beast::Journal journal_; + std::mutex mutex_; + boost::optional ledger_; + boost::container::flat_set items_; + boost::container::flat_map sources_; + boost::container::flat_set good_; + + static + std::string + makePrefix (int id) + { + std::stringstream ss; + ss << "[" << std::setfill('0') << std::setw(3) << id << "] "; + return ss.str(); + } + +public: + template + ConnectionImp (int id, Logic& logic, Clock& clock) + : logic_ (logic) + , sink_ (logic.journal(), makePrefix(id)) + , journal_ (sink_) + { + logic_.add(*this); + } + + ~ConnectionImp() + { + logic_.remove(*this); + } + + void + onValidation (STValidation const& v) override + { + auto const key = v.getSignerPublic(); + auto const ledger = v.getLedgerHash(); + + { + std::lock_guard lock(mutex_); + if (! items_.emplace(ledger, key).second) + return; + if (journal_.debug) journal_.debug << + "onValidation: " << ledger; +#if 0 + auto const result = sources_.emplace( + std::piecewise_construct, std::make_tuple(key), + std::make_tuple()); +#else + // Work-around for boost::container + auto const result = sources_.emplace(key, Source{}); +#endif + if (result.second) + good_.insert(key); + // register a hit for slightly late validations + if (ledger_ && ledger == ledger_) + if (result.first->second.onHit()) + good_.insert(key); + } + + // This can call onLedger, do it last + logic_.onValidation(v); + } + + // Called when a supermajority of + // validations are received for the next ledger. + void + onLedger (LedgerHash const& ledger) + { + std::lock_guard lock(mutex_); + if (journal_.debug) journal_.debug << + "onLedger: " << ledger; + assert(ledger != ledger_); + ledger_ = ledger; + auto item = items_.lower_bound( + std::make_pair(ledger, RippleAddress())); + auto source = sources_.begin(); + while (item != items_.end() && + source != sources_.end() && + item->first == ledger) + { + if (item->second < source->first) + { + ++item; + } + else if (item->second == source->first) + { + if (source->second.onHit()) + good_.insert(source->first); + ++item; + ++source; + } + else + { + if (source->second.onMiss()) + good_.erase(source->first); + ++source; + } + } + while (source != sources_.end()) + { + if (source->second.onMiss()) + good_.erase(source->first); + ++source; + } + // VFALCO What if there are validations + // for the ledger AFTER this one in the map? + items_.clear(); + } +}; + +} +} + +#endif diff --git a/src/ripple/validators/impl/Count.h b/src/ripple/validators/impl/Count.h deleted file mode 100644 index fa33a6eb2..000000000 --- a/src/ripple/validators/impl/Count.h +++ /dev/null @@ -1,87 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_VALIDATORS_COUNT_H_INCLUDED -#define RIPPLE_VALIDATORS_COUNT_H_INCLUDED - -namespace ripple { -namespace Validators { - -/** Measures Validator performance statistics. */ -struct Count -{ - Count() - : received (0) - , expected (0) - , closed (0) - { - } - - Count (std::size_t received_, - std::size_t expected_, - std::size_t closed_) - : received (received_) - , expected (expected_) - , closed (closed_) - { - } - - /** Reset the statistics. */ - void clear () - { - *this = Count(); - } - - /** Returns the percentage of ledger participation. */ - int percent () const - { - int const total (closed + expected); - if (total > 0) - return (closed * 100) / total; - return 0; - } - - /** Returns the percentage of orphaned validations. */ - int percent_orphaned () const - { - int const total (received + closed); - if (total > 0) - return (received * 100) / total; - return 0; - } - - /** Output to PropertyStream. */ - void onWrite (beast::PropertyStream::Map& map) const - { - map["received"] = received; - map["expected"] = expected; - map["closed"] = closed; - map["percent"] = percent (); - map["percent_orphan"] = percent_orphaned(); - } - - std::size_t received; // Count of validations without a closed ledger - std::size_t expected; // Count of closed ledgers without a validation - std::size_t closed; // Number of validations with closed ledgers -}; - -} -} - -#endif diff --git a/src/ripple/validators/impl/Logic.cpp b/src/ripple/validators/impl/Logic.cpp new file mode 100644 index 000000000..b69742a28 --- /dev/null +++ b/src/ripple/validators/impl/Logic.cpp @@ -0,0 +1,148 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include + +/* + +Questions the code should answer: + +Most important thing that we do: + Determine the new last fully validated ledger + + + +- Are we robustly connected to the Ripple network? + +- Given a new recent validation for a ledger with a sequence number higher + than the last fully validated ledger, do we have a new last fully validated + ledger? + +- What's the latest fully validated ledger? + + Sequence number must always be known to set a fully validated ledger + + Accumulate validations from nodes you trust at least a little bit, + and that aren't stale. + + If you have a last fully validated ledger then validations for ledgers + with lower sequence numbers can be ignored. + + Flow of validations recent in time for sequence numbers greater or equal than + the last fully validated ledger. + +- What ledger is the current consenus round built on? + +- Determine when the current consensus round is over? + Criteria: Number of validations for a ledger that comes after. + +*/ + +namespace ripple { +namespace Validators { + +Logic::Logic (Store& store, beast::Journal journal) + : /*store_ (store) + , */journal_ (journal) + , ledgers_(get_seconds_clock()) +{ +} + +void +Logic::stop() +{ +} + +void +Logic::load() +{ +} + +void +Logic::add (ConnectionImp& c) +{ + std::lock_guard lock(mutex_); + connections_.insert(&c); +} + +void +Logic::remove (ConnectionImp& c) +{ + std::lock_guard lock(mutex_); + connections_.erase(&c); +} + +bool +Logic::isStale (STValidation const& v) +{ + return false; +} + +void +Logic::onTimer() +{ + std::lock_guard lock(mutex_); + beast::expire(ledgers_, std::chrono::minutes(5)); +} + +void +Logic::onValidation (STValidation const& v) +{ + assert(v.isFieldPresent (sfLedgerSequence)); + auto const seq_no = + v.getFieldU32 (sfLedgerSequence); + auto const key = v.getSignerPublic(); + auto const ledger = v.getLedgerHash(); + + std::lock_guard lock(mutex_); + if (journal_.trace) journal_.trace << + "onValidation: " << ledger; + auto const result = ledgers_.emplace (std::piecewise_construct, + std::make_tuple(ledger), std::make_tuple()); + auto& meta = result.first->second; + assert(result.second || seq_no == meta.seq_no); + if (result.second) + meta.seq_no = seq_no; + meta.keys.insert (v.getSignerPublic()); + if (meta.seq_no > latest_.second.seq_no) + { + if (policy_.acceptLedgerMeta (*result.first)) + { + //ledgers_.clear(); + latest_ = *result.first; + if (journal_.info) journal_.info << + "Accepted " << latest_.second.seq_no << + " (" << ledger << ")"; + for (auto& _ : connections_) + _->onLedger(latest_.first); + } + } +} + +void +Logic::onLedgerClosed (LedgerIndex index, + LedgerHash const& hash, LedgerHash const& parent) +{ + if (journal_.info) journal_.info << + "onLedgerClosed: " << index << " " << hash << " (parent " << parent << ")"; +} + +} +} diff --git a/src/ripple/validators/impl/Logic.h b/src/ripple/validators/impl/Logic.h index 854785035..b23be4b8a 100644 --- a/src/ripple/validators/impl/Logic.h +++ b/src/ripple/validators/impl/Logic.h @@ -20,568 +20,94 @@ #ifndef RIPPLE_VALIDATORS_LOGIC_H_INCLUDED #define RIPPLE_VALIDATORS_LOGIC_H_INCLUDED +#include +#include +#include #include -#include -#include -#include #include #include +#include +#include #include #include +#include +#include #include +#include namespace ripple { namespace Validators { -// Forward declare unit test so it can be a friend to LRUCache. -class Logic_test; +class ConnectionImp; -namespace detail { - -// The LRUCache class (ab)uses an aged_unordered_set so it can hold on -// to a limited number of values. When the container gets too full the -// LRUCache expires the oldest values. -// -// An aged_unordered_set gives us the functionality we want by keeping the -// chronological list. We don't care about the actual time of entry, only -// the time ordering. So we hook the aged_unordered_set up to a maunual_clock -// (which we never bother to increment). -// -// The implementation could potentially be changed to be time-based, rather -// than count-based, by hooking up a beast::basic_second_clock in place of the -// manual_clock and deleting a range of expired entries on insert. -// -template , - class KeyEqual = std::equal_to , - class Allocator = std::allocator > -class LRUCache -{ -private: - typedef beast::manual_clock Clock; - typedef beast::aged_unordered_set < - Key, std::chrono::steady_clock, Hash, KeyEqual, Allocator> ContainerType; - -public: - LRUCache () = delete; - - LRUCache (LRUCache const& lhs) = delete; - - explicit LRUCache ( - size_t item_max, - Hash hash = Hash(), - KeyEqual equal = KeyEqual(), - Allocator alloc = Allocator()) - : m_clock () - , m_cache (m_clock, hash, equal, alloc) - , m_item_max (item_max) - { - m_cache.reserve (m_item_max + 1); - } - - LRUCache& operator= (LRUCache const& lhs) = delete; - - // Add the entry. Remove the oldest entry if we went over our limit. - // Returns true on insertion (the entry was not already in the cache). - bool insert (Key const& key) - { - auto const insertRet (m_cache.insert (key)); - if (insertRet.second == false) - { - // key is re-referenced. Mark it as MRU. - m_cache.touch (insertRet.first); - } - else if (m_cache.size () > m_item_max) - { - // Added key and cache is too big. Erase oldest element. - m_cache.erase (m_cache.chronological.begin ()); - } - return insertRet.second; - } - - size_t size () - { - return m_cache.size(); - } - - Key const* oldest () - { - return m_cache.empty() ? nullptr : &(*m_cache.chronological.begin()); - } - -private: - Clock m_clock; - ContainerType m_cache; - const size_t m_item_max; -}; -} // namespace detail - -//------------------------------------------------------------------------------ - -// Encapsulates the logic for creating the chosen validators. -// This is a separate class to facilitate the unit tests. -// class Logic { public: - struct State - { - State () - : stopping (false) - { } - - /** True if we are stopping. */ - bool stopping; - - /** The source we are currently fetching. */ - beast::SharedPtr fetchSource; - }; + using clock_type = beast::abstract_clock; private: - typedef beast::SharedData SharedState; + struct LedgerMeta + { + std::uint32_t seq_no = 0; + std::unordered_set> keys; + }; - SharedState m_state; + class Policy + { + public: + /** Returns `true` if we should accept this as the last validated. */ + bool + acceptLedgerMeta (std::pair const& value) + { + return value.second.keys.size() >= 3; // quorum + } + }; - Store& m_store; - beast::Journal m_journal; + std::mutex mutex_; + //Store& store_; + beast::Journal journal_; - // A small integer assigned to each closed ledger - // - int m_ledgerID; + Policy policy_; + beast::aged_unordered_map > ledgers_; + std::pair latest_; // last fully validated + boost::container::flat_set connections_; + + //boost::container::flat_set< - // The chosen set of trusted validators (formerly the "UNL") - // - bool m_rebuildChosenList; - ChosenList::Ptr m_chosenList; - - // Holds the list of sources - // - typedef std::vector SourceTable; - SourceTable m_sources; - - // Holds the internal list of trusted validators - // - typedef hardened_hash_map ValidatorTable; - ValidatorTable m_validators; - - // Filters duplicate validations - // - typedef detail::LRUCache RecentValidations; - RecentValidations m_recentValidations; - - // Filters duplicate ledger hashes - // - typedef detail::LRUCache RecentLedgerHashes; - RecentLedgerHashes m_recentLedgerHashes; - - //-------------------------------------------------------------------------- public: + explicit + Logic (Store& store, beast::Journal journal); - explicit Logic (Store& store, beast::Journal journal = beast::Journal ()) - : m_store (store) - , m_journal (journal) - , m_ledgerID (0) - , m_rebuildChosenList (false) - , m_recentValidations (recentValidationsCacheSize) - , m_recentLedgerHashes (recentLedgersCacheSize) + beast::Journal const& + journal() const { - m_sources.reserve (16); + return journal_; } - /** Stop the logic. - This will cancel the current fetch and set the stopping flag - to `true` to prevent further fetches. - Thread safety: - Safe to call from any thread. - */ - void stop () - { - SharedState::Access state (m_state); - state->stopping = true; - if (state->fetchSource != nullptr) - state->fetchSource->cancel (); - } + void stop(); - //-------------------------------------------------------------------------- + void load(); - void load () - { - // load data from the database - } + void + add (ConnectionImp& c); - // Returns `true` if a Source with the same unique ID already exists - // - bool findSourceByID (std::string id) - { - for (SourceTable::const_iterator iter (m_sources.begin()); - iter != m_sources.end(); ++iter) - if (iter->source->uniqueID() == id) - return true; - return false; - } + void + remove (ConnectionImp& c); - // Add a one-time static source. - // Fetch is called right away, this call blocks. - // - void addStatic (beast::SharedPtr source) - { - if (findSourceByID (source->uniqueID())) - { - m_journal.error << - "Duplicate static " << *source; - return; - } + bool + isStale (STValidation const& v); - m_journal.trace << - "Addding static " << *source; + void + onTimer(); - Source::Results results; - source->fetch (results, m_journal); + void + onValidation (STValidation const& v); - if (results.success) - { - std::size_t const numAdded (merge (results.list, source)); - m_journal.trace << - "Added " << numAdded << - " trusted validators from " << *source; - } - else - { - // TODO: Report the error - } - } - - // Add a live source to the list of sources. - // - void add (beast::SharedPtr source) - { - if (findSourceByID (source->uniqueID())) - { - std::unique_ptr object (source); - m_journal.error << - "Duplicate " << *source; - return; - } - - m_journal.info << - "Adding " << *source; - - { - m_sources.resize (m_sources.size() + 1); - SourceDesc& desc (m_sources.back()); - desc.source = source; - m_store.insert (desc); - merge (desc.results.list, desc.source); - } - } - - // Add each entry in the list to the map, incrementing the - // reference count if it already exists, and updating fields. - // - std::size_t merge (std::vector const& list, Source* source) - { - std::size_t numAdded (0); - for (std::size_t i = 0; i < list.size (); ++i) - { - Source::Item const& item (list [i]); - std::pair results ( - m_validators.emplace (item.publicKey, Validator ())); - Validator& validatorInfo (results.first->second); - validatorInfo.addRef(); - if (results.second) - { - // This is a new one - ++numAdded; - dirtyChosen (); - } - } - - return numAdded; - } - - // Decrement the reference count of each item in the list - // in the map. - // - std::size_t remove (std::vector const& list, - Source* source) - { - std::size_t numRemoved (0); - for (std::size_t i = 0; i < list.size (); ++i) - { - Source::Item const& item (list [i]); - ValidatorTable::iterator iter (m_validators.find (item.publicKey)); - bassert (iter != m_validators.end ()); - Validator& validatorInfo (iter->second); - if (validatorInfo.release()) - { - // Last reference removed - ++numRemoved; - m_validators.erase (iter); - dirtyChosen (); - } - } - - return numRemoved; - } - - /** Return reference to m_sources for Mangager::PropertyStream. */ - SourceTable const& getSources () - { - return m_sources; - } - - /** Return reference to m_validators for Manager::PropertyStream. */ - ValidatorTable const& getValidators () - { - return m_validators; - } - - //-------------------------------------------------------------------------- - // - // Chosen - // - //-------------------------------------------------------------------------- - - /** Rebuild the Chosen List. */ - void buildChosen () - { - ChosenList::Ptr list (new ChosenList (m_validators.size ())); - - for (ValidatorTable::const_iterator iter = m_validators.begin (); - iter != m_validators.end (); ++iter) - { - ChosenList::Info item; - list->insert (iter->first, item); - } - - // This is thread safe - m_chosenList = list; - - m_journal.debug << - "Rebuilt chosen list with " << - std::to_string (m_chosenList->size()) << " entries"; - } - - /** Mark the Chosen List for a rebuild. */ - void dirtyChosen () - { - m_rebuildChosenList = true; - } - - /** Rebuild the Chosen List if necessary. */ - void checkChosen () - { - if (m_rebuildChosenList) - { - buildChosen (); - m_rebuildChosenList = false; - } - } - - /** Returns number of elements in the current Chosen list. */ - std::uint32_t getChosenSize() - { - return m_chosenList ? m_chosenList->size() : 0; - } - - //-------------------------------------------------------------------------- - // - // Fetching - // - //-------------------------------------------------------------------------- - - /** Perform a fetch on the source. */ - void fetch (SourceDesc& desc) - { - beast::SharedPtr const& source (desc.source); - Source::Results results; - - { - { - SharedState::Access state (m_state); - if (state->stopping) - return; - state->fetchSource = source; - } - - source->fetch (results, m_journal); - - { - SharedState::Access state (m_state); - if (state->stopping) - return; - state->fetchSource = nullptr; - } - } - - // Reset fetch timer for the source-> - desc.whenToFetch = beast::Time::getCurrentTime () + - beast::RelativeTime (secondsBetweenFetches); - - if (results.success) - { - // Count the number fetched - std::size_t const numFetched ( - results.list.size()); - - // Add the new source item to the map - std::size_t const numAdded ( - merge (results.list, source)); - - // Swap lists - std::swap (desc.results, results); - - // Remove the old source item from the map - std::size_t const numRemoved (remove (results.list, source)); - - // Report - if (numAdded > numRemoved) - { - m_journal.info << - "Fetched " << numFetched << - "(" << (numAdded - numRemoved) << " new) " << - " trusted validators from " << *source; - } - else if (numRemoved > numAdded) - { - m_journal.info << - "Fetched " << numFetched << - "(" << numRemoved - numAdded << " removed) " << - " trusted validators from " << *source; - } - else - { - m_journal.debug << - "Fetched " << numFetched << - " trusted validators from " << *source; - } - - // See if we need to rebuild - checkChosen (); - - // Reset failure status - desc.numberOfFailures = 0; - desc.status = SourceDesc::statusFetched; - - // Update the source's list in the store - m_store.update (desc, true); - } - else - { - m_journal.error << - "Failed to fetch " << *source; - - ++desc.numberOfFailures; - desc.status = SourceDesc::statusFailed; - // Record the failure in the Store - m_store.update (desc); - } - } - - /** Expire a source's list of validators. */ - void expire (SourceDesc& desc) - { - // Decrement reference count on each validator - remove (desc.results.list, desc.source); - - m_store.update (desc); - } - - /** Process up to one source that needs fetching. - @return The number of sources that were fetched. - */ - std::size_t fetch_one () - { - std::size_t n (0); - beast::Time const currentTime (beast::Time::getCurrentTime ()); - - for (SourceTable::iterator iter = m_sources.begin (); - (n == 0) && iter != m_sources.end (); ++iter) - { - SourceDesc& desc (*iter); - - // See if we should fetch - // - if (desc.whenToFetch <= currentTime) - { - fetch (desc); - ++n; - } - - // See if we need to expire - // - if (desc.expirationTime.isNotNull () && - desc.expirationTime <= currentTime) - { - expire (desc); - } - } - - return n; - } - - //-------------------------------------------------------------------------- - // - // Ripple interface - // - //-------------------------------------------------------------------------- - - // Called when we receive a signed validation - // - void receiveValidation (ReceivedValidation const& rv) - { - // Accept validation from the trusted list - ValidatorTable::iterator iter (m_validators.find (rv.publicKey)); - if (iter != m_validators.end ()) - { - // Filter duplicates (defensive programming) - if (! m_recentValidations.insert (rv)) - return; - - iter->second.on_validation (rv.ledgerHash); - - m_journal.trace << - "New trusted validation for " << rv.ledgerHash << - " from " << rv.publicKey; - } - else - { - m_journal.trace << - "Untrusted validation for " << rv.ledgerHash << - " from " << rv.publicKey; - } - } - - // Called when a ledger is closed - // - void ledgerClosed (RippleLedgerHash const& ledgerHash) - { - // Filter duplicates (defensive programming) - if (! m_recentLedgerHashes.insert (ledgerHash)) - return; - - ++m_ledgerID; - - m_journal.trace << - "Closed ledger " << m_ledgerID; - - for (ValidatorTable::iterator iter (m_validators.begin()); - iter != m_validators.end(); ++iter) - iter->second.on_ledger (ledgerHash); - } - - // Returns `true` if the public key hash is contained in the Chosen List. - // - bool isTrustedPublicKeyHash (RipplePublicKeyHash const& publicKeyHash) - { - return m_chosenList->containsPublicKeyHash (publicKeyHash); - } - - //-------------------------------------------------------------------------- + void + onLedgerClosed (LedgerIndex index, + LedgerHash const& hash, LedgerHash const& parent); }; } diff --git a/src/ripple/validators/impl/Manager.cpp b/src/ripple/validators/impl/Manager.cpp index aa9f4841a..47825f483 100644 --- a/src/ripple/validators/impl/Manager.cpp +++ b/src/ripple/validators/impl/Manager.cpp @@ -18,15 +18,16 @@ //============================================================================== #include +#include +#include #include -#include -#include -#include #include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include // /** ChosenValidators (formerly known as UNL) @@ -133,63 +134,99 @@ */ +#include +#include + namespace ripple { + +/** Executor which dispatches to JobQueue threads at a given JobType. */ +class job_executor +{ +private: + struct impl + { + impl (JobQueue& ex_, JobType type_, std::string const& name_) + : ex(ex_), type(type_), name(name_) + { + } + + JobQueue& ex; + JobType type; + std::string name; + }; + + std::shared_ptr impl_; + +public: + job_executor (JobType type, std::string const& name, + JobQueue& ex) + : impl_(std::make_shared(ex, type, name)) + { + } + + template + void + post (Handler&& handler) + { + impl_->ex.addJob(impl_->type, impl_->name, + std::forward(handler)); + } + + template + void + dispatch (Handler&& handler) + { + impl_->ex.addJob(impl_->type, impl_->name, + std::forward(handler)); + } + + template + void + defer (Handler&& handler) + { + impl_->ex.addJob(impl_->type, impl_->name, + std::forward(handler)); + } +}; + +//------------------------------------------------------------------------------ + namespace Validators { +// template class ManagerImp : public Manager , public beast::Stoppable - , public beast::Thread - , public beast::DeadlineTimer::Listener - , public beast::LeakChecked { public: - beast::Journal m_journal; - beast::File m_databaseFile; - StoreSqdb m_store; - Logic m_logic; - beast::DeadlineTimer m_checkTimer; - beast::ServiceQueue m_queue; + boost::asio::io_service& io_service_; + boost::asio::io_service::strand strand_; + beast::asio::waitable_executor exec_; + boost::asio::basic_waitable_timer< + std::chrono::steady_clock> timer_; + beast::Journal journal_; + beast::File dbFile_; + StoreSqdb store_; + Logic logic_; - typedef beast::ScopedWrapperContext < - beast::RecursiveMutex, beast::RecursiveMutex::ScopedLockType> Context; - - Context m_context; - - // True if we should call check on idle. - // This gets set to false once we make it through the whole list. - // - bool m_checkSources; - - ManagerImp ( - Stoppable& parent, - beast::File const& pathToDbFileOrDirectory, - beast::Journal journal) + ManagerImp (Stoppable& parent, boost::asio::io_service& io_service, + beast::File const& pathToDbFileOrDirectory, beast::Journal journal) : Stoppable ("Validators::Manager", parent) - , Thread ("Validators") - , m_journal (journal) - , m_databaseFile (pathToDbFileOrDirectory) - , m_store (m_journal) - , m_logic (m_store, m_journal) - , m_checkTimer (this) - , m_checkSources (false) + , io_service_(io_service) + , strand_(io_service_) + , timer_(io_service_) + , journal_ (journal) + , dbFile_ (pathToDbFileOrDirectory) + , store_ (journal_) + , logic_ (store_, journal_) { - m_journal.trace << - "Validators constructed"; - m_journal.debug << - "Validators constructed (debug)"; - m_journal.info << - "Validators constructed (info)"; - - if (m_databaseFile.isDirectory ()) - m_databaseFile = m_databaseFile.getChildFile("validators.sqlite"); - + if (dbFile_.isDirectory ()) + dbFile_ = dbFile_.getChildFile("validators.sqlite"); } - ~ManagerImp () + ~ManagerImp() { - stopThread (); } //-------------------------------------------------------------------------- @@ -198,53 +235,18 @@ public: // //-------------------------------------------------------------------------- - void addStrings (std::string const& name, std::vector const& strings) + std::unique_ptr + newConnection (int id) override { - if (strings.empty ()) - { - m_journal.debug << "Static source '" << name << "' is empty."; - return; - } - - addStaticSource (SourceStrings::New (name, strings)); + return std::make_unique( + id, logic_, get_seconds_clock()); } - void addFile (beast::File const& file) + void + onLedgerClosed (LedgerIndex index, + LedgerHash const& hash, LedgerHash const& parent) override { - addStaticSource (SourceFile::New (file)); - } - - void addStaticSource (Validators::Source* source) - { - m_queue.dispatch (m_context.wrap (std::bind ( - &Logic::addStatic, &m_logic, source))); - } - - void addURL (beast::URL const& url) - { - addSource (SourceURL::New (url)); - } - - void addSource (Validators::Source* source) - { - m_queue.dispatch (m_context.wrap (std::bind ( - &Logic::add, &m_logic, source))); - } - - //-------------------------------------------------------------------------- - - void on_receive_validation (ReceivedValidation const& rv) - { - if (! isStopping()) - m_queue.dispatch (m_context.wrap (std::bind ( - &Logic::receiveValidation, &m_logic, rv))); - } - - void on_ledger_closed (RippleLedgerHash const& ledgerHash) - { - if (! isStopping()) - m_queue.dispatch (m_context.wrap (std::bind ( - &Logic::ledgerClosed, &m_logic, ledgerHash))); + logic_.onLedgerClosed (index, hash, parent); } //-------------------------------------------------------------------------- @@ -253,25 +255,23 @@ public: // //-------------------------------------------------------------------------- - void onPrepare () + void onPrepare() + { + init(); + } + + void onStart() { } - void onStart () + void onStop() { - // Do this late so the sources have a chance to be added. - m_queue.dispatch (m_context.wrap (std::bind ( - &ManagerImp::setCheckSources, this))); + boost::system::error_code ec; + timer_.cancel(ec); - startThread(); - } + logic_.stop(); - void onStop () - { - m_logic.stop (); - - m_queue.dispatch (m_context.wrap (std::bind ( - &Thread::signalThreadShouldExit, this))); + exec_.async_wait([this]() { stopped(); }); } //-------------------------------------------------------------------------- @@ -282,28 +282,6 @@ public: void onWrite (beast::PropertyStream::Map& map) { - Context::Scope scope (m_context); - - map ["trusted"] = m_logic.getChosenSize(); - { - beast::PropertyStream::Set items ("sources", map); - for (auto const& entry : m_logic.getSources()) - { - items.add (entry.source->to_string()); - } - } - - { - beast::PropertyStream::Set items ("validators", map); - for (auto const& entry : m_logic.getValidators()) - { - RipplePublicKey const& publicKey (entry.first); - Validator const& validator (entry.second); - beast::PropertyStream::Map item (items); - item["public_key"] = publicKey.to_string(); - validator.count().onWrite (item); - } - } } //-------------------------------------------------------------------------- @@ -312,64 +290,33 @@ public: // //-------------------------------------------------------------------------- - void init () + void init() { - beast::Error error (m_store.open (m_databaseFile)); + beast::Error error (store_.open (dbFile_)); if (! error) { - m_logic.load (); + logic_.load (); } } - void onDeadlineTimer (beast::DeadlineTimer& timer) + void + onTimer (boost::system::error_code ec) { - if (timer == m_checkTimer) + if (ec) { - m_journal.trace << "Check timer expired"; - m_queue.dispatch (m_context.wrap (std::bind ( - &ManagerImp::setCheckSources, this))); - } - } - - void setCheckSources () - { - m_journal.trace << "Checking sources"; - m_checkSources = true; - } - - void checkSources () - { - if (m_checkSources) - { - if (m_logic.fetch_one () == 0) - { - m_journal.trace << "All sources checked"; - - // Made it through the list without interruption! - // Clear the flag and set the deadline timer again. - // - m_checkSources = false; - - m_journal.trace << "Next check timer expires in " << - beast::RelativeTime::seconds (checkEverySeconds); - - m_checkTimer.setExpiration (checkEverySeconds); - } - } - } - - void run () - { - init (); - - while (! this->threadShouldExit()) - { - checkSources (); - m_queue.run_one(); + if (ec != boost::asio::error::operation_aborted) + journal_.error << + "onTimer: " << ec.message(); + return; } - stopped(); + logic_.onTimer(); + + timer_.expires_from_now(std::chrono::seconds(1), ec); + timer_.async_wait(strand_.wrap(exec_.wrap( + std::bind(&ManagerImp::onTimer, this, + beast::asio::placeholders::error)))); } }; @@ -380,12 +327,14 @@ Manager::Manager () { } -Validators::Manager* Validators::Manager::New ( - beast::Stoppable& parent, - beast::File const& pathToDbFileOrDirectory, - beast::Journal journal) +std::unique_ptr +make_Manager(beast::Stoppable& parent, + boost::asio::io_service& io_service, + beast::File const& pathToDbFileOrDirectory, + beast::Journal journal) { - return new Validators::ManagerImp (parent, pathToDbFileOrDirectory, journal); + return std::make_unique (parent, + io_service, pathToDbFileOrDirectory, journal); } } diff --git a/src/ripple/validators/impl/SourceDesc.h b/src/ripple/validators/impl/SourceDesc.h deleted file mode 100644 index 67b2c723d..000000000 --- a/src/ripple/validators/impl/SourceDesc.h +++ /dev/null @@ -1,71 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_VALIDATORS_SOURCEDESC_H_INCLUDED -#define RIPPLE_VALIDATORS_SOURCEDESC_H_INCLUDED - -#include - -namespace ripple { -namespace Validators { - -/** Additional state information associated with a Source. */ -struct SourceDesc -{ - enum Status - { - statusNone, - statusFetched, - statusFailed - }; - - beast::SharedPtr source; - Status status; - beast::Time whenToFetch; - int numberOfFailures; - - // The result of the last fetch - Source::Results results; - - //------------------------------------------------------------------ - - /** The time of the last successful fetch. */ - beast::Time lastFetchTime; - - /** When to expire this source's list of cached results (if any) */ - beast::Time expirationTime; - - //------------------------------------------------------------------ - - SourceDesc () noexcept - : status (statusNone) - , whenToFetch (beast::Time::getCurrentTime ()) - , numberOfFailures (0) - { - } - - ~SourceDesc () - { - } -}; - -} -} - -#endif diff --git a/src/ripple/validators/impl/SourceFile.cpp b/src/ripple/validators/impl/SourceFile.cpp deleted file mode 100644 index 402e6c0b1..000000000 --- a/src/ripple/validators/impl/SourceFile.cpp +++ /dev/null @@ -1,94 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include - -#include -#include - -namespace ripple { -namespace Validators { - -class SourceFileImp - : public SourceFile - , public beast::LeakChecked -{ -public: - SourceFileImp (beast::File const& file) - : m_file (file) - { - } - - ~SourceFileImp () - { - } - - std::string to_string () const - { - return "File: '" + m_file.getFullPathName().toStdString() + "'"; - } - - std::string uniqueID () const - { - return "File," + m_file.getFullPathName ().toStdString(); - } - - std::string createParam () - { - return m_file.getFullPathName ().toStdString(); - } - - void fetch (Results& results, beast::Journal journal) - { - // 8MB is a somewhat arbitrary maximum file size, but it should be - // enough to cover all cases in the foreseeable future. - - std::int64_t const maxFileSize = 8 * 1024 * 1024; - std::int64_t const fileSize = m_file.getSize (); - - if (fileSize != 0 && (fileSize < maxFileSize)) - { - std::ifstream datafile (m_file.getFullPathName().toStdString ()); - std::string line; - - if (datafile.is_open ()) - { - while (std::getline(datafile, line)) - Utilities::parseResultLine (results, line, journal); - } - } - else - { - // file doesn't exist - } - } - -private: - beast::File m_file; -}; - -//------------------------------------------------------------------------------ - -SourceFile* SourceFile::New (beast::File const& file) -{ - return new SourceFileImp (file); -} - -} -} diff --git a/src/ripple/validators/impl/SourceFile.h b/src/ripple/validators/impl/SourceFile.h deleted file mode 100644 index cf7863cd5..000000000 --- a/src/ripple/validators/impl/SourceFile.h +++ /dev/null @@ -1,38 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_VALIDATORS_SOURCEFILE_H_INCLUDED -#define RIPPLE_VALIDATORS_SOURCEFILE_H_INCLUDED - -namespace ripple { -namespace Validators { - -/** Provides validators from a text file. - Typically this will come from a local configuration file. -*/ -class SourceFile : public Source -{ -public: - static SourceFile* New (beast::File const& path); -}; - -} -} - -#endif diff --git a/src/ripple/validators/impl/SourceStrings.cpp b/src/ripple/validators/impl/SourceStrings.cpp deleted file mode 100644 index c3dde0a7b..000000000 --- a/src/ripple/validators/impl/SourceStrings.cpp +++ /dev/null @@ -1,80 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -namespace ripple { -namespace Validators { - -class SourceStringsImp - : public SourceStrings - , public beast::LeakChecked -{ -public: - SourceStringsImp (std::string const& name, std::vector const& strings) - : m_name (name) - , m_strings (strings) - { - } - - ~SourceStringsImp () - { - } - - std::string to_string () const - { - return m_name; - } - - std::string uniqueID () const - { - // VFALCO TODO This can't be right...? - return std::string{}; - } - - std::string createParam () - { - return std::string{}; - } - - void fetch (Results& results, beast::Journal journal) - { - results.list.reserve (m_strings.size ()); - - for (int i = 0; i < m_strings.size (); ++i) - Utilities::parseResultLine (results, m_strings [i]); - - results.success = results.list.size () > 0; - results.expirationTime = beast::Time::getCurrentTime () + - beast::RelativeTime::hours (24); - } - -private: - std::string m_name; - std::vector m_strings; -}; - -//------------------------------------------------------------------------------ - -SourceStrings* SourceStrings::New ( - std::string const& name, std::vector const& strings) -{ - return new SourceStringsImp (name, strings); -} - -} -} diff --git a/src/ripple/validators/impl/SourceStrings.h b/src/ripple/validators/impl/SourceStrings.h deleted file mode 100644 index 66058c397..000000000 --- a/src/ripple/validators/impl/SourceStrings.h +++ /dev/null @@ -1,39 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_VALIDATORS_SOURCESTRINGS_H_INCLUDED -#define RIPPLE_VALIDATORS_SOURCESTRINGS_H_INCLUDED - -namespace ripple { -namespace Validators { - -/** Provides validators from a set of Validator strings. - Typically this will come from a local configuration file. -*/ -class SourceStrings : public Source -{ -public: - static SourceStrings* New ( - std::string const& name, std::vector const& strings); -}; - -} -} - -#endif diff --git a/src/ripple/validators/impl/SourceURL.cpp b/src/ripple/validators/impl/SourceURL.cpp deleted file mode 100644 index 04ec1ce36..000000000 --- a/src/ripple/validators/impl/SourceURL.cpp +++ /dev/null @@ -1,105 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include - -namespace ripple { -namespace Validators { - -class SourceURLImp - : public SourceURL - , public beast::LeakChecked -{ -private: - beast::URL m_url; - - // VFALCO This is turned off because the HTTPClient - // implementation is now obsolete. A new HTTP client - // that uses the latest best practices (asio coroutines, - // beast::http::message and beast::http::parser) should - // be used. -#if 0 - std::unique_ptr m_client; -#endif - -public: - explicit SourceURLImp (beast::URL const& url) - : m_url (url) - //, m_client (beast::asio::HTTPClientBase::New ()) - { - } - - ~SourceURLImp () - { - } - - std::string to_string () const - { - using std::to_string; - return "URL: '" + to_string (m_url) + "'"; - } - - std::string uniqueID () const - { - using std::to_string; - return "URL," + to_string (m_url); - } - - std::string createParam () - { - using std::to_string; - return to_string (m_url); - } - - void cancel () - { - //m_client->cancel (); - } - - void fetch (Results& results, beast::Journal journal) - { -#if 0 - auto httpResult (m_client->get (m_url)); - - if (httpResult.first == 0) - { - Utilities::ParseResultLine lineFunction (results, journal); - std::string const s (httpResult.second->body().to_string()); - Utilities::processLines (s.begin(), s.end(), lineFunction); - } - else - { - journal.error << - "HTTP GET to " << m_url << - " failed: '" << httpResult.first.message () << "'"; - } -#endif - } -}; - -//------------------------------------------------------------------------------ - -SourceURL* SourceURL::New ( - beast::URL const& url) -{ - return new SourceURLImp (url); -} - -} -} diff --git a/src/ripple/validators/impl/Store.h b/src/ripple/validators/impl/Store.h index f2f2fa5bb..abc300387 100644 --- a/src/ripple/validators/impl/Store.h +++ b/src/ripple/validators/impl/Store.h @@ -20,8 +20,6 @@ #ifndef RIPPLE_VALIDATORS_STORE_H_INCLUDED #define RIPPLE_VALIDATORS_STORE_H_INCLUDED -#include - namespace ripple { namespace Validators { @@ -29,19 +27,7 @@ namespace Validators { class Store { public: - virtual ~Store () { } - - /** Insert a new SourceDesc to the Store. - The caller's SourceDesc will have any available persistent - information filled in from the Store. - */ - virtual void insert (SourceDesc& desc) = 0; - - /** Update the SourceDesc fixed fields. */ - virtual void update (SourceDesc& desc, bool updateFetchResults = false) = 0; - -protected: - Store () { } + virtual ~Store() = default; }; } diff --git a/src/ripple/validators/impl/StoreSqdb.cpp b/src/ripple/validators/impl/StoreSqdb.cpp index a71d7e47b..12e39b739 100644 --- a/src/ripple/validators/impl/StoreSqdb.cpp +++ b/src/ripple/validators/impl/StoreSqdb.cpp @@ -17,6 +17,7 @@ */ //============================================================================== +#include #include #include #include @@ -33,19 +34,14 @@ StoreSqdb::~StoreSqdb () { } -beast::Error StoreSqdb::open (beast::File const& file) +beast::Error +StoreSqdb::open (beast::File const& file) { beast::Error error (m_session.open (file.getFullPathName ())); m_journal.info << "Opening " << file.getFullPathName(); - if (!error) - error = init (); - - if (!error) - error = update (); - if (error) m_journal.error << "Failed opening database: " << error.what(); @@ -53,532 +49,5 @@ beast::Error StoreSqdb::open (beast::File const& file) return error; } -//-------------------------------------------------------------------------- - -void StoreSqdb::insert (SourceDesc& desc) -{ - beast::sqdb::transaction tr (m_session); - - bool const found (select (desc)); - - if (found) - { - selectList (desc); - } - else - { - beast::Error error; - - auto const sourceID (desc.source->uniqueID()); - auto const createParam (desc.source->createParam()); - auto const lastFetchTime (timeToString (desc.lastFetchTime)); - auto const expirationTime (timeToString (desc.expirationTime)); - - beast::sqdb::statement st = (m_session.prepare << - "INSERT INTO Validators_Source ( " - " sourceID, " - " createParam, " - " lastFetchTime, " - " expirationTime " - ") VALUES ( " - " ?, ?, ?, ? " - "); " - ,beast::sqdb::use (sourceID) - ,beast::sqdb::use (createParam) - ,beast::sqdb::use (lastFetchTime) - ,beast::sqdb::use (expirationTime) - ); - - st.execute_and_fetch (error); - - if (! error) - { - error = tr.commit (); - } - - if (error) - { - tr.rollback (); - report (error, __FILE__, __LINE__); - } - } -} - -//-------------------------------------------------------------------------- -std::string StoreSqdb::itos (int i, std::size_t width) -{ - auto s = std::to_string (i); - - if (s.length () < width) - s = std::string (width - s.length(), '0').append (s); - - return s; -} - -beast::Time StoreSqdb::stringToTime (std::string const& s) -{ - static boost::regex const date_pattern ( - "^" // the beginning of the string - "(19[789][0-9]|[2-9][0-9][0-9][0-9])-" // 1970-9999 followed by - - "(0[0-9]|1[01])-" // 0-11 followed by - - "(0[1-9]|[12][0-9]|3[01]) " // 1-31 followed by space - "([01][0-9]|2[0-3]):" // 0-23 followed by : - "([0-5][0-9]):" // 0-59 followed by : - "([0-5][0-9])" // 0-59 - "$", - boost::regex_constants::optimize); - - boost::smatch match; - - if (boost::regex_match (s, match, date_pattern)) - { - int const year = beast::lexicalCast (std::string (match[1]), -1); - int const mon = beast::lexicalCast (std::string (match[2]), -1); - int const day = beast::lexicalCast (std::string (match[3]), -1); - int const hour = beast::lexicalCast (std::string (match[4]), -1); - int const min = beast::lexicalCast (std::string (match[5]), -1); - int const sec = beast::lexicalCast (std::string (match[6]), -1); - - if (year != -1 && - mon != -1 && - day != -1 && - hour != -1 && - min != -1 && - sec != -1) - { - // local time - return beast::Time (year, mon, day, hour, min, sec, 0, true); - } - } - - return beast::Time (0); -} - -std::string StoreSqdb::timeToString (beast::Time const& t) -{ - std::string ret; - - if (t.isNotNull ()) - { - ret = itos (t.getYear(), 4) + "-" + - itos (t.getMonth(), 2) + "-" + - itos (t.getDayOfMonth (), 2) + " " + - itos (t.getHours () , 2) + ":" + - itos (t.getMinutes (), 2) + ":" + - itos (t.getSeconds(), 2); - } - - return ret; -} - -//-------------------------------------------------------------------------- -void StoreSqdb::update (SourceDesc& desc, bool updateFetchResults) -{ - beast::Error error; - - std::string const sourceID (desc.source->uniqueID()); - std::string const lastFetchTime (timeToString (desc.lastFetchTime)); - std::string const expirationTime (timeToString (desc.expirationTime)); - - beast::sqdb::transaction tr (m_session); - - m_session.once (error) << - "UPDATE Validators_Source SET " - " lastFetchTime = ?, " - " expirationTime = ? " - "WHERE " - " sourceID = ? " - ,beast::sqdb::use (lastFetchTime) - ,beast::sqdb::use (expirationTime) - ,beast::sqdb::use (sourceID) - ; - - if (! error && updateFetchResults) - { - // Delete the previous data set - m_session.once (error) << - "DELETE FROM Validators_SourceItem WHERE " - " sourceID = ?; " - ,beast::sqdb::use (sourceID) - ; - - // Insert the new data set - if (! error) - { - std::string publicKeyString; - std::string label; - - beast::sqdb::statement st = (m_session.prepare << - "INSERT INTO Validators_SourceItem ( " - " sourceID, " - " publicKey, " - " label " - ") VALUES ( " - " ?, ?, ? " - ");" - ,beast::sqdb::use (sourceID) - ,beast::sqdb::use (publicKeyString) - ,beast::sqdb::use (label) - ); - - std::vector & list (desc.results.list); - for (std::size_t i = 0; ! error && i < list.size(); ++i) - { - Source::Item& item (list [i]); - publicKeyString = item.publicKey.to_string (); - label = list[i].label; - st.execute_and_fetch (error); - } - } - } - - if (! error) - { - error = tr.commit(); - } - - if (error) - { - tr.rollback (); - report (error, __FILE__, __LINE__); - } -} - -//-------------------------------------------------------------------------- - -void StoreSqdb::report (beast::Error const& error, char const* fileName, int lineNumber) -{ - if (error) - { - m_journal.error << - "Failure: '"<< error.getReasonText() << "' " << - " at " << beast::Debug::getSourceLocation (fileName, lineNumber); - } -} - -//-------------------------------------------------------------------------- - -/** Reads the fixed information into the SourceDesc if it exists. - Returns `true` if the record was found. -*/ -bool StoreSqdb::select (SourceDesc& desc) -{ - bool found (false); - - beast::Error error; - - std::string const sourceID (desc.source->uniqueID()); - std::string lastFetchTime; - std::string expirationTime; - - beast::sqdb::statement st = (m_session.prepare << - "SELECT " - " lastFetchTime, " - " expirationTime " - "FROM Validators_Source WHERE " - " sourceID = ? " - ,beast::sqdb::into (lastFetchTime) - ,beast::sqdb::into (expirationTime) - ,beast::sqdb::use (sourceID) - ); - - if (st.execute_and_fetch (error)) - { - m_journal.debug << - "Found record for " << *desc.source; - - found = true; - desc.lastFetchTime = stringToTime (lastFetchTime); - desc.expirationTime = stringToTime (expirationTime); - } - else if (! error) - { - m_journal.info << - "No previous record for " << *desc.source; - } - - if (error) - { - report (error, __FILE__, __LINE__); - } - - return found; -} - -//-------------------------------------------------------------------------- - -/** Reads the variable information into the SourceDesc. - This should only be called when the sourceID was already found. -*/ -void StoreSqdb::selectList (SourceDesc& desc) -{ - beast::Error error; - - std::string const sourceID (desc.source->uniqueID()); - - // Get the count - std::size_t count; - if (! error) - { - m_session.once (error) << - "SELECT " - " COUNT(*) " - "FROM Validators_SourceItem WHERE " - " sourceID = ? " - ,beast::sqdb::into (count) - ,beast::sqdb::use (sourceID) - ; - } - - if (error) - { - report (error, __FILE__, __LINE__); - return; - } - - // Precondition: the list must be empty. - bassert (desc.results.list.size() == 0); - - // Pre-allocate some storage - desc.results.list.reserve (count); - - // Prepare the select - { - std::string publicKeyString; - std::string label; - beast::sqdb::statement st = (m_session.prepare << - "SELECT " - " publicKey, " - " label " - "FROM Validators_SourceItem WHERE " - " sourceID = ? " - ,beast::sqdb::into (publicKeyString) - ,beast::sqdb::into (label) - ,beast::sqdb::use (sourceID) - ); - - // Add all the records to the list - if (st.execute_and_fetch (error)) - { - do - { - Source::Item info; - std::pair result ( - RipplePublicKey::from_string (publicKeyString)); - if (result.second) - { - bassert (result.first.to_string() == publicKeyString); - info.publicKey = result.first; - info.label = label; - desc.results.list.push_back (info); - } - else - { - m_journal.error << - "Invalid public key '" << publicKeyString << - "' found in database"; - } - } - while (st.fetch (error)); - - if (! error) - { - m_journal.info << - "Loaded " << desc.results.list.size() << - " trusted validators for " << *desc.source; - } - } - } - - if (error) - { - report (error, __FILE__, __LINE__); - } -} - -//-------------------------------------------------------------------------- - -// Update the database for the current schema -beast::Error StoreSqdb::update () -{ - beast::Error error; - - beast::sqdb::transaction tr (m_session); - - // Get the version from the database - int version (0); - if (! error) - { - m_session.once (error) << - "SELECT " - " version " - "FROM SchemaVersion WHERE " - " name = 'Validators' " - ,beast::sqdb::into (version) - ; - - if (! m_session.got_data ()) - { - // pre-dates the "SchemaVersion" table - version = 0; - } - } - - if (! error && version != currentSchemaVersion) - { - m_journal.info << - "Update database to version " << currentSchemaVersion << - " from version " << version; - } - - // Update database based on version - if (! error && version < 2) - { - if (! error) - m_session.once (error) << - "DROP TABLE IF EXISTS ValidatorsSource"; - - if (! error) - m_session.once (error) << - "DROP TABLE IF EXISTS ValidatorsSourceInfo"; - - if (! error) - m_session.once (error) << - "DROP INDEX IF EXISTS ValidatorsSourceInfoIndex"; - } - - // Update the version to the current version - if (! error) - { - int const version (currentSchemaVersion); - - m_session.once (error) << - "INSERT OR REPLACE INTO SchemaVersion ( " - " name, " - " version " - ") VALUES ( " - " 'Validators', ? " - "); " - ,beast::sqdb::use (version) - ; - } - - if (! error) - { - error = tr.commit(); - } - - if (error) - { - tr.rollback (); - report (error, __FILE__, __LINE__); - } - - return error; -} - -//-------------------------------------------------------------------------- - -beast::Error StoreSqdb::init () -{ - beast::Error error; - - beast::sqdb::transaction tr (m_session); - - if (! error) - { - m_session.once (error) << - "PRAGMA encoding=\"UTF-8\""; - } - - if (! error) - { - // This table maps component names like "Validators" to their - // corresponding schema version number. This method allows us - // to keep all logic data in one database, or each in its own - // database, or in any grouping of databases, while still being - // able to let an individual component know what version of its - // schema it is opening. - // - m_session.once (error) << - "CREATE TABLE IF NOT EXISTS SchemaVersion ( " - " name TEXT PRIMARY KEY, " - " version INTEGER" - ");" - ; - } - - if (! error) - { - m_session.once (error) << - "CREATE TABLE IF NOT EXISTS Validators_Source ( " - " id INTEGER PRIMARY KEY AUTOINCREMENT, " - " sourceID TEXT UNIQUE, " - " createParam TEXT NOT NULL, " - " lastFetchTime TEXT NOT NULL, " - " expirationTime TEXT NOT NULL " - ");" - ; - } - - if (! error) - { - m_session.once (error) << - "CREATE TABLE IF NOT EXISTS Validators_SourceItem ( " - " id INTEGER PRIMARY KEY AUTOINCREMENT, " - " sourceID TEXT NOT NULL, " - " publicKey TEXT NOT NULL, " - " label TEXT NOT NULL " - ");" - ; - } - - if (! error) - { - m_session.once (error) << - "CREATE INDEX IF NOT EXISTS " - " Validators_SourceItem_Index ON Validators_SourceItem " - " ( " - " sourceID " - " ); " - ; - } - -#if 0 - if (! error) - { - m_session.once (error) << - "CREATE TABLE IF NOT EXISTS ValidatorsValidator ( " - " id INTEGER PRIMARY KEY AUTOINCREMENT, " - " publicKey TEXT UNIQUE " - ");" - ; - } - - if (! error) - { - m_session.once (error) << - "CREATE TABLE IF NOT EXISTS ValidatorsValidatorStats ( " - " id INTEGER PRIMARY KEY AUTOINCREMENT, " - " publicKey TEXT UNIQUE " - ");" - ; - } -#endif - - if (! error) - { - error = tr.commit(); - } - - if (error) - { - tr.rollback (); - report (error, __FILE__, __LINE__); - } - - return error; -} - } } diff --git a/src/ripple/validators/impl/StoreSqdb.h b/src/ripple/validators/impl/StoreSqdb.h index 62f769044..863438b1a 100644 --- a/src/ripple/validators/impl/StoreSqdb.h +++ b/src/ripple/validators/impl/StoreSqdb.h @@ -21,53 +21,35 @@ #define RIPPLE_VALIDATORS_STORESQDB_H_INCLUDED #include +#include #include #include +#include namespace ripple { namespace Validators { /** Database persistence for Validators using SQLite */ -class StoreSqdb - : public Store - , public beast::LeakChecked +class StoreSqdb : public Store { +private: + beast::Journal m_journal; + beast::sqdb::session m_session; + public: enum { // This affects the format of the data! - currentSchemaVersion = 2 + currentSchemaVersion = 1 }; - explicit StoreSqdb (beast::Journal journal = beast::Journal()); + explicit + StoreSqdb (beast::Journal journal); - ~StoreSqdb (); + ~StoreSqdb(); - beast::Error open (beast::File const& file); - - void insert (SourceDesc& desc); - - void update (SourceDesc& desc, bool updateFetchResults); - - void remove (RipplePublicKey const& publicKey); - -private: - void report (beast::Error const& error, char const* fileName, int lineNumber); - - bool select (SourceDesc& desc); - void selectList (SourceDesc& desc); - - beast::Error update (); - beast::Error init (); - - beast::Journal m_journal; - beast::sqdb::session m_session; - - - // DEPRECATED - static std::string itos (int i, std::size_t fieldSize = 0); - static std::string timeToString (beast::Time const& t); - static beast::Time stringToTime (std::string const& s); + beast::Error + open (beast::File const& file); }; } diff --git a/src/ripple/validators/impl/Tests.cpp b/src/ripple/validators/impl/Tests.cpp index 23c265201..729949e67 100644 --- a/src/ripple/validators/impl/Tests.cpp +++ b/src/ripple/validators/impl/Tests.cpp @@ -17,7 +17,7 @@ */ //============================================================================== -#include +#include #include namespace ripple { @@ -26,258 +26,14 @@ namespace Validators { class Logic_test : public beast::unit_test::suite { public: - enum - { - numberOfTestValidators = 1000, - numberofTestSources = 50 - }; - - //-------------------------------------------------------------------------- - - struct TestSource : Source - { - TestSource (std::string const& name, std::uint32_t start, std::uint32_t end) - : m_name (name) - , m_start (start) - , m_end (end) - { - } - - std::string to_string () const - { - return uniqueID(); - } - - std::string uniqueID () const - { - return "Test," + m_name + "," + - std::to_string (m_start) + "," + - std::to_string (m_end); - } - - std::string createParam () - { - return std::string{}; - } - - void fetch (Results& results, beast::Journal) - { - results.success = true; - results.message = std::string{}; - results.list.reserve (numberOfTestValidators); - - for (std::uint32_t i = m_start ; i < m_end; ++i) - { - Item item; - item.publicKey = RipplePublicKey::createFromInteger (i); - item.label = std::to_string (i); - results.list.push_back (item); - } - } - - std::string m_name; - std::size_t m_start; - std::size_t m_end; - }; - - //-------------------------------------------------------------------------- - - class TestStore : public Store - { - public: - TestStore () - { - } - - ~TestStore () - { - } - - void insertSourceDesc (SourceDesc& desc) - { - } - - void updateSourceDesc (SourceDesc& desc) - { - } - - void updateSourceDescInfo (SourceDesc& desc) - { - } - }; - - //-------------------------------------------------------------------------- - - void addSources (Logic& logic) - { - beast::Random r; - for (int i = 1; i <= numberofTestSources; ++i) - { - std::string const name (std::to_string (i)); - std::uint32_t const start = r.nextInt (numberOfTestValidators); - std::uint32_t const end = start + r.nextInt (numberOfTestValidators); - logic.add (new TestSource (name, start, end)); - } - } - - void testLRUCache () - { - detail::LRUCache testCache {3}; - expect (testCache.size () == 0, "Wrong initial size"); - - struct TestValues - { - char const* const value; - bool const insertResult; - }; - { - std::array const v1 { - {{"A", true}, {"B", true}, {"C", true}}}; - for (auto const& v : v1) - { - expect (testCache.insert (v.value) == v.insertResult, - "Failed first insert tests"); - } - expect (testCache.size() == 3, "Unexpected intermediate size"); - expect (*testCache.oldest() == "A", "Unexpected oldest member"); - } - { - std::array const v2 { - {{"A", false}, {"D", true}, {"C", false}}}; - for (auto const& v : v2) - { - expect (testCache.insert (v.value) == v.insertResult, - "Failed second insert tests"); - } - expect (testCache.size() == 3, "Unexpected final size"); - expect (*testCache.oldest() == "A", - "Unexpected oldest member"); - } - } - - void testValidator () - { - int receivedCount = 0; - int expectedCount = 0; - int closedCount = 0; - - // Lambda as local function - auto updateCounts = [&](bool received, bool validated) - { - bool const sent = received || validated; - - receivedCount += sent && !validated ? 1 : 0; - expectedCount += sent && !received ? 1 : 0; - closedCount += validated && received ? 1 : 0; - }; - - auto checkCounts = [&] (Count const& count) - { -// std::cout << "Received actual: " << count.received << " expected: " << receivedCount << std::endl; -// std::cout << "Expected actual: " << count.expected << " expected: " << expectedCount << std::endl; -// std::cout << "Closed actual: " << count.closed << " expected: " << closedCount << std::endl; - expect (count.received == receivedCount, "Bad received count"); - expect (count.expected == expectedCount, "Bad expected count"); - expect (count.closed == closedCount, "Bad closed count"); - }; - - Validator validator; - std::uint64_t i = 1; - - // Received before closed - for (; i <= ledgersPerValidator; ++i) - { - RippleLedgerHash const hash {i}; - - bool const received = (i % 13 != 0); - bool const validated = (i % 7 != 0); - updateCounts (received, validated); - - if (received) - validator.on_validation (hash); - - if (validated) - validator.on_ledger (hash); - } - checkCounts (validator.count ()); - - // Closed before received - for (; i <= ledgersPerValidator * 2; ++i) - { - RippleLedgerHash const hash {i}; - - bool const received = (i % 11 != 0); - bool const validated = (i % 17 != 0); - updateCounts (received, validated); - - if (validated) - validator.on_ledger (hash); - - if (received) - validator.on_validation (hash); - } - checkCounts (validator.count ()); - - { - // Repeated receives - RippleLedgerHash const hash {++i}; - receivedCount += 1; - for (auto j = 0; j < 100; ++j) - { - validator.on_validation (hash); - } - } - checkCounts (validator.count ()); - - { - // Repeated closes - RippleLedgerHash const hash {++i}; - expectedCount += 1; - for (auto j = 0; j < 100; ++j) - { - validator.on_ledger (hash); - } - } - checkCounts (validator.count ()); - } - - void testLogic () - { - //TestStore store; - StoreSqdb storage; - - beast::File const file ( - beast::File::getSpecialLocation ( - beast::File::userDocumentsDirectory).getChildFile ( - "validators-test.sqlite")); - - // Can't call this 'error' because of ADL and Journal::error - beast::Error err (storage.open (file)); - - expect (! err, err.what()); - - Logic logic (storage, beast::Journal ()); - logic.load (); - - addSources (logic); - - logic.fetch_one (); - -// auto chosenSize (logic.getChosenSize ()); - - pass (); - } - void - run () + run() { - testLRUCache (); - testValidator (); - testLogic (); + pass(); } }; -BEAST_DEFINE_TESTSUITE(Logic,validators,ripple); +BEAST_DEFINE_TESTSUITE_MANUAL(Logic,validators,ripple); } } diff --git a/src/ripple/validators/impl/Tuning.h b/src/ripple/validators/impl/Tuning.h index 4343b5e48..d1aabdbe0 100644 --- a/src/ripple/validators/impl/Tuning.h +++ b/src/ripple/validators/impl/Tuning.h @@ -23,34 +23,6 @@ namespace ripple { namespace Validators { -// Tunable constants -// -enum -{ -#if 1 - // We will fetch a source at this interval - hoursBetweenFetches = 24 - ,secondsBetweenFetches = hoursBetweenFetches * 60 * 60 - // We check Source expirations on this time interval - ,checkEverySeconds = 60 * 60 -#else - secondsBetweenFetches = 59 - ,checkEverySeconds = 60 -#endif - - // This tunes the preallocated arrays - ,expectedNumberOfResults = 1000 - - // Number of entries in the recent validations cache - ,recentValidationsCacheSize = 1000 - - // Number of entries in the recent ledgers cache - ,recentLedgersCacheSize = 1000 // about half an hour at 2/sec - - // Number of closed Ledger entries per Validator - ,ledgersPerValidator = 100 // this shouldn't be too large -}; - } } diff --git a/src/ripple/validators/impl/Utilities.cpp b/src/ripple/validators/impl/Utilities.cpp deleted file mode 100644 index 6406b03ab..000000000 --- a/src/ripple/validators/impl/Utilities.cpp +++ /dev/null @@ -1,140 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include - -namespace ripple { -namespace Validators { - -struct Utilities::Helpers -{ - // Matches a validator info line. - // - static boost::regex const& reInfo () - { - // e.g. - // - // n9KorY8QtTdRx7TVDpwnG9NvyxsDwHUKUEeDLY3AkiGncVaSXZi5 Comment Text - // - static boost::regex re ( - "\\G" // end of last match (or start) - "(?:[\\v\\h]*)" // white (optional) - "([^\\h\\v]+)" // [1] non-white run - "(?:\\h*)" // horiz-white (optional) - "([^\\v]*?)" // [2] non vert-white text (optional) - "(?:\\h*)" // white run (optional) - "(?:\\v*)" // vert-white (optional) - - //"(?:\\')" // buffer boundary - - , boost::regex::perl - //| boost::regex_constants::match_flags::match_not_dot_newline - ); - - return re; - } - - // Matches a comment or whitespace line. - // - static boost::regex const& reComment () - { - // e.g. - // - // n9KorY8QtTdRx7TVDpwnG9NvyxsDwHUKUEeDLY3AkiGncVaSXZi5 Comment Text - // - static boost::regex re ( - "^" // start of line - "(?:\\h*)" // horiz-white (optional) - "(?:#.*)" // # then any text - "|" // - or - - "(?:\\h*)" // horiz-white - "$" // end of line - - , boost::regex::perl | - boost::regex_constants::match_flags::match_not_dot_null - ); - - return re; - } -}; - -//------------------------------------------------------------------------------ - -bool Utilities::parseInfoLine ( - Source::Item& item, - std::string const& line, - beast::Journal journal) -{ - bool success (false); - - boost::smatch match; - - if (boost::regex_match (line, match, Helpers::reInfo ())) - { - std::string const encodedKey (match [1]); - std::string const commentText (match [2]); - - std::pair results ( - RipplePublicKey::from_string (encodedKey)); - - if (results.second) - { - // We got a public key. - item.label = commentText; - item.publicKey = results.first; - success = true; - } - else - { - // Some other junk. - journal.error << "Invalid RipplePublicKey: '" << encodedKey << "'"; - } - } - else if (boost::regex_match (line, match, Helpers::reComment ())) - { - // it's a comment - } - else - { - // Log a warning about a parsing error - journal.error << "Invalid Validators source line:" << std::endl << line; - } - - return success; -} - -//------------------------------------------------------------------------------ - -void Utilities::parseResultLine ( - Source::Results& results, - std::string const& line, - beast::Journal journal) -{ - Source::Item item; - - bool success = parseInfoLine (item, line, journal); - if (success) - { - results.list.push_back (item); - results.success = true; - } -} - -} -} diff --git a/src/ripple/validators/impl/Utilities.h b/src/ripple/validators/impl/Utilities.h deleted file mode 100644 index 3ebf457fd..000000000 --- a/src/ripple/validators/impl/Utilities.h +++ /dev/null @@ -1,136 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_VALIDATORS_UTILITIES_H_INCLUDED -#define RIPPLE_VALIDATORS_UTILITIES_H_INCLUDED - -namespace ripple { -namespace Validators { - -/** Common code for Validators classes. */ -class Utilities -{ -public: - typedef std::vector Strings; - - /** A suitable LineFunction for parsing items into a fetch results. */ - class ParseResultLine - { - public: - ParseResultLine (Source::Results& results, beast::Journal journal) - : m_result (&results) - , m_journal (journal) - { } - - template - void operator() (BidirectionalIterator first, BidirectionalIterator last) - { - std::string s (first, last); - Utilities::parseResultLine (*m_result, s, m_journal); - } - - private: - Source::Results* m_result; - beast::Journal m_journal; - }; - - /** UnaryPredicate for breaking up lines. - This returns `true` for the first non-vertical whitespace character that - follows a vertical whitespace character. - */ - class FollowingVerticalWhite - { - public: - FollowingVerticalWhite () - : m_gotWhite (false) - { - } - - template - static bool isVerticalWhitespace (CharT c) - { - return c == '\r' || c == '\n'; - } - - template - bool operator() (CharT c) - { - if (isVerticalWhitespace (c)) - { - m_gotWhite = true; - return false; - } - else if (m_gotWhite) - { - m_gotWhite = false; - return true; - } - return false; - } - - private: - bool m_gotWhite; - }; - - /** Call LineFunction for each newline-separated line in the input. - LineFunction will be called with this signature: - @code - void LineFunction (BidirectionalIterator first, BidirectionalIterator last) - @endcode - Where first and last mark the beginning and ending of the line. - The last line in the input may or may not contain the trailing newline. - */ - template - static void processLines (BidirectionalIterator first, - BidirectionalIterator last, LineFunction f) - { - for (;;) - { - BidirectionalIterator split (std::find_if ( - first, last, FollowingVerticalWhite ())); - f (first, split); - if (split == last) - break; - first = split; - } - } - - /** Parse a string into the Source::Results. - Invalid or comment lines will be skipped. - Lines containing validator info will be added to the Results object. - Metadata lines will update the corresponding Results fields. - */ - static void parseResultLine ( - Source::Results& results, - std::string const& line, - beast::Journal journal = beast::Journal()); - - struct Helpers; - - /** Parse a string into a Source::Item. - @return `true` on success. - */ - static bool parseInfoLine ( - Source::Item& item, std::string const& line, beast::Journal journal); -}; - -} -} - -#endif diff --git a/src/ripple/validators/impl/Validation.h b/src/ripple/validators/impl/Validation.h deleted file mode 100644 index 6f2724ffa..000000000 --- a/src/ripple/validators/impl/Validation.h +++ /dev/null @@ -1,58 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_VALIDATORS_VALIDATION_INCLUDED -#define RIPPLE_VALIDATORS_VALIDATION_INCLUDED - -namespace ripple { -namespace Validators { - -/** Hash function for ReceivedValidation. */ -class ReceivedValidationHash -{ -public: - std::size_t operator() (ReceivedValidation const& key) const - { - return m_ledger_hasher (key.ledgerHash) + - m_key_hasher (key.publicKey); - } - -private: - RippleLedgerHash::hasher m_ledger_hasher; - RipplePublicKey::hasher m_key_hasher; -}; - -//------------------------------------------------------------------------------ - -/** KeyEqual function for ReceivedValidation. */ -class ReceivedValidationKeyEqual -{ -public: - bool operator() (ReceivedValidation const& lhs, - ReceivedValidation const& rhs) const - { - return lhs.ledgerHash == rhs.ledgerHash && - lhs.publicKey == rhs.publicKey; - } -}; - -} -} - -#endif diff --git a/src/ripple/validators/impl/Validator.h b/src/ripple/validators/impl/Validator.h deleted file mode 100644 index 7789f5842..000000000 --- a/src/ripple/validators/impl/Validator.h +++ /dev/null @@ -1,151 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_VALIDATORS_VALIDATOR_H_INCLUDED -#define RIPPLE_VALIDATORS_VALIDATOR_H_INCLUDED - -#include -#include -#include -#include -#include - -namespace ripple { -namespace Validators { - -/** Tracks statistics on a validator. */ -class Validator -{ -private: - // State of a ledger. - struct Entry - { - bool closed = false; // `true` if the ledger was closed - bool received = false; // `true` if we got a validation - }; - - // Holds the Entry of all recent ledgers for this validator. -#if 1 - typedef beast::aged_unordered_map , - RippleLedgerHash::key_equal> Table; -#else - typedef beast::aged_map > Table; -#endif - - int refs_; // Number of sources that reference this validator. - Table table_; - Count count_; - -public: - Validator() - : refs_ (0) - , table_ (get_seconds_clock ()) - { - } - - /** Increment the number of references to this validator. */ - void - addRef() - { - ++refs_; - } - - /** Decrement the number of references to this validator. - When the reference count reaches zero, the validator will - be removed and no longer tracked. - */ - bool - release() - { - return (--refs_) == 0; - } - - size_t - size () const - { - return table_.size (); - } - - /** Returns the composite performance statistics. */ - Count const& - count () const - { - return count_; - } - - /** Called upon receipt of a validation. */ - void - on_validation (RippleLedgerHash const& ledgerHash) - { - //expire(); - auto const result (table_.insert ( - std::make_pair (ledgerHash, Entry()))); - auto& entry (result.first->second); - if (entry.received) - return; - entry.received = true; - if (entry.closed) - { - --count_.expected; - ++count_.closed; - table_.erase (result.first); - } - else - { - ++count_.received; - } - } - - /** Called when a ledger is closed. */ - void - on_ledger (RippleLedgerHash const& ledgerHash) - { - //expire(); - auto const result (table_.insert ( - std::make_pair (ledgerHash, Entry()))); - auto& entry (result.first->second); - if (entry.closed) - return; - entry.closed = true; - if (entry.received) - { - --count_.received; - ++count_.closed; - table_.erase (result.first); - } - else - { - ++count_.expected; - } - } - - /** Prunes old entries. */ - void - expire() - { - beast::expire (table_, std::chrono::minutes(5)); - } -}; - -} -} - -#endif diff --git a/src/ripple/validators/Types.h b/src/ripple/validators/make_Manager.h similarity index 68% rename from src/ripple/validators/Types.h rename to src/ripple/validators/make_Manager.h index 76004e242..07ca2cb47 100644 --- a/src/ripple/validators/Types.h +++ b/src/ripple/validators/make_Manager.h @@ -17,31 +17,24 @@ */ //============================================================================== -#ifndef RIPPLE_VALIDATORS_TYPES_H_INCLUDED -#define RIPPLE_VALIDATORS_TYPES_H_INCLUDED +#ifndef RIPPLE_VALIDATORS_MAKE_MANAGER_H_INCLUDED +#define RIPPLE_VALIDATORS_MAKE_MANAGER_H_INCLUDED -#include +#include +#include +#include +#include +#include +#include namespace ripple { namespace Validators { -struct ReceivedValidation -{ - ReceivedValidation () - { - } - - ReceivedValidation ( - LedgerHash const& ledgerHash_, - RipplePublicKey const& publicKey_) - : ledgerHash (ledgerHash_) - , publicKey (publicKey_) - { - } - - RippleLedgerHash ledgerHash; - RipplePublicKey publicKey; -}; +std::unique_ptr +make_Manager (beast::Stoppable& stoppableParent, + boost::asio::io_service& io_service, + beast::File const& pathToDbFileOrDirectory, + beast::Journal journal); } } diff --git a/src/ripple/validators/tests/Validators.test.cpp b/src/ripple/validators/tests/Validators.test.cpp index 62c70ec9c..8233d3381 100644 --- a/src/ripple/validators/tests/Validators.test.cpp +++ b/src/ripple/validators/tests/Validators.test.cpp @@ -17,11 +17,7 @@ */ //============================================================================== -#include -#include -#include -#include -#include +#include namespace ripple { namespace Validators { @@ -29,99 +25,14 @@ namespace Validators { class Validators_test : public beast::unit_test::suite { public: - struct Entry - { - bool closed = false; // `true` if the ledger was closed - bool received = false; // `true` if we got a validation - }; - - typedef beast::aged_unordered_map , - RippleLedgerHash::key_equal> Table; - - template - static void - fillrand (void* buffer, std::size_t bytes, Gen& gen) + run() { - auto p = reinterpret_cast(buffer); - typedef typename Gen::result_type result_type; - while (bytes >= sizeof(result_type)) - { - *reinterpret_cast(p) = gen(); - p += sizeof(result_type); - bytes -= sizeof(result_type); - } - if (bytes > 0) - { - auto const v = gen(); - memcpy (p, &v, bytes); - } - } - - void - test_aged_insert() - { - testcase ("aged insert"); - std::random_device rng; - std::mt19937_64 gen {rng()}; - Table table (get_seconds_clock()); - for (int i = 0; i < 10000; ++i) - { - std::array buf; - fillrand (buf.data(), buf.size(), gen); - RippleLedgerHash h (buf.data(), buf.data() + buf.size()); - table.insert (std::make_pair (h, Entry())); - } pass(); } - - void - test_Validators() - { - int const N (5); - testcase ("Validators"); - typedef hardened_hash_map Validators; - Validators vv; - for (int i = 0; i < N; ++i) - vv.emplace (i, Validator{}); - std::random_device rng; - std::mt19937_64 gen {rng()}; - std::array buf; - fillrand (buf.data(), buf.size(), gen); - for (int i = 0; i < 100000; ++i) - { - // maybe change the ledger hash - if ((gen() % 20) == 0) - fillrand (buf.data(), buf.size(), gen); - RippleLedgerHash h (buf.data(), buf.data() + buf.size()); - // choose random validator - Validator& v (vv[gen() % vv.size()]); - // choose random operation - //int const choice = gen() % 2; - int const choice = 1; - switch (choice) - { - case 0: - v.on_ledger(h); - break; - case 1: - v.on_validation(h); - break; - }; - } - pass(); - } - - void - run () - { - test_aged_insert(); - test_Validators(); - } }; -BEAST_DEFINE_TESTSUITE(Validators,validators,ripple); +BEAST_DEFINE_TESTSUITE_MANUAL(Validators,validators,ripple); } }