diff --git a/Builds/VisualStudio2012/RippleD.vcxproj b/Builds/VisualStudio2012/RippleD.vcxproj index fc5784660..33d7ed818 100644 --- a/Builds/VisualStudio2012/RippleD.vcxproj +++ b/Builds/VisualStudio2012/RippleD.vcxproj @@ -33,31 +33,43 @@ true - + true true true true - + true true true true - + true true true true - + true true true true - + + true + true + true + true + + + true + true + true + true + + true true true @@ -1409,11 +1421,17 @@ - - - - - + + + + + + + + + + + diff --git a/Builds/VisualStudio2012/RippleD.vcxproj.filters b/Builds/VisualStudio2012/RippleD.vcxproj.filters index 7c746036d..079fc8487 100644 --- a/Builds/VisualStudio2012/RippleD.vcxproj.filters +++ b/Builds/VisualStudio2012/RippleD.vcxproj.filters @@ -882,21 +882,6 @@ [2] Ripple %28New%29\validators - - [2] Ripple %28New%29\validators\impl - - - [2] Ripple %28New%29\validators\impl - - - [2] Ripple %28New%29\validators\impl - - - [2] Ripple %28New%29\validators\impl - - - [2] Ripple %28New%29\validators\impl - [1] Ripple\ripple_app\main @@ -915,6 +900,27 @@ [1] Ripple\ripple_basics\log + + [2] Ripple %28New%29\validators\impl + + + [2] Ripple %28New%29\validators\impl + + + [2] Ripple %28New%29\validators\impl + + + [2] Ripple %28New%29\validators\impl + + + [2] Ripple %28New%29\validators\impl + + + [2] Ripple %28New%29\validators\impl + + + [2] Ripple %28New%29\validators\impl + @@ -1746,21 +1752,6 @@ [2] Ripple %28New%29\validators - - [2] Ripple %28New%29\validators\impl - - - [2] Ripple %28New%29\validators\impl - - - [2] Ripple %28New%29\validators\impl - - - [2] Ripple %28New%29\validators\impl - - - [2] Ripple %28New%29\validators\api - [1] Ripple\ripple_app\main @@ -1785,6 +1776,39 @@ [1] Ripple\ripple_basics\log + + [2] Ripple %28New%29\validators\impl + + + [2] Ripple %28New%29\validators\api + + + [2] Ripple %28New%29\validators\api + + + [2] Ripple %28New%29\validators\api + + + [2] Ripple %28New%29\validators\impl + + + [2] Ripple %28New%29\validators\impl + + + [2] Ripple %28New%29\validators\impl + + + [2] Ripple %28New%29\validators\impl + + + [2] Ripple %28New%29\validators\impl + + + [2] Ripple %28New%29\validators\impl + + + [2] Ripple %28New%29\validators\impl + diff --git a/src/ripple/validators/api/Validators.h b/src/ripple/validators/api/Manager.h similarity index 52% rename from src/ripple/validators/api/Validators.h rename to src/ripple/validators/api/Manager.h index bb9bcce1c..e6ce08cbb 100644 --- a/src/ripple/validators/api/Validators.h +++ b/src/ripple/validators/api/Manager.h @@ -4,8 +4,11 @@ */ //============================================================================== -#ifndef RIPPLE_CORE_VALIDATOR_VALIDATORS_H_INCLUDED -#define RIPPLE_CORE_VALIDATOR_VALIDATORS_H_INCLUDED +#ifndef RIPPLE_VALIDATORS_MANAGER_H_INCLUDED +#define RIPPLE_VALIDATORS_MANAGER_H_INCLUDED + +namespace Validators +{ /** Maintains the list of chosen validators. @@ -14,60 +17,12 @@ All operations are performed asynchronously on an internal thread. */ -class Validators : public Uncopyable +class Manager : public Uncopyable { public: - typedef RipplePublicKeyHash KeyType; - - //-------------------------------------------------------------------------- - - /** A source of validator descriptors. */ - class Source - { - public: - /** A Source's descriptor for a Validator. */ - struct Info - { - /** The unique key for this validator. */ - KeyType key; - }; - - /** Destroy the Source. - This can be called from any thread. If the Source is busy - fetching, the destructor must block until the operation is either - canceled or complete. - */ - virtual ~Source () { } - - struct CancelCallback - { - virtual bool shouldCancel () = 0; - }; - - /** Fetch the most recent list from the Source. - If possible, the Source should periodically poll the - CancelCallback, and abort the operation if shouldCancel - returns `true`. - This call will block. - */ - struct Result - { - Result (); - void swapWith (Result& other); - - bool success; - String message; - Time expirationTime; - Array list; - }; - virtual Result fetch (CancelCallback& callback) = 0; - }; - - //-------------------------------------------------------------------------- - - /** Create a new Validators object. + /** Create a new Manager object. */ - static Validators* New (); + static Manager* New (Journal journal); /** Destroy the object. @@ -76,12 +31,14 @@ public: There may be some listener calls made before the destructor returns. */ - virtual ~Validators () { } + virtual ~Manager () { } /** Add a static source of validators from a string array. */ /** @{ */ - virtual void addStrings (std::vector const& strings) = 0; - virtual void addStrings (StringArray const& stringArray) = 0; + virtual void addStrings (String name, + std::vector const& strings) = 0; + virtual void addStrings (String name, + StringArray const& stringArray) = 0; /** @} */ /** Add a static source of validators from a text file. */ @@ -113,19 +70,15 @@ public: // Trusted Validators - //virtual bool isPublicKeyTrusted (Validator::PublicKey const&) = 0; + //virtual bool isPublicKeyTrusted (PublicKey const& publicKey) = 0; //-------------------------------------------------------------------------- - struct ReceivedValidation - { - uint256 ledgerHash; - RipplePublicKeyHash signerPublicKeyHash; - }; - /** Called when a validation with a proper signature is received. */ virtual void receiveValidation (ReceivedValidation const& rv) = 0; }; +} + #endif diff --git a/src/ripple/validators/api/Source.h b/src/ripple/validators/api/Source.h new file mode 100644 index 000000000..88ac345ce --- /dev/null +++ b/src/ripple/validators/api/Source.h @@ -0,0 +1,60 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_VALIDATORS_SOURCE_H_INCLUDED +#define RIPPLE_VALIDATORS_SOURCE_H_INCLUDED + +namespace Validators +{ + +/** A source of validator descriptors. */ +class Source +{ +public: + /** A Source's descriptor for a Validator. */ + struct Info + { + /** The unique key for this validator. */ + PublicKey key; + }; + + /** Destroy the Source. + This can be called from any thread. If the Source is busy + fetching, the destructor must block until the operation is either + canceled or complete. + */ + virtual ~Source () { } + + virtual String name () = 0; + + struct CancelCallback + { + virtual bool shouldCancel () = 0; + }; + + /** Fetch the most recent list from the Source. + If possible, the Source should periodically poll the + CancelCallback, and abort the operation if shouldCancel + returns `true`. + This call will block. + */ + struct Result + { + Result (); + void swapWith (Result& other); + + bool success; + String message; + Time expirationTime; + Array list; + }; + + virtual Result fetch (CancelCallback& callback, Journal journal) = 0; +}; + +} + +#endif diff --git a/src/ripple/validators/api/Types.h b/src/ripple/validators/api/Types.h new file mode 100644 index 000000000..187ad3a2e --- /dev/null +++ b/src/ripple/validators/api/Types.h @@ -0,0 +1,24 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_VALIDATORS_TYPES_H_INCLUDED +#define RIPPLE_VALIDATORS_TYPES_H_INCLUDED + +namespace Validators +{ + +typedef RipplePublicKey PublicKey; +typedef RipplePublicKeyHash PublicKeyHash; + +struct ReceivedValidation +{ + uint256 ledgerHash; + PublicKeyHash signerPublicKeyHash; +}; + +} + +#endif diff --git a/src/ripple/validators/impl/CancelCallbacks.h b/src/ripple/validators/impl/CancelCallbacks.h new file mode 100644 index 000000000..842b5e710 --- /dev/null +++ b/src/ripple/validators/impl/CancelCallbacks.h @@ -0,0 +1,54 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_VALIDATORS_CANCELCALLBACKS_H_INCLUDED +#define RIPPLE_VALIDATORS_CANCELCALLBACKS_H_INCLUDED + +namespace Validators +{ + +// Dummy CancelCallback that does nothing +// +class NoOpCancelCallback : public Source::CancelCallback +{ +public: + bool shouldCancel () + { + return false; + } + +}; + +//------------------------------------------------------------------------------ + +// CancelCallback attached to ThreadWithCallQueue +// +class ThreadCancelCallback + : public Source::CancelCallback + , public Uncopyable +{ +public: + explicit ThreadCancelCallback (ThreadWithCallQueue& thread) + : m_thread (thread) + , m_interrupted (false) + { + } + + bool shouldCancel () + { + if (m_interrupted) + return true; + return m_interrupted = m_thread.interruptionPoint (); + } + +private: + ThreadWithCallQueue& m_thread; + bool m_interrupted; +}; + +} + +#endif diff --git a/src/ripple/validators/impl/ChosenList.h b/src/ripple/validators/impl/ChosenList.h new file mode 100644 index 000000000..1e7f8991c --- /dev/null +++ b/src/ripple/validators/impl/ChosenList.h @@ -0,0 +1,59 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_VALIDATORS_CHOSENLIST_H_INCLUDED +#define RIPPLE_VALIDATORS_CHOSENLIST_H_INCLUDED + +namespace Validators +{ + +class ChosenList : public SharedObject +{ +public: + typedef SharedPtr Ptr; + + struct Info + { + Info () + { + } + }; + + typedef boost::unordered_map MapType; + + ChosenList (std::size_t expectedSize = 0) + { + // Available only in recent boost versions? + //m_map.reserve (expectedSize); + } + + std::size_t size () const noexcept + { + return m_map.size (); + } + + void insert (PublicKey const& key, Info const& info) noexcept + { + m_map [key] = info; + } + + bool containsPublicKey (PublicKey const& publicKey) const noexcept + { + return m_map.find (publicKey) != m_map.cend (); + } + + bool containsPublicKeyHash (PublicKeyHash const& publicKeyHash) const noexcept + { + return false; + } + +private: + MapType m_map; +}; + +} + +#endif diff --git a/src/ripple/validators/impl/Logic.h b/src/ripple/validators/impl/Logic.h new file mode 100644 index 000000000..44c3d9652 --- /dev/null +++ b/src/ripple/validators/impl/Logic.h @@ -0,0 +1,320 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_VALIDATORS_LOGIC_H_INCLUDED +#define RIPPLE_VALIDATORS_LOGIC_H_INCLUDED + +namespace Validators +{ + +// Tunable constants +enum +{ + // We will fetch a source at this interval + hoursBetweenFetches = 24 + + ,secondsBetweenFetches = hoursBetweenFetches * 60 * 60 + + // We check Source expirations on this time interval + ,checkEverySeconds = 60 * 60 + + // This tunes the preallocated arrays + ,expectedNumberOfResults = 1000 +}; + +// Encapsulates the logic for creating the chosen validators. +// This is a separate class to facilitate the unit tests. +// +class Logic +{ +public: + // Information associated with each Source + // + struct SourceDesc + { + enum + { + keysPreallocationSize = 1000 + }; + + enum Status + { + statusNone, + statusFetched, + statusFailed + }; + + ScopedPointer source; + Status status; + Time whenToFetch; + int numberOfFailures; + + // The result of the last fetch + Source::Result result; + + //------------------------------------------------------------------ + + SourceDesc () noexcept + : status (statusNone) + , whenToFetch (Time::getCurrentTime ()) + , numberOfFailures (0) + { + } + + ~SourceDesc () + { + } + }; + + typedef DynamicList SourcesType; + + //---------------------------------------------------------------------- + + // Information associated with each distinguishable validator + // + struct ValidatorInfo + { + ValidatorInfo () + : refCount (0) + { + } + + int refCount; + }; + + typedef boost::unordered_map MapType; + + //---------------------------------------------------------------------- + + explicit Logic (Journal journal = Journal ()) + : m_journal (journal) + , m_chosenListNeedsUpdate (false) + { + } + + // Add a one-time static source. + // Fetch is called right away, this call blocks. + // + void addStaticSource (Source* source) + { + m_journal.info() << "Add static Source, " << source->name(); + + ScopedPointer object (source); + + NoOpCancelCallback cancelCallback; + + Source::Result result (object->fetch (cancelCallback, m_journal)); + + if (result.success) + { + addSourceInfo (result.list); + } + else + { + // VFALCO NOTE Maybe log the error and message? + } + } + + // Add a live source to the list of sources. + // + void addSource (Source* source) + { + m_journal.info() << "Add Source, " << source->name(); + + SourceDesc& desc (*m_sources.emplace_back ()); + desc.source = source; + } + + // Called when we receive a validation from a peer. + // + void receiveValidation (ReceivedValidation const& rv) + { +#if 0 + MapType::iterator iter (m_map.find (rv.signerPublicKeyHash)); + if (iter != m_map.end ()) + { + // Exists + //ValidatorInfo& validatorInfo (iter->value ()); + } + else + { + // New + //ValidatorInfo& validatorInfo (m_map.insert (rv.signerPublicKeyHash)); + } +#endif + } + + // Add each entry in the list to the map, incrementing the + // reference count if it already exists, and updating fields. + // + void addSourceInfo (Array const& list) + { + for (std::size_t i = 0; i < list.size (); ++i) + { + Source::Info const& info (list.getReference (i)); + std::pair result ( + m_map.emplace (info.key, ValidatorInfo ())); + ValidatorInfo& validatorInfo (result.first->second); + ++validatorInfo.refCount; + if (result.second) + { + // This is a new one + markDirtyChosenList (); + } + } + } + + // Decrement the reference count of each item in the list + // in the map + // + void removeSourceInfo (Array const& list) + { + for (std::size_t i = 0; i < list.size (); ++i) + { + Source::Info const& info (list.getReference (i)); + MapType::iterator iter (m_map.find (info.key)); + bassert (iter != m_map.end ()); + ValidatorInfo& validatorInfo (iter->second); + if (--validatorInfo.refCount == 0) + { + // Last reference removed + m_map.erase (iter); + markDirtyChosenList (); + } + } + } + + // Fetch one source + // + void fetchSource (SourceDesc& desc, Source::CancelCallback& callback) + { + m_journal.info() << "Fetching Source, " << desc.source->name(); + + Source::Result result (desc.source->fetch (callback, m_journal)); + + if (! callback.shouldCancel ()) + { + // Reset fetch timer for the source. + desc.whenToFetch = Time::getCurrentTime () + + RelativeTime (secondsBetweenFetches); + + if (result.success) + { + // Add the new source info to the map + addSourceInfo (result.list); + + // Swap lists + desc.result.swapWith (result); + + // Remove the old source info from the map + removeSourceInfo (result.list); + + // See if we need to rebuild + checkDirtyChosenList (); + + // Reset failure status + desc.numberOfFailures = 0; + desc.status = SourceDesc::statusFetched; + } + else + { + ++desc.numberOfFailures; + desc.status = SourceDesc::statusFailed; + } + } + } + + // Check each source to see if it needs fetching. + // + void checkSources (Source::CancelCallback& callback) + { + m_journal.info() << "Checking Sources"; + + Time const currentTime (Time::getCurrentTime ()); + for (SourcesType::iterator iter = m_sources.begin (); + ! callback.shouldCancel () && iter != m_sources.end (); ++iter) + { + SourceDesc& desc (*iter); + if (desc.whenToFetch <= currentTime) + fetchSource (desc, callback); + } + } + + // Signal that the Chosen List needs to be rebuilt. + // + void markDirtyChosenList () + { + m_chosenListNeedsUpdate = true; + } + + // Check the dirty state of the Chosen List, and rebuild it + // if necessary. + // + void checkDirtyChosenList () + { + if (m_chosenListNeedsUpdate) + { + buildChosenList (); + m_chosenListNeedsUpdate = false; + } + } + + // Rebuilds the Chosen List + // + void buildChosenList () + { + ChosenList::Ptr list (new ChosenList (m_map.size ())); + + for (MapType::iterator iter = m_map.begin (); + iter != m_map.end (); ++iter) + { + ChosenList::Info info; + list->insert (iter->first, info); + } + + // This is thread safe + m_chosenList = list; + + m_journal.debug() << + "Rebuilt chosen list with " << + String::fromNumber (m_chosenList->size()) << " entries"; + } + + // Get a reference to the chosen list. + // This is safe to call from any thread at any time. + // + ChosenList::Ptr getChosenList () + { + return m_chosenList; + } + + //---------------------------------------------------------------------- + // + // Ripple interface + // + // These routines are modeled after UniqueNodeList + + bool isTrustedPublicKeyHash ( + PublicKeyHash const& publicKeyHash) + { + return m_chosenList->containsPublicKeyHash (publicKeyHash); + } + + // + // + //---------------------------------------------------------------------- + +private: + Journal m_journal; + SourcesType m_sources; + MapType m_map; + bool m_chosenListNeedsUpdate; + ChosenList::Ptr m_chosenList; +}; + +} + +#endif diff --git a/src/ripple/validators/impl/Manager.cpp b/src/ripple/validators/impl/Manager.cpp new file mode 100644 index 000000000..a9e4b0743 --- /dev/null +++ b/src/ripple/validators/impl/Manager.cpp @@ -0,0 +1,200 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +/* + +Information to track: + +- Percentage of validations that the validator has signed +- Number of validations the validator signed that never got accepted + + +- Target number for Chosen +- Pseudo-randomly choose a subset from Chosen + + + + + +Goal: + + Provide the listener with a ValidatorList. + - This forms the UNL + +Task: + + fetch ValidatorInfo array from a source + + - We have the old one and the new one, compute the following: + + * unchanged validators list + * new validators list + * removed validators list + + - From the unchanged / new / removed, figure out what to do. + +Two important questions: + +- Are there any validators in my ChosenValidators that I dont want + * For example, they have dropped off all the trusted lists + +- Do I have enough? + +-------------------------------------------------------------------------------- +ChosenValidators +-------------------------------------------------------------------------------- + +David: + Maybe OC should have a URL that you can query to get the latest list of URI's + for OC-approved organzations that publish lists of validators. The server and + client can ship with that master trust URL and also the list of URI's at the + time it's released, in case for some reason it can't pull from OC. That would + make the default installation safe even against major changes in the + organizations that publish validator lists. + + The difference is that if an organization that provides lists of validators + goes rogue, administrators don't have to act. + +TODO: + Write up from end-user perspective on the deployment and administration + of this feature, on the wiki. "DRAFT" or "PROPOSE" to mark it as provisional. + Template: https://ripple.com/wiki/Federation_protocol + - What to do if you're a publisher of ValidatorList + - What to do if you're a rippled administrator + - Overview of how ChosenValidators works + +Goals: + Make default configuration of rippled secure. + * Ship with TrustedUriList + * Also have a preset RankedValidators + Eliminate administrative burden of maintaining + Produce the ChosenValidators list. + Allow quantitative analysis of network health. + +What determines that a validator is good? + - Are they present (i.e. sending validations) + - Are they on the consensus ledger + - What percentage of consensus rounds do they participate in + - Are they stalling consensus + * Measurements of constructive/destructive behavior is + calculated in units of percentage of ledgers for which + the behavior is measured. +*/ + +namespace Validators +{ + +class ManagerImp + : public Manager + , private ThreadWithCallQueue::EntryPoints + , private DeadlineTimer::Listener + , private LeakChecked +{ +public: + explicit ManagerImp (Journal journal) + : m_logic (journal) + , m_journal (journal) + , m_thread ("Validators") + , m_checkTimer (this) + { + m_thread.start (this); + } + + ~ManagerImp () + { + } + + void addStrings (String name, std::vector const& strings) + { + StringArray stringArray; + stringArray.ensureStorageAllocated (strings.size()); + for (std::size_t i = 0; i < strings.size(); ++i) + stringArray.add (strings [i]); + addStrings (name, stringArray); + } + + void addStrings (String name, StringArray const& stringArray) + { + addStaticSource (SourceStrings::New ( + name, stringArray)); + } + + void addFile (File const& file) + { + addStaticSource (SourceFile::New (file)); + } + + void addURL (UniformResourceLocator const& url) + { + addSource (SourceURL::New (url)); + } + + //-------------------------------------------------------------------------- + + void addSource (Source* source) + { + m_thread.call (&Logic::addSource, &m_logic, source); + } + + void addStaticSource (Source* source) + { + m_thread.call (&Logic::addStaticSource, &m_logic, source); + } + + void receiveValidation (ReceivedValidation const& rv) + { + m_thread.call (&Logic::receiveValidation, &m_logic, rv); + } + + //-------------------------------------------------------------------------- + + // This intermediate function is used to provide the CancelCallback + void checkSources () + { + ThreadCancelCallback cancelCallback (m_thread); + + m_logic.checkSources (cancelCallback); + } + + void onDeadlineTimer (DeadlineTimer& timer) + { + if (timer == m_checkTimer) + m_thread.call (&ManagerImp::checkSources, this); + } + + //-------------------------------------------------------------------------- + + void threadInit () + { + m_checkTimer.setRecurringExpiration (checkEverySeconds); + } + + void threadExit () + { + } + + bool threadIdle () + { + bool interrupted = false; + + return interrupted; + } + +private: + Logic m_logic; + Journal m_journal; + ThreadWithCallQueue m_thread; + DeadlineTimer m_checkTimer; +}; + +//------------------------------------------------------------------------------ + +Manager* Manager::New (Journal journal) +{ + return new ManagerImp (journal); +} + +} diff --git a/src/ripple/validators/impl/Source.cpp b/src/ripple/validators/impl/Source.cpp new file mode 100644 index 000000000..3095574a9 --- /dev/null +++ b/src/ripple/validators/impl/Source.cpp @@ -0,0 +1,23 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +namespace Validators +{ + +Source::Result::Result () + : success (false) + , message ("uninitialized") +{ +} + +void Source::Result::swapWith (Result& other) +{ + std::swap (success, other.success); + std::swap (message, other.message); + list.swapWith (other.list); +} + +} diff --git a/src/ripple/validators/impl/ValidatorSourceFile.cpp b/src/ripple/validators/impl/SourceFile.cpp similarity index 55% rename from src/ripple/validators/impl/ValidatorSourceFile.cpp rename to src/ripple/validators/impl/SourceFile.cpp index f937ca887..3589e870a 100644 --- a/src/ripple/validators/impl/ValidatorSourceFile.cpp +++ b/src/ripple/validators/impl/SourceFile.cpp @@ -4,19 +4,27 @@ */ //============================================================================== -class ValidatorSourceFileImp : public ValidatorSourceFile +namespace Validators +{ + +class SourceFileImp : public SourceFile { public: - ValidatorSourceFileImp (File const& file) + SourceFileImp (File const& file) : m_file (file) { } - ~ValidatorSourceFileImp () + ~SourceFileImp () { } + + String name () + { + return "File :'" + m_file.getFullPathName () + "'"; + } - Result fetch (CancelCallback&) + Result fetch (CancelCallback&, Journal journal) { Result result; @@ -29,10 +37,12 @@ private: //------------------------------------------------------------------------------ -ValidatorSourceFile* ValidatorSourceFile::New (File const& file) +SourceFile* SourceFile::New (File const& file) { - ScopedPointer object ( - new ValidatorSourceFileImp (file)); + ScopedPointer object ( + new SourceFileImp (file)); return object.release (); } + +} diff --git a/src/ripple/validators/impl/ValidatorSourceFile.h b/src/ripple/validators/impl/SourceFile.h similarity index 59% rename from src/ripple/validators/impl/ValidatorSourceFile.h rename to src/ripple/validators/impl/SourceFile.h index c978b4fe5..0b6418d96 100644 --- a/src/ripple/validators/impl/ValidatorSourceFile.h +++ b/src/ripple/validators/impl/SourceFile.h @@ -4,16 +4,21 @@ */ //============================================================================== -#ifndef RIPPLE_CORE_VALIDATOR_VALIDATORSOURCEFILE_H_INCLUDED -#define RIPPLE_CORE_VALIDATOR_VALIDATORSOURCEFILE_H_INCLUDED +#ifndef RIPPLE_VALIDATORS_SOURCEFILE_H_INCLUDED +#define RIPPLE_VALIDATORS_SOURCEFILE_H_INCLUDED + +namespace Validators +{ /** Provides validators from a text file. Typically this will come from a local configuration file. */ -class ValidatorSourceFile : public Validators::Source +class SourceFile : public Source { public: - static ValidatorSourceFile* New (File const& path); + static SourceFile* New (File const& path); }; +} + #endif diff --git a/src/ripple/validators/impl/ValidatorSourceStrings.cpp b/src/ripple/validators/impl/SourceStrings.cpp similarity index 53% rename from src/ripple/validators/impl/ValidatorSourceStrings.cpp rename to src/ripple/validators/impl/SourceStrings.cpp index 2ab8d2cc7..e87948bb8 100644 --- a/src/ripple/validators/impl/ValidatorSourceStrings.cpp +++ b/src/ripple/validators/impl/SourceStrings.cpp @@ -4,19 +4,29 @@ */ //============================================================================== -class ValidatorSourceStringsImp : public ValidatorSourceStrings +namespace Validators +{ + +class SourceStringsImp : public SourceStrings { public: - ValidatorSourceStringsImp (StringArray const& strings) - : m_strings (strings) + SourceStringsImp ( + String name, StringArray const& strings) + : m_name (name) + , m_strings (strings) { } - ~ValidatorSourceStringsImp () + ~SourceStringsImp () { } - Result fetch (CancelCallback&) + String name () + { + return m_name; + } + + Result fetch (CancelCallback&, Journal journal) { Result result; @@ -24,7 +34,8 @@ public: for (int i = 0; i < m_strings.size (); ++i) { - ValidatorsUtilities::parseResultLine (result, m_strings [i]); + std::string const s (m_strings [i].toStdString ()); + Utilities::parseResultLine (result, s); } result.success = result.list.size () > 0; @@ -33,15 +44,19 @@ public: } private: + String m_name; StringArray m_strings; }; //------------------------------------------------------------------------------ -ValidatorSourceStrings* ValidatorSourceStrings::New (StringArray const& strings) +SourceStrings* SourceStrings::New ( + String name, StringArray const& strings) { - ScopedPointer object ( - new ValidatorSourceStringsImp (strings)); + ScopedPointer object ( + new SourceStringsImp (name, strings)); return object.release (); } + +} diff --git a/src/ripple/validators/impl/ValidatorSourceStrings.h b/src/ripple/validators/impl/SourceStrings.h similarity index 58% rename from src/ripple/validators/impl/ValidatorSourceStrings.h rename to src/ripple/validators/impl/SourceStrings.h index 9fc1fa427..3e1092a54 100644 --- a/src/ripple/validators/impl/ValidatorSourceStrings.h +++ b/src/ripple/validators/impl/SourceStrings.h @@ -4,16 +4,22 @@ */ //============================================================================== -#ifndef RIPPLE_CORE_VALIDATOR_VALIDATORSOURCESTRINGS_H_INCLUDED -#define RIPPLE_CORE_VALIDATOR_VALIDATORSOURCESTRINGS_H_INCLUDED +#ifndef RIPPLE_VALIDATORS_SOURCESTRINGS_H_INCLUDED +#define RIPPLE_VALIDATORS_SOURCESTRINGS_H_INCLUDED + +namespace Validators +{ /** Provides validators from a set of Validator strings. Typically this will come from a local configuration file. */ -class ValidatorSourceStrings : public Validators::Source +class SourceStrings : public Source { public: - static ValidatorSourceStrings* New (StringArray const& strings); + static SourceStrings* New ( + String name, StringArray const& strings); }; +} + #endif diff --git a/src/ripple/validators/impl/ValidatorSourceURL.cpp b/src/ripple/validators/impl/SourceURL.cpp similarity index 58% rename from src/ripple/validators/impl/ValidatorSourceURL.cpp rename to src/ripple/validators/impl/SourceURL.cpp index b4948286b..d3026fe10 100644 --- a/src/ripple/validators/impl/ValidatorSourceURL.cpp +++ b/src/ripple/validators/impl/SourceURL.cpp @@ -4,30 +4,44 @@ */ //============================================================================== -class ValidatorSourceURLImp : public ValidatorSourceURL +namespace Validators +{ + +class SourceURLImp : public SourceURL { public: - explicit ValidatorSourceURLImp (UniformResourceLocator const& url) + explicit SourceURLImp (UniformResourceLocator const& url) : m_url (url) { } - ~ValidatorSourceURLImp () + ~SourceURLImp () { } - Result fetch (CancelCallback&) + String name () + { + return "URL: '" + m_url.full() + "'"; + } + + Result fetch (CancelCallback&, Journal journal) { Result result; ScopedPointer client (HTTPClientBase::New ()); HTTPClientBase::Result httpResult (client->get (m_url)); - + if (httpResult.error == 0) { //Logger::outputDebugString (httpResult.response->toString ()); } + else + { + journal.error() << + "HTTP GET to " << m_url.full().toStdString() << + " failed: '" << httpResult.error.message () << "'"; + } return result; } @@ -38,11 +52,13 @@ private: //------------------------------------------------------------------------------ -ValidatorSourceURL* ValidatorSourceURL::New ( +SourceURL* SourceURL::New ( UniformResourceLocator const& url) { - ScopedPointer object ( - new ValidatorSourceURLImp (url)); + ScopedPointer object ( + new SourceURLImp (url)); return object.release (); } + +} diff --git a/src/ripple/validators/impl/ValidatorSourceURL.h b/src/ripple/validators/impl/SourceURL.h similarity index 53% rename from src/ripple/validators/impl/ValidatorSourceURL.h rename to src/ripple/validators/impl/SourceURL.h index df3c55fa6..9015e003f 100644 --- a/src/ripple/validators/impl/ValidatorSourceURL.h +++ b/src/ripple/validators/impl/SourceURL.h @@ -4,15 +4,20 @@ */ //============================================================================== -#ifndef RIPPLE_CORE_VALIDATOR_VALIDATORSOURCETRUSTEDURL_H_INCLUDED -#define RIPPLE_CORE_VALIDATOR_VALIDATORSOURCETRUSTEDURL_H_INCLUDED +#ifndef RIPPLE_VALIDATORS_SOURCEURL_H_INCLUDED +#define RIPPLE_VALIDATORS_SOURCEURL_H_INCLUDED + +namespace Validators +{ /** Provides validators from a trusted URI (e.g. HTTPS) */ -class ValidatorSourceURL : public Validators::Source +class SourceURL : public Source { public: - static ValidatorSourceURL* New (UniformResourceLocator const& url); + static SourceURL* New (UniformResourceLocator const& url); }; +} + #endif diff --git a/src/ripple/validators/impl/Store.h b/src/ripple/validators/impl/Store.h new file mode 100644 index 000000000..96a88775a --- /dev/null +++ b/src/ripple/validators/impl/Store.h @@ -0,0 +1,24 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +#ifndef RIPPLE_VALIDATORS_VALIDATORSSTORE_H_INCLUDED +#define RIPPLE_VALIDATORS_VALIDATORSSTORE_H_INCLUDED + +namespace Validators +{ + +/** Database persistence for Validators. */ +class Store +{ +public: + virtual ~Store () { } + + +}; + +} + +#endif diff --git a/src/ripple/validators/impl/Tests.cpp b/src/ripple/validators/impl/Tests.cpp new file mode 100644 index 000000000..2ff908618 --- /dev/null +++ b/src/ripple/validators/impl/Tests.cpp @@ -0,0 +1,163 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +namespace Validators +{ + +class Tests : public UnitTest +{ +public: + enum + { + numberOfTestValidators = 1000 + }; + + //-------------------------------------------------------------------------- + + struct Payload + { + Payload () + { + } + }; + + template + class PeerLogic : public TestOverlay::PeerLogicBase + { + public: + typedef TestOverlay::PeerLogicBase Base; + typedef typename Config::Payload Payload; + typedef typename Base::Connection Connection; + typedef typename Base::Peer Peer; + typedef typename Base::Message Message; + typedef typename Config::SizeType SizeType; + + explicit PeerLogic (Peer& peer) + : TestOverlay::PeerLogicBase (peer) + { + } + + ~PeerLogic () + { + } + + void step () + { + if (this->peer().id () == 1) + { + if (this->peer().network().steps() == 0) + { + this->peer().network().state().increment(); + this->peer().send_all (Payload (1)); + } + } + } + + void receive (Connection const& c, Message const& m) + { + if (this->peer().id () != 1) + { + this->peer().network().state().increment(); + this->peer().send_all_if (Message (m.id(), + m.payload().withHop ()), + typename Connection::IsNotPeer (c.peer())); + } + } + }; + + struct Params : TestOverlay::ConfigType < + Params, + TestOverlay::StateBase, + PeerLogic + > + { + typedef TestOverlay::PremadeInitPolicy <250, 3> InitPolicy; + }; + + typedef Params::Network Network; + + //-------------------------------------------------------------------------- + + struct TestSource : Source + { + TestSource (String const& name, uint32 start, uint32 end) + : m_name (name) + , m_start (start) + , m_end (end) + { + } + + String name () + { + return "Test"; + } + + Result fetch (CancelCallback& cancel, Journal) + { + Result result; + + result.success = true; + result.message = String::empty; + result.list.ensureStorageAllocated (numberOfTestValidators); + + for (uint32 i = m_start ; i < m_end; ++i) + { + Info info; + info.key = Validators::PublicKey::createFromInteger (i); + result.list.add (info); + } + + return result; + } + + String m_name; + std::size_t m_start; + std::size_t m_end; + }; + + //-------------------------------------------------------------------------- + + void addSources (Logic& logic) + { +#if 0 + logic.addSource (new TestSource ("source 1", 0, 1000)); + logic.addSource (new TestSource ("source 2", 200, 1500)); + logic.addSource (new TestSource ("source 3", 500, 2000)); + logic.addSource (new TestSource ("source 4", 750, 2200)); + logic.addSource (new TestSource ("source 5", 1500, 3200)); +#else + logic.addSource (new TestSource ("source 1", 0, 1)); +#endif + } + + void testLogic () + { + beginTestCase ("logic"); + + Logic logic; + addSources (logic); + + NoOpCancelCallback cancelCallback; + logic.checkSources (cancelCallback); + + ChosenList::Ptr list (logic.getChosenList ()); + + pass (); + } + + void runTest () + { + testLogic (); + } + + Tests () : UnitTest ("Validators", "ripple", runManual) + { + } +}; + +static Tests tests; + +} diff --git a/src/ripple/validators/impl/Utilities.cpp b/src/ripple/validators/impl/Utilities.cpp new file mode 100644 index 000000000..f54705e2f --- /dev/null +++ b/src/ripple/validators/impl/Utilities.cpp @@ -0,0 +1,173 @@ +//------------------------------------------------------------------------------ +/* + Copyright (c) 2011-2013, OpenCoin, Inc. +*/ +//============================================================================== + +namespace Validators +{ + +struct Utilities::Helpers +{ + // Matches a validator info line. + // + static boost::regex& reInfo () + { + // e.g. + // + // n9KorY8QtTdRx7TVDpwnG9NvyxsDwHUKUEeDLY3AkiGncVaSXZi5 Comment Text + // + static boost::regex re ( + "^" // start of line + "(?:\\h*)" // horiz-white (optional) + "([^\\h\\v]+)" // [1] non-white run + "(?:\\h*)" // horiz-white (optional) + "([^\\h\\v]*)" // [2] any text (optional) + "$" // end of line + + , boost::regex::perl | + boost::regex_constants::match_flags::match_not_dot_null + ); + + return re; + } + + // Matches a comment or whitespace line. + // + static boost::regex& reComment () + { + // e.g. + // + // n9KorY8QtTdRx7TVDpwnG9NvyxsDwHUKUEeDLY3AkiGncVaSXZi5 Comment Text + // + static boost::regex re ( + "^" // start of line + "(?:\\h*)" // horiz-white (optional) + "(?:#.*)" // # then any text + "|" // - or - + "(?:\\h*)" // horiz-white + "$" // end of line + + , boost::regex::perl | + boost::regex_constants::match_flags::match_not_dot_null + ); + + return re; + } +}; + +//------------------------------------------------------------------------------ + +bool Utilities::parseInfoLine ( + Source::Info& info, + std::string const& line, + Journal journal) +{ + bool success (false); + + boost::smatch match; + + if (boost::regex_match (line, match, Helpers::reInfo ())) + { + std::string const encodedKey (match [1]); + std::string const commentText (match [2]); + + RippleAddress deprecatedPublicKey; + + // VFALCO NOTE These bool return values are poorlydocumented + // + if (deprecatedPublicKey.setSeedGeneric (encodedKey)) + { + // expected a domain or public key but got a generic seed? + // log? + } + else if (deprecatedPublicKey.setNodePublic (encodedKey)) + { + // We got a public key. + RipplePublicKey publicKey (deprecatedPublicKey.toRipplePublicKey ()); + success = true; + } + else + { + // Some other junk. + // log? + } + } + else if (boost::regex_match (line, match, Helpers::reComment ())) + { + // it's a comment + } + else + { + // Log a warning about a parsing error + } + +#if 0 + static boost::regex reReferral ("\\`\\s*(\\S+)(?:\\s+(.+))?\\s*\\'"); + + if (!boost::regex_match (strReferral, smMatch, reReferral)) + { + WriteLog (lsWARNING, UniqueNodeList) << str (boost::format ("Bad validator: syntax error: %s: %s") % strSite % strReferral); + } + else + { + std::string strRefered = smMatch[1]; + std::string strComment = smMatch[2]; + RippleAddress naValidator; + + if (naValidator.setSeedGeneric (strRefered)) + { + + WriteLog (lsWARNING, UniqueNodeList) << str (boost::format ("Bad validator: domain or public key required: %s %s") % strRefered % strComment); + } + else if (naValidator.setNodePublic (strRefered)) + { + // A public key. + // XXX Schedule for CAS lookup. + nodeAddPublic (naValidator, vsWhy, strComment); + + WriteLog (lsINFO, UniqueNodeList) << str (boost::format ("Node Public: %s %s") % strRefered % strComment); + + if (naNodePublic.isValid ()) + vstrValues.push_back (str (boost::format ("('%s',%d,'%s')") % strNodePublic % iValues % naValidator.humanNodePublic ())); + + iValues++; + } + else + { + // A domain: need to look it up. + nodeAddDomain (strRefered, vsWhy, strComment); + + WriteLog (lsINFO, UniqueNodeList) << str (boost::format ("Node Domain: %s %s") % strRefered % strComment); + + if (naNodePublic.isValid ()) + vstrValues.push_back (str (boost::format ("('%s',%d,%s)") % strNodePublic % iValues % sqlEscape (strRefered))); + + iValues++; + } + } +#endif + + return success; +} + +//------------------------------------------------------------------------------ + +void Utilities::parseResultLine ( + Source::Result& result, + std::string const& line, + Journal journal) +{ + bool success = false; + + if (! success) + { + Source::Info info; + + success = parseInfoLine (info, line, journal); + if (success) + result.list.add (info); + } +} + +} diff --git a/src/ripple/validators/impl/ValidatorsUtilities.h b/src/ripple/validators/impl/Utilities.h similarity index 79% rename from src/ripple/validators/impl/ValidatorsUtilities.h rename to src/ripple/validators/impl/Utilities.h index e4af59206..522dbce75 100644 --- a/src/ripple/validators/impl/ValidatorsUtilities.h +++ b/src/ripple/validators/impl/Utilities.h @@ -4,12 +4,15 @@ */ //============================================================================== -#ifndef RIPPLE_CORE_VALIDATORSUTILITIES_H_INCLUDED -#define RIPPLE_CORE_VALIDATORSUTILITIES_H_INCLUDED +#ifndef RIPPLE_VALIDATORS_UTILITIES_H_INCLUDED +#define RIPPLE_VALIDATORS_UTILITIES_H_INCLUDED + +namespace Validators +{ /** Common code for Validators classes. */ -class ValidatorsUtilities +class Utilities { public: typedef std::vector Strings; @@ -39,14 +42,20 @@ public: Metadata lines will update the corresponding Result fields. */ static void parseResultLine ( - Validators::Source::Result& result, - String line); + Source::Result& result, + std::string const& line, + Journal journal = Journal()); private: + struct Helpers; + /** Parse a string into a Source::Info. @return `true` on success. */ - static bool parseInfoLine (Validators::Source::Info& info, String line); + static bool parseInfoLine ( + Source::Info& info, std::string const& line, Journal journal); }; +} + #endif diff --git a/src/ripple/validators/impl/Validators.cpp b/src/ripple/validators/impl/Validators.cpp deleted file mode 100644 index ce6a9c9af..000000000 --- a/src/ripple/validators/impl/Validators.cpp +++ /dev/null @@ -1,733 +0,0 @@ -//------------------------------------------------------------------------------ -/* - Copyright (c) 2011-2013, OpenCoin, Inc. -*/ -//============================================================================== - -/* - -Information to track: - -- Percentage of validations that the validator has signed -- Number of validations the validator signed that never got accepted - - -- Target number for Chosen -- Pseudo-randomly choose a subset from Chosen - - - - - -Goal: - - Provide the listener with a ValidatorList. - - This forms the UNL - -Task: - - fetch ValidatorInfo array from a source - - - We have the old one and the new one, compute the following: - - * unchanged validators list - * new validators list - * removed validators list - - - From the unchanged / new / removed, figure out what to do. - -Two important questions: - -- Are there any validators in my ChosenValidators that I dont want - * For example, they have dropped off all the trusted lists - -- Do I have enough? - --------------------------------------------------------------------------------- -ChosenValidators --------------------------------------------------------------------------------- - -David: - Maybe OC should have a URL that you can query to get the latest list of URI's - for OC-approved organzations that publish lists of validators. The server and - client can ship with that master trust URL and also the list of URI's at the - time it's released, in case for some reason it can't pull from OC. That would - make the default installation safe even against major changes in the - organizations that publish validator lists. - - The difference is that if an organization that provides lists of validators - goes rogue, administrators don't have to act. - -TODO: - Write up from end-user perspective on the deployment and administration - of this feature, on the wiki. "DRAFT" or "PROPOSE" to mark it as provisional. - Template: https://ripple.com/wiki/Federation_protocol - - What to do if you're a publisher of ValidatorList - - What to do if you're a rippled administrator - - Overview of how ChosenValidators works - -Goals: - Make default configuration of rippled secure. - * Ship with TrustedUriList - * Also have a preset RankedValidators - Eliminate administrative burden of maintaining - Produce the ChosenValidators list. - Allow quantitative analysis of network health. - -What determines that a validator is good? - - Are they present (i.e. sending validations) - - Are they on the consensus ledger - - What percentage of consensus rounds do they participate in - - Are they stalling consensus - * Measurements of constructive/destructive behavior is - calculated in units of percentage of ledgers for which - the behavior is measured. -*/ - -//------------------------------------------------------------------------------ - -Validators::Source::Result::Result () - : success (false) - , message ("uninitialized") -{ -} - -void Validators::Source::Result::swapWith (Result& other) -{ - std::swap (success, other.success); - std::swap (message, other.message); - list.swapWith (other.list); -} - -//------------------------------------------------------------------------------ - -class ValidatorsImp - : public Validators - , private ThreadWithCallQueue::EntryPoints - , private DeadlineTimer::Listener - , LeakChecked -{ -public: - // Tunable constants - enum - { - // We will fetch a source at this interval - hoursBetweenFetches = 24 - - ,secondsBetweenFetches = hoursBetweenFetches * 60 * 60 - - // Wake up every hour to check source times - ,secondsPerUpdate = 60 * 60 - - // This tunes the preallocated arrays - ,expectedNumberOfResults = 1000 - }; - - //-------------------------------------------------------------------------- - - // Dummy CancelCallback that does nothing - // - struct NoOpCancelCallback : Source::CancelCallback - { - bool shouldCancel () - { - return false; - } - - }; - - //-------------------------------------------------------------------------- - - /** Receive event notifications on Validators operations. - */ - class Listener - { - public: - }; - - //-------------------------------------------------------------------------- - - class ChosenList : public SharedObject - { - public: - typedef SharedPtr Ptr; - - struct Info - { - Info () - { - } - }; - - //typedef HashMap MapType; - typedef boost::unordered_map MapType; - - ChosenList (std::size_t expectedSize = 0) - { - // Available only in recent boost versions? - //m_map.reserve (expectedSize); - } - - std::size_t size () const noexcept - { - return m_map.size (); - } - - void insert (KeyType const& key, Info const& info) noexcept - { - m_map [key] = info; - } - - bool contains (KeyType const& key) const noexcept - { - return m_map.find (key) != m_map.cend (); - } - - private: - MapType m_map; - }; - - //-------------------------------------------------------------------------- - - // Encapsulates the logic for creating the chosen validators. - // This is a separate class to facilitate the unit tests. - // - class Logic - { - public: - // Information associated with each Source - // - struct SourceDesc - { - enum - { - keysPreallocationSize = 1000 - }; - - enum Status - { - statusNone, - statusFetched, - statusFailed - }; - - ScopedPointer source; - Status status; - Time whenToFetch; - int numberOfFailures; - - // The result of the last fetch - Source::Result result; - - //------------------------------------------------------------------ - - SourceDesc () noexcept - : status (statusNone) - , whenToFetch (Time::getCurrentTime ()) - , numberOfFailures (0) - { - } - - ~SourceDesc () - { - } - }; - - typedef DynamicList SourcesType; - - //---------------------------------------------------------------------- - - // Information associated with each distinguishable validator - // - struct ValidatorInfo - { - ValidatorInfo () - : refCount (0) - { - } - - int refCount; - }; - - //typedef HashMap MapType; - typedef boost::unordered_map MapType; - - //---------------------------------------------------------------------- - - Logic () - : m_chosenListNeedsUpdate (false) - { - } - - // Add a one-time static source. - // Fetch is called right away, this call blocks. - // - void addStaticSource (Source* source) - { - ScopedPointer object (source); - - NoOpCancelCallback cancelCallback; - - Source::Result result (object->fetch (cancelCallback)); - - if (result.success) - { - addSourceInfo (result.list); - } - else - { - // VFALCO NOTE Maybe log the error and message? - } - } - - // Add a live source to the list of sources. - // - void addSource (Source* source) - { - SourceDesc& desc (*m_sources.emplace_back ()); - desc.source = source; - } - - // Called when we receive a validation from a peer. - // - void receiveValidation (Validators::ReceivedValidation const& rv) - { - MapType::iterator iter (m_map.find (rv.signerPublicKeyHash)); - if (iter != m_map.end ()) - { - // Exists - //ValidatorInfo& validatorInfo (iter->value ()); - } - else - { - // New - //ValidatorInfo& validatorInfo (m_map.insert (rv.signerPublicKeyHash)); - } - } - - // Add each entry in the list to the map, incrementing the - // reference count if it already exists, and updating fields. - // - void addSourceInfo (Array const& list) - { - for (std::size_t i = 0; i < list.size (); ++i) - { - Source::Info const& info (list.getReference (i)); - std::pair result ( - m_map.emplace (info.key, ValidatorInfo ())); - ValidatorInfo& validatorInfo (result.first->second); - ++validatorInfo.refCount; - if (result.second) - { - // This is a new one - markDirtyChosenList (); - } - } - } - - // Decrement the reference count of each item in the list - // in the map - // - void removeSourceInfo (Array const& list) - { - for (std::size_t i = 0; i < list.size (); ++i) - { - Source::Info const& info (list.getReference (i)); - MapType::iterator iter (m_map.find (info.key)); - bassert (iter != m_map.end ()); - ValidatorInfo& validatorInfo (iter->second); - if (--validatorInfo.refCount == 0) - { - // Last reference removed - m_map.erase (iter); - markDirtyChosenList (); - } - } - } - - // Fetch one source - // - void fetchSource (SourceDesc& desc, Source::CancelCallback& callback) - { - Source::Result result (desc.source->fetch (callback)); - - if (! callback.shouldCancel ()) - { - // Reset fetch timer for the source. - desc.whenToFetch = Time::getCurrentTime () + - RelativeTime (secondsBetweenFetches); - - if (result.success) - { - // Add the new source info to the map - addSourceInfo (result.list); - - // Swap lists - desc.result.swapWith (result); - - // Remove the old source info from the map - removeSourceInfo (result.list); - - // See if we need to rebuild - checkDirtyChosenList (); - - // Reset failure status - desc.numberOfFailures = 0; - desc.status = SourceDesc::statusFetched; - } - else - { - ++desc.numberOfFailures; - desc.status = SourceDesc::statusFailed; - } - } - } - - // Check each source to see if it needs fetching. - // - void checkSources (Source::CancelCallback& callback) - { - Time const currentTime (Time::getCurrentTime ()); - for (SourcesType::iterator iter = m_sources.begin (); - ! callback.shouldCancel () && iter != m_sources.end (); ++iter) - { - SourceDesc& desc (*iter); - if (desc.whenToFetch <= currentTime) - fetchSource (desc, callback); - } - } - - // Signal that the Chosen List needs to be rebuilt. - // - void markDirtyChosenList () - { - m_chosenListNeedsUpdate = true; - } - - // Check the dirty state of the Chosen List, and rebuild it - // if necessary. - // - void checkDirtyChosenList () - { - if (m_chosenListNeedsUpdate) - { - buildChosenList (); - m_chosenListNeedsUpdate = false; - } - } - - // Rebuilds the Chosen List - // - void buildChosenList () - { - ChosenList::Ptr list (new ChosenList (m_map.size ())); - - for (MapType::iterator iter = m_map.begin (); - iter != m_map.end (); ++iter) - { - ChosenList::Info info; - list->insert (iter->first, info); - } - - // This is thread safe - m_chosenList = list; - } - - // Get a reference to the chosen list. - // This is safe to call from any thread at any time. - // - ChosenList::Ptr getChosenList () - { - return m_chosenList; - } - - //---------------------------------------------------------------------- - // - // Ripple interface - // - // These routines are modeled after UniqueNodeList - - bool isTrustedPublicKeyHash (RipplePublicKeyHash const& key) - { - return m_chosenList->contains (key); - } - - // - // - //---------------------------------------------------------------------- - - private: - SourcesType m_sources; - MapType m_map; - bool m_chosenListNeedsUpdate; - ChosenList::Ptr m_chosenList; - }; - - //-------------------------------------------------------------------------- - -public: - explicit ValidatorsImp (Listener* listener) - : m_listener (listener) - , m_thread ("Validators") - , m_timer (this) - { - m_thread.start (this); - } - - ~ValidatorsImp () - { - } - - void addStrings (std::vector const& strings) - { - StringArray stringArray; - stringArray.ensureStorageAllocated (strings.size()); - for (std::size_t i = 0; i < strings.size(); ++i) - stringArray.add (strings [i]); - addStrings (stringArray); - } - - void addStrings (StringArray const& stringArray) - { - addStaticSource ( - ValidatorSourceStrings::New (stringArray)); - } - - void addFile (File const& file) - { - addStaticSource (ValidatorSourceFile::New (file)); - } - - void addURL (UniformResourceLocator const& url) - { - addSource (ValidatorSourceURL::New (url)); - } - - void addSource (Source* source) - { - m_thread.call (&Logic::addSource, &m_logic, source); - } - - void addStaticSource (Source* source) - { - m_thread.call (&Logic::addStaticSource, &m_logic, source); - } - - void receiveValidation (ReceivedValidation const& rv) - { - m_thread.call (&Logic::receiveValidation, &m_logic, rv); - } - - //-------------------------------------------------------------------------- - - void onDeadlineTimer (DeadlineTimer&) - { - // This will make us fall into the idle proc as needed - // - m_thread.interrupt (); - } - - void threadInit () - { - m_timer.setRecurringExpiration (secondsPerUpdate); - } - - void threadExit () - { - } - - bool threadIdle () - { - bool interrupted = false; - - struct ThreadCancelCallback : Source::CancelCallback, Uncopyable - { - explicit ThreadCancelCallback (ThreadWithCallQueue& thread) - : m_thread (thread) - , m_interrupted (false) - { - } - - bool shouldCancel () - { - if (m_interrupted) - return true; - return m_interrupted = m_thread.interruptionPoint (); - } - - private: - ThreadWithCallQueue& m_thread; - bool m_interrupted; - }; - - ThreadCancelCallback cancelCallback (m_thread); - - m_logic.checkSources (cancelCallback); - - return interrupted; - } - -private: - Logic m_logic; - Listener* const m_listener; - ThreadWithCallQueue m_thread; - DeadlineTimer m_timer; -}; - -//------------------------------------------------------------------------------ - -Validators* Validators::New () -{ - return new ValidatorsImp (nullptr); -} - -//------------------------------------------------------------------------------ - -class ValidatorsTests : public UnitTest -{ -public: - enum - { - numberOfTestValidators = 1000 - }; - - //-------------------------------------------------------------------------- - - struct Payload - { - Payload () - { - } - }; - - template - class PeerLogic : public TestOverlay::PeerLogicBase - { - public: - typedef TestOverlay::PeerLogicBase Base; - typedef typename Config::Payload Payload; - typedef typename Base::Connection Connection; - typedef typename Base::Peer Peer; - typedef typename Base::Message Message; - typedef typename Config::SizeType SizeType; - - explicit PeerLogic (Peer& peer) - : TestOverlay::PeerLogicBase (peer) - { - } - - ~PeerLogic () - { - } - - void step () - { - if (this->peer().id () == 1) - { - if (this->peer().network().steps() == 0) - { - this->peer().network().state().increment(); - this->peer().send_all (Payload (1)); - } - } - } - - void receive (Connection const& c, Message const& m) - { - if (this->peer().id () != 1) - { - this->peer().network().state().increment(); - this->peer().send_all_if (Message (m.id(), - m.payload().withHop ()), - typename Connection::IsNotPeer (c.peer())); - } - } - }; - - struct Params : TestOverlay::ConfigType < - Params, - TestOverlay::StateBase, - PeerLogic - > - { - typedef TestOverlay::PremadeInitPolicy <250, 3> InitPolicy; - }; - - typedef Params::Network Network; - - //-------------------------------------------------------------------------- - - struct TestSource : Validators::Source - { - TestSource (String const& name, uint32 start, uint32 end) - : m_name (name) - , m_start (start) - , m_end (end) - { - } - - Result fetch (CancelCallback& cancel) - { - Result result; - - result.success = true; - result.message = String::empty; - result.list.ensureStorageAllocated (numberOfTestValidators); - - for (uint32 i = m_start ; i < m_end; ++i) - { - Info info; - info.key = Validators::KeyType::createFromInteger (i); - result.list.add (info); - } - - return result; - } - - String m_name; - std::size_t m_start; - std::size_t m_end; - }; - - //-------------------------------------------------------------------------- - - void addSources (ValidatorsImp::Logic& logic) - { -#if 0 - logic.addSource (new TestSource ("source 1", 0, 1000)); - logic.addSource (new TestSource ("source 2", 200, 1500)); - logic.addSource (new TestSource ("source 3", 500, 2000)); - logic.addSource (new TestSource ("source 4", 750, 2200)); - logic.addSource (new TestSource ("source 5", 1500, 3200)); -#else - logic.addSource (new TestSource ("source 1", 0, 1)); -#endif - } - - void testLogic () - { - beginTestCase ("logic"); - - ValidatorsImp::Logic logic; - addSources (logic); - - ValidatorsImp::NoOpCancelCallback cancelCallback; - logic.checkSources (cancelCallback); - - ValidatorsImp::ChosenList::Ptr list (logic.getChosenList ()); - - pass (); - } - - void runTest () - { - testLogic (); - } - - ValidatorsTests () : UnitTest ("Validators", "ripple", runManual) - { - } -}; - -static ValidatorsTests validatorsTests; - diff --git a/src/ripple/validators/impl/ValidatorsUtilities.cpp b/src/ripple/validators/impl/ValidatorsUtilities.cpp deleted file mode 100644 index 38a02a17c..000000000 --- a/src/ripple/validators/impl/ValidatorsUtilities.cpp +++ /dev/null @@ -1,30 +0,0 @@ -//------------------------------------------------------------------------------ -/* - Copyright (c) 2011-2013, OpenCoin, Inc. -*/ -//============================================================================== - -bool ValidatorsUtilities::parseInfoLine (Validators::Source::Info& info, String line) -{ - bool success (false); - - return success; -} - -void ValidatorsUtilities::parseResultLine ( - Validators::Source::Result& result, - String line) -{ - bool success = false; - - if (! success) - { - Validators::Source::Info info; - - success = parseInfoLine (info, line); - if (success) - result.list.add (info); - } - - -} diff --git a/src/ripple/validators/ripple_validators.cpp b/src/ripple/validators/ripple_validators.cpp index cf6180339..f4366eb42 100644 --- a/src/ripple/validators/ripple_validators.cpp +++ b/src/ripple/validators/ripple_validators.cpp @@ -9,24 +9,31 @@ #include "ripple_validators.h" #include "beast/modules/beast_core/system/BeforeBoost.h" // must come first -//#include -//#include -//#include +#include #include +#include "beast/modules/beast_sqdb/beast_sqdb.h" + #include "../testoverlay/ripple_testoverlay.h" // for unit test namespace ripple { -# include "impl/ValidatorsUtilities.h" -#include "impl/ValidatorsUtilities.cpp" -# include "impl/ValidatorSourceFile.h" -# include "impl/ValidatorSourceStrings.h" -# include "impl/ValidatorSourceURL.h" -#include "impl/ValidatorSourceFile.cpp" -#include "impl/ValidatorSourceStrings.cpp" -#include "impl/ValidatorSourceURL.cpp" -#include "impl/Validators.cpp" +# include "impl/CancelCallbacks.h" +# include "impl/ChosenList.h" +# include "impl/SourceFile.h" +# include "impl/SourceStrings.h" +# include "impl/SourceURL.h" +# include "impl/Store.h" +# include "impl/Utilities.h" +#include "impl/Logic.h" + +#include "impl/Manager.cpp" +#include "impl/Source.cpp" +#include "impl/SourceFile.cpp" +#include "impl/SourceStrings.cpp" +#include "impl/SourceURL.cpp" +#include "impl/Tests.cpp" +#include "impl/Utilities.cpp" } diff --git a/src/ripple/validators/ripple_validators.h b/src/ripple/validators/ripple_validators.h index 282e06401..9c322d7a8 100644 --- a/src/ripple/validators/ripple_validators.h +++ b/src/ripple/validators/ripple_validators.h @@ -25,7 +25,9 @@ namespace ripple using namespace beast; -#include "api/Validators.h" +# include "api/Types.h" +# include "api/Source.h" +#include "api/Manager.h" } diff --git a/src/ripple_app/main/ripple_Application.cpp b/src/ripple_app/main/ripple_Application.cpp index 8a69f98b7..e696ac8f1 100644 --- a/src/ripple_app/main/ripple_Application.cpp +++ b/src/ripple_app/main/ripple_Application.cpp @@ -15,7 +15,8 @@ SETUP_LOG (Application) // // Specializations for LogPartition names -template <> char const* LogPartition::getPartitionName () { return "Validators"; } +class ValidatorsLog; +template <> char const* LogPartition::getPartitionName () { return "Validators"; } // //------------------------------------------------------------------------------ @@ -165,7 +166,7 @@ public: getConfig ().nodeDatabase, getConfig ().ephemeralNodeDatabase, *this)) - , m_validators (Validators::New ()) + , m_validators (Validators::Manager::New (LogJournal::get ())) , mFeatures (IFeatures::New (2 * 7 * 24 * 60 * 60, 200)) // two weeks, 200/256 , mFeeVote (IFeeVote::New (10, 50 * SYSTEM_CURRENCY_PARTS, 12.5 * SYSTEM_CURRENCY_PARTS)) , mFeeTrack (ILoadFeeTrack::New ()) @@ -231,7 +232,7 @@ public: { std::vector const& strings (getConfig().validators); if (! strings.empty ()) - m_validators->addStrings (strings); + m_validators->addStrings ("rippled.cfg", strings); } if (! getConfig().getValidatorsURL().empty()) @@ -333,7 +334,7 @@ public: return mSLECache; } - Validators& getValidators () + Validators::Manager& getValidators () { return *m_validators; } @@ -740,7 +741,7 @@ private: ScopedPointer m_wsSSLContext; ScopedPointer m_txQueue; ScopedPointer m_nodeStore; - ScopedPointer m_validators; + ScopedPointer m_validators; ScopedPointer mFeatures; ScopedPointer mFeeVote; ScopedPointer mFeeTrack; diff --git a/src/ripple_app/main/ripple_Application.h b/src/ripple_app/main/ripple_Application.h index 06a8fb54e..2d1e8ca8d 100644 --- a/src/ripple_app/main/ripple_Application.h +++ b/src/ripple_app/main/ripple_Application.h @@ -7,6 +7,8 @@ #ifndef RIPPLE_IAPPLICATION_H #define RIPPLE_IAPPLICATION_H +namespace Validators { class Manager; } + // VFALCO TODO Fix forward declares required for header dependency loops class IFeatures; class IFeeVote; @@ -14,7 +16,6 @@ class IHashRouter; class ILoadFeeTrack; class Peers; class UniqueNodeList; -class Validators; class NodeStore; class JobQueue; @@ -79,7 +80,7 @@ public: virtual NodeCache& getTempNodeCache () = 0; virtual SLECache& getSLECache () = 0; - virtual Validators& getValidators () = 0; + virtual Validators::Manager& getValidators () = 0; virtual IFeatures& getFeatureTable () = 0; virtual IFeeVote& getFeeVote () = 0; virtual IHashRouter& getHashRouter () = 0;