Add ChosenValidators logic

Builds/VisualStudio2012/RippleD.vcxproj

@@ -355,12 +355,36 @@
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
     </ClCompile>
     <ClCompile Include="..\..\modules\ripple_core\ripple_core.cpp" />
+    <ClCompile Include="..\..\modules\ripple_core\validator\ripple_ChosenValidators.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
+    </ClCompile>
+    <ClCompile Include="..\..\modules\ripple_core\validator\ripple_StringsValidatorSource.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
+    </ClCompile>
+    <ClCompile Include="..\..\modules\ripple_core\validator\ripple_TrustedUriValidatorSource.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
+    </ClCompile>
     <ClCompile Include="..\..\modules\ripple_core\validator\ripple_Validator.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
     </ClCompile>
+    <ClCompile Include="..\..\modules\ripple_core\validator\ripple_ValidatorList.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
+    </ClCompile>
     <ClCompile Include="..\..\modules\ripple_core\validator\ripple_Validators.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
@@ -1428,7 +1452,11 @@
     <ClInclude Include="..\..\modules\ripple_core\functional\ripple_LoadFeeTrack.h" />
     <ClInclude Include="..\..\modules\ripple_core\functional\ripple_LoadMonitor.h" />
     <ClInclude Include="..\..\modules\ripple_core\ripple_core.h" />
+    <ClInclude Include="..\..\modules\ripple_core\validator\ripple_ChosenValidators.h" />
+    <ClInclude Include="..\..\modules\ripple_core\validator\ripple_StringsValidatorSource.h" />
+    <ClInclude Include="..\..\modules\ripple_core\validator\ripple_TrustedUriValidatorSource.h" />
     <ClInclude Include="..\..\modules\ripple_core\validator\ripple_Validator.h" />
+    <ClInclude Include="..\..\modules\ripple_core\validator\ripple_ValidatorList.h" />
     <ClInclude Include="..\..\modules\ripple_core\validator\ripple_Validators.h" />
     <ClInclude Include="..\..\modules\ripple_data\crypto\ripple_Base58.h" />
     <ClInclude Include="..\..\modules\ripple_data\crypto\ripple_Base58Data.h" />

Builds/VisualStudio2012/RippleD.vcxproj.filters

@@ -885,6 +885,18 @@
     <ClCompile Include="..\..\modules\ripple_app\ledger\SerializedValidation.cpp">
       <Filter>[1] Ripple\ripple_app\ledger</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\modules\ripple_core\validator\ripple_ValidatorList.cpp">
+      <Filter>[1] Ripple\ripple_core\validator</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\modules\ripple_core\validator\ripple_ChosenValidators.cpp">
+      <Filter>[1] Ripple\ripple_core\validator</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\modules\ripple_core\validator\ripple_TrustedUriValidatorSource.cpp">
+      <Filter>[1] Ripple\ripple_core\validator</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\modules\ripple_core\validator\ripple_StringsValidatorSource.cpp">
+      <Filter>[1] Ripple\ripple_core\validator</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\Subtrees\sqlite\sqlite3.h">
@@ -1647,6 +1659,18 @@
     <ClInclude Include="..\..\src\cpp\ripple\ripple_Application.h">
       <Filter>[1] Ripple\ripple_app\_main</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\modules\ripple_core\validator\ripple_ValidatorList.h">
+      <Filter>[1] Ripple\ripple_core\validator</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\modules\ripple_core\validator\ripple_ChosenValidators.h">
+      <Filter>[1] Ripple\ripple_core\validator</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\modules\ripple_core\validator\ripple_TrustedUriValidatorSource.h">
+      <Filter>[1] Ripple\ripple_core\validator</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\modules\ripple_core\validator\ripple_StringsValidatorSource.h">
+      <Filter>[1] Ripple\ripple_core\validator</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <CustomBuild Include="..\..\src\cpp\ripple\ripple.proto" />

TODO.txt

@@ -2,6 +2,14 @@
 RIPPLE TODO
 --------------------------------------------------------------------------------
 
+- Unit Test attention
+
+- NodeStore backend unit test
+
+- Validations unit test
+
+--------------------------------------------------------------------------------
+
 - Rename RPCHandler to CallHandler
 
 - Move everything in src/cpp/ripple into ripple_app and sort them into
@@ -227,7 +235,7 @@ Interfaces
 boost
 
     Unclear from the class declaration what style of shared object management
-    is used. Prefer to derive from a ReferenceCountedObject class so that the
+    is used. Prefer to derive from a SharedObject class so that the
     behavior is explicit. Furthermore the use of intrusive containers is
     preferred over the alternative.
 
@@ -302,6 +310,17 @@ What we want from the unique node list:
 
 
 --------------------------------------------------------------------------------
+ChosenValidators
+--------------------------------------------------------------------------------
+
+TODO:
+  Write up from end-user perspective on the deployment and administration
+  of this feature, on the wiki. "DRAFT" or "PROPOSE" to mark it as provisional.
+  Template: https://ripple.com/wiki/Federation_protocol
+  - What to do if you're a publisher of ValidatorList
+  - What to do if you're a rippled administrator
+  - Overview of how ChosenValidators works
+
 
 Goals:
   Make default configuration of rippled secure.

modules/ripple_core/ripple_core.cpp

@@ -31,8 +31,12 @@ namespace ripple
 #include "functional/ripple_LoadEvent.cpp"
 #include "functional/ripple_LoadMonitor.cpp"
 
+#include "validator/ripple_ChosenValidators.cpp"
 #include "validator/ripple_Validator.cpp"
+#include "validator/ripple_ValidatorList.cpp"
 #include "validator/ripple_Validators.cpp"
+#include "validator/ripple_StringsValidatorSource.cpp"
+#include "validator/ripple_TrustedUriValidatorSource.cpp"
 
 }
 

modules/ripple_core/ripple_core.h

@@ -38,7 +38,11 @@ namespace ripple
 /**/#include "functional/ripple_JobQueue.h"
 
 #include "validator/ripple_Validator.h"
+#include "validator/ripple_ValidatorList.h"
+#include "validator/ripple_ChosenValidators.h"
 #include "validator/ripple_Validators.h"
+#include "validator/ripple_StringsValidatorSource.h"
+#include "validator/ripple_TrustedUriValidatorSource.h"
 
 }
 

modules/ripple_core/validator/ripple_ChosenValidators.cpp

@@ -0,0 +1,27 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+ChosenValidators::ChosenValidators (ValidatorList::Ptr chosenValidators,
+                                    ValidatorList::Ptr wellKnownValidators)
+    : m_list (chosenValidators)
+    , m_wellKnownList (wellKnownValidators)
+{
+}
+
+ChosenValidators::~ChosenValidators ()
+{
+}
+
+ValidatorList::Ptr ChosenValidators::getList ()
+{
+    return m_list;
+}
+
+ValidatorList::Ptr ChosenValidators::getWellKnownList ()
+{
+    return m_wellKnownList;
+}
+

modules/ripple_core/validator/ripple_ChosenValidators.h

@@ -0,0 +1,41 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_CHOSENVALIDATORS_H_INCLUDED
+#define RIPPLE_CHOSENVALIDATORS_H_INCLUDED
+
+/** A subset of validators chosen from a list of well known validators.
+
+    @see Validators
+*/
+class ChosenValidators : public SharedObject
+{
+public:
+    typedef SharedObjectPtr <ChosenValidators> Ptr;
+
+    ChosenValidators (ValidatorList::Ptr chosenValidators,
+                      ValidatorList::Ptr wellKnownValidators);
+
+    ~ChosenValidators ();
+
+    /** Retrieve the list of chosen validators.
+
+        This is the subset of validators that we care about.
+    */
+    ValidatorList::Ptr getList ();
+
+    /** Retrieve the list of well known validators.
+
+        This is the set from which the chosen validators were chosen.
+    */
+    ValidatorList::Ptr getWellKnownList ();
+
+private:
+    ValidatorList::Ptr m_list;
+    ValidatorList::Ptr m_wellKnownList;
+};
+
+#endif

modules/ripple_core/validator/ripple_StringsValidatorSource.cpp

@@ -0,0 +1,31 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+class StringsValidatorSourceImp : public StringsValidatorSource
+{
+public:
+    StringsValidatorSourceImp (StringArray const& strings)
+    {
+    }
+
+    ~StringsValidatorSourceImp ()
+    {
+    }
+
+    ValidatorList::Ptr fetch ()
+    {
+        ValidatorList::Ptr list = new ValidatorList;
+
+        return list;
+    }
+
+private:
+};
+
+StringsValidatorSource* StringsValidatorSource::New (StringArray const& strings)
+{
+    return new StringsValidatorSourceImp (strings);
+}

modules/ripple_core/validator/ripple_StringsValidatorSource.h

@@ -0,0 +1,20 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_STRINGSVALIDATORSOURCE_H_INCLUDED
+#define RIPPLE_STRINGSVALIDATORSOURCE_H_INCLUDED
+
+/** Provides validators from a set of Validator strings.
+
+    Typically this will come from a local configuration file.
+*/
+class StringsValidatorSource : public Validators::Source
+{
+public:
+    static StringsValidatorSource* New (StringArray const& strings);
+};
+
+#endif

modules/ripple_core/validator/ripple_TrustedUriValidatorSource.cpp

@@ -0,0 +1,33 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+class TrustedUriValidatorSourceImp : public TrustedUriValidatorSource
+{
+public:
+    TrustedUriValidatorSourceImp (String const url)
+        : m_url (url)
+    {
+    }
+
+    ~TrustedUriValidatorSourceImp ()
+    {
+    }
+
+    ValidatorList::Ptr fetch ()
+    {
+        ValidatorList::Ptr list = new ValidatorList;
+
+        return list;
+    }
+
+private:
+    String const m_url;
+};
+
+TrustedUriValidatorSource* TrustedUriValidatorSource::New (String url)
+{
+    return new TrustedUriValidatorSourceImp (url);
+}

modules/ripple_core/validator/ripple_TrustedUriValidatorSource.h

@@ -0,0 +1,18 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_TRUSTEDURIVALIDATORSOURCE_H_INCLUDED
+#define RIPPLE_TRUSTEDURIVALIDATORSOURCE_H_INCLUDED
+
+/** Provides validators from a trusted URI (e.g. HTTPS)
+*/
+class TrustedUriValidatorSource : public Validators::Source
+{
+public:
+    static TrustedUriValidatorSource* New (String url);
+};
+
+#endif

modules/ripple_core/validator/ripple_Validator.cpp

@@ -4,6 +4,7 @@
 */
 //==============================================================================
 
-Validator::Validator ()
+Validator::Validator (PublicKey const& publicKey)
+    : m_publicKey (publicKey)
 {
 }

modules/ripple_core/validator/ripple_Validator.h

@@ -16,10 +16,18 @@ class Validator
 public:
     typedef RippleAddress PublicKey;
 
-    Validator ();
+    explicit Validator (PublicKey const& publicKey);
+
+    //Validator (Validator const&);
+
+    PublicKey const& getPublicKey () const { return m_publicKey; }
+
+    // not thread safe
+    void incrementWeight () { ++m_weight; }
 
 private:
     PublicKey m_publicKey;
+    int m_weight;
 };
 
 #endif

modules/ripple_core/validator/ripple_ValidatorList.cpp

@@ -0,0 +1,35 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+ValidatorList::ValidatorList ()
+    : m_isSigned (false)
+{
+}
+
+ValidatorList::~ValidatorList ()
+{
+}
+
+int ValidatorList::size () const
+{
+    return m_list.size ();
+}
+
+Validator& ValidatorList::operator [] (int index)
+{
+    return m_list.getReference (index);
+}
+
+bool ValidatorList::isSigned () const
+{
+    return m_isSigned;
+}
+
+void ValidatorList::add (Validator const& validator)
+{
+    m_list.add (validator);
+}
+

modules/ripple_core/validator/ripple_ValidatorList.h

@@ -0,0 +1,44 @@
+//------------------------------------------------------------------------------
+/*
+    Copyright (c) 2011-2013, OpenCoin, Inc.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_VALIDATORLIST_H_INCLUDED
+#define RIPPLE_VALIDATORLIST_H_INCLUDED
+
+/** A list of Validator that comes from a source of validators.
+
+    Sources include trusted URIs, or a local file.
+
+    The list may be signed.
+*/
+class ValidatorList : public SharedObject
+{
+public:
+    typedef SharedObjectPtr <ValidatorList> Ptr;
+
+    /** Create an empty list.
+    */
+    ValidatorList ();
+
+    ~ValidatorList ();
+
+    /** Retrieve the number of items.
+    */
+    int size () const;
+
+    Validator& operator[] (int index);
+
+    bool isSigned () const;
+
+    /** Add a validator to the list.
+    */
+    void add (Validator const& validator);
+
+private:
+    bool m_isSigned;
+    Array <Validator> m_list;
+};
+
+#endif

modules/ripple_core/validator/ripple_Validators.cpp

@@ -4,40 +4,214 @@
 */
 //==============================================================================
 
+/*
+
+Two important questions:
+
+- Are there any validators in my ChosenValidators that I dont want
+  * For example, they have dropped off all the trusted lists
+
+- Do I have enough?
+
+*/
+
+
 class ValidatorsImp
     : public Validators
-    , private InterruptibleThread::EntryPoint
+    , private ThreadWithCallQueue::EntryPoints
+    , private DeadlineTimer::Listener
 {
 public:
-    explicit ValidatorsImp (Listener* listener)
+    // Tunable constants
+    enum
+    {
+        hoursBetweenFetches = 24,
+
+        secondsBetweenFetches = hoursBetweenFetches * 60 * 60,
+
+        timerGranularity = 60 * 60  // Wake up every hour
+    };
+
+    //--------------------------------------------------------------------------
+
+    struct SourceInfo
+    {
+        enum Status
+        {
+            statusNone,
+            statusFetched,
+            statusFailed,
+        };
+
+        explicit SourceInfo (Source* source_)
+            : source (source_)
+            , status (statusNone)
+        {
+        }
+
+        Source* source;
+        Status status;
+        Time lastFetch;
+        ValidatorList::Ptr list;
+    };
+
+    //--------------------------------------------------------------------------
+
+public:
+    explicit ValidatorsImp (Validators::Listener* listener)
         : m_thread ("Validators")
+        , m_timer (this)
         , m_listener (listener)
     {
+        m_thread.start (this);
     }
 
     ~ValidatorsImp ()
     {
     }
 
-    void addTrustedUri (String uri)
+    void addSource (Source* source)
     {
-        m_trustedUris.add (uri);
+        m_thread.call (&ValidatorsImp::doAddSource, this, source);
     }
 
-    void start ()
+    void doAddSource (Source* source)
     {
-        m_thread.start (this);
+        m_sources.add (source);
+
+        m_info.add (SourceInfo (source));
     }
 
-    void threadRun ()
+    void onDeadlineTimer (DeadlineTimer&)
     {
-        // process the trustedUri list and blah blah
+        // This will make us fall into the idle proc as needed
+        //
+        m_thread.interrupt ();
+    }
+
+    void mergeValidators (ValidatorList::Ptr dest, ValidatorList::Ptr source)
+    {
+    }
+
+    // Construct the list of well known validators
+    ValidatorList::Ptr buildWellKnownValidators ()
+    {
+        ValidatorList::Ptr list = new ValidatorList;
+
+        // Go through each source and merge its list
+        for (int i = 0; i < m_info.size (); ++i)
+        {
+            SourceInfo const& info (m_info.getReference (i));
+
+            if (info.status == SourceInfo::statusFetched)
+            {
+                mergeValidators (list, info.list);
+            }
+        }
+
+        return list;
+    }
+
+    // Choose a subset of validators from the well known list
+    //
+    ValidatorList::Ptr chooseSubset (ValidatorList::Ptr list)
+    {
+        ValidatorList::Ptr result = new ValidatorList;
+
+        return result;
+    }
+
+    // Create a composite object representing the chosen validators.
+    //
+    ChosenValidators::Ptr createChosenValidators ()
+    {
+        ValidatorList::Ptr wellKnownValidators = buildWellKnownValidators ();
+
+        ValidatorList::Ptr validatorSubset = chooseSubset (wellKnownValidators);
+
+        ChosenValidators::Ptr chosenValidators = new ChosenValidators (
+            validatorSubset,
+            wellKnownValidators);
+
+        return chosenValidators;
+    }
+
+    // Create a fresh chosen validators from our source information
+    // and broadcast it.
+    //
+    void updateChosenValidators ()
+    {
+        ChosenValidators::Ptr chosenValidators = createChosenValidators ();
+
+        m_listener->onValidatorsChosen (chosenValidators);
+    }
+
+    // Goes through all the sources and refreshes them as needed
+    //
+    bool scanSources ()
+    {
+        bool interrupted = false;
+
+        Time currentTime = Time::getCurrentTime ();
+
+        // Find a source that needs to be processed
+        //
+        for (int i = 0; i < m_info.size (); ++i)
+        {
+            SourceInfo& info (m_info.getReference (i));
+
+            // See if we need to refresh its list
+            //
+            if ((currentTime - info.lastFetch).inSeconds () > secondsBetweenFetches)
+            {
+                ValidatorList::Ptr list = info.source->fetch ();
+
+                currentTime = Time::getCurrentTime ();
+
+                if (list != nullptr)
+                {
+                    info.status = SourceInfo::statusFetched;
+                    info.lastFetch = currentTime;
+                    info.list = list;
+
+                    updateChosenValidators ();
+                }
+            }
+
+            interrupted = m_thread.interruptionPoint ();
+
+            if (interrupted)
+                break;
+        }
+
+        return interrupted;
+    }
+
+    void threadInit ()
+    {
+        m_timer.setRecurringExpiration (timerGranularity);
+    }
+
+    void threadExit ()
+    {
+    }
+
+    bool threadIdle ()
+    {
+        bool interrupted = false;
+
+        interrupted = scanSources ();
+
+        return interrupted;
     }
 
 private:
-    InterruptibleThread m_thread;
-    Listener* const m_listener;
-    StringArray m_trustedUris;
+    ThreadWithCallQueue m_thread;
+    DeadlineTimer m_timer;
+    Validators::Listener* const m_listener;
+    OwnedArray <Source> m_sources;
+    Array <SourceInfo> m_info;
+    ValidatorList::Ptr m_chosenValidators;
 };
 
 Validators* Validators::New (Listener* listener)

modules/ripple_core/validator/ripple_Validators.h

@@ -17,20 +17,51 @@
 class Validators : Uncopyable
 {
 public:
+    /** Provides a ValidatorList.
+    */
+    class Source
+    {
+    public:
+        /** Destroy the source.
+
+            If a fetch is active, it will be aborted before the
+            destructor returns.
+        */
+        virtual ~Source () { }
+
+        /** Fetch the validator list from this source.
+
+            This call blocks.
+        */
+        virtual ValidatorList::Ptr fetch () = 0;
+    };
+
+public:
+    /** Receive event notifications on Validators operations.
+    */
     class Listener
     {
     public:
-        //virtual void onValidatorsChosen (ValidatorList validators) { }
+        virtual void onValidatorsChosen (ChosenValidators::Ptr list) { }
     };
 
 public:
+    /** Create a new Validators object.
+    */
     static Validators* New (Listener* listener);
 
+    /** Destroy the object.
+
+        Any pending source fetch operations are aborted.
+
+        There may be some listener calls made before the
+        destructor returns.
+    */
     virtual ~Validators () { }
 
-    virtual void addTrustedUri (String uri) = 0;
-
-    virtual void start () = 0;
+    /** Add a source of validators.
+    */
+    virtual void addSource (Source* source) = 0;
 };
 
 #endif