Alphabet class for base58 conversions, Validators work

src/ripple/types/api/Base58.h

@@ -43,18 +43,49 @@ namespace ripple {
 class Base58
 {
 public:
-    static char const* getBitcoinAlphabet ();
+    class Alphabet
-    static char const* getRippleAlphabet ();
+    {
-    static char const* getTestnetAlphabet ();
+    public:
+        // chars may not contain high-ASCII values
+        explicit Alphabet (char const* chars)
+            : m_chars (chars)
+        {
+            std::fill (m_inverse, m_inverse + 128, -1);
+            int i (0);
+            for (char const* c (chars); *c; ++c)
+                m_inverse [*c] = i++;
+        }
+
+        char const* chars () const
+            { return m_chars; }
+
+        char to_char (int digit) const
+            { return m_chars [digit]; }
+
+        char operator[] (int digit) const
+            { return to_char (digit); }
+
+        int from_char (char c) const
+            { return m_inverse [c]; }
+
+    private:
+        char const* m_chars;
+        int m_inverse [128];
+    };
+
+    static Alphabet const& getBitcoinAlphabet ();
+    static Alphabet const& getRippleAlphabet ();
+    static Alphabet const& getTestnetAlphabet ();
 
     static std::string raw_encode (
         unsigned char const* begin, unsigned char const* end,
-            char const* alphabet, bool withCheck);
+            Alphabet const& alphabet, bool withCheck);
 
     static void fourbyte_hash256 (void* out, void const* in, std::size_t bytes);
 
     template <class InputIt>
-    static std::string encode (InputIt first, InputIt last, char const* alphabet, bool withCheck)
+    static std::string encode (InputIt first, InputIt last,
+        Alphabet const& alphabet, bool withCheck)
     {
         typedef typename std::iterator_traits<InputIt>::value_type value_type;
         std::vector <typename mpl::RemoveConst <value_type>::type> v;
@@ -87,8 +118,8 @@ public:
     // VFALCO NOTE Avoid this interface which uses globals, explicitly
     //             pass the alphabet in the call to encode!
     //
-    static char const* getCurrentAlphabet ();
+    static Alphabet const& getCurrentAlphabet ();
-    static void setCurrentAlphabet (char const* alphabet);
+    static void setCurrentAlphabet (Alphabet const& alphabet);
 
     template <class Container>
     static std::string encode (Container const& container)
@@ -111,13 +142,18 @@ public:
 
     //--------------------------------------------------------------------------
 
-    static bool decode (const char* psz, Blob& vchRet, const char* pAlphabet = getCurrentAlphabet ());
+    // Raw decoder leaves the check bytes in place if present
+
+    static bool raw_decode (char const* first, char const* last,
+        void* dest, std::size_t size, bool checked, Alphabet const& alphabet);
+
+    static bool decode (const char* psz, Blob& vchRet, Alphabet const& alphabet = getCurrentAlphabet ());
     static bool decode (const std::string& str, Blob& vchRet);
-    static bool decodeWithCheck (const char* psz, Blob& vchRet, const char* pAlphabet = getCurrentAlphabet ());
+    static bool decodeWithCheck (const char* psz, Blob& vchRet, Alphabet const& alphabet = getCurrentAlphabet());
-    static bool decodeWithCheck (const std::string& str, Blob& vchRet, const char* pAlphabet);
+    static bool decodeWithCheck (const std::string& str, Blob& vchRet, Alphabet const& alphabet = getCurrentAlphabet());
 
 private:
-    static char const* s_currentAlphabet;
+    static Alphabet const* s_currentAlphabet;
 };
 
 }

src/ripple/types/api/CryptoIdentifier.h

@@ -112,6 +112,20 @@ public:
         return Base58::raw_encode (le.begin(), le.end(),
             Base58::getRippleAlphabet(), Checked);
     }
+
+    /** Convert from std::string. */
+    static std::pair <value_type, bool> from_string (std::string const& s)
+    {
+        value_type value;
+        bool success (! s.empty());
+        if (success && !Base58::raw_decode (&s.front(), &s.back()+1,
+            value.storage().begin(), value_type::storage_size, Checked,
+                Base58::getRippleAlphabet()))
+            success = false;
+        if (success && value.storage()[0] != Token)
+            success = false;
+        return std::make_pair (value, success);
+    }
 };
 
 }

src/ripple/types/api/IdentifierType.h

@@ -154,6 +154,14 @@ public:
         return Traits::to_string (m_value);
     }
 
+    /** Conversion from std::string.
+        The `bool` indicates the success of the conversion.
+    */
+    static std::pair <IdentifierType, bool> from_string (std::string const& s)
+    {
+        return Traits::from_string (s);
+    }
+
 private:
     value_type m_value;
 };

src/ripple/types/impl/Base58.cpp

@@ -25,7 +25,7 @@
 
 namespace ripple {
 
-char const* Base58::s_currentAlphabet = Base58::getRippleAlphabet ();
+Base58::Alphabet const* Base58::s_currentAlphabet = &Base58::getRippleAlphabet ();
 
 void Base58::fourbyte_hash256 (void* out, void const* in, std::size_t bytes)
 {
@@ -35,24 +35,33 @@ void Base58::fourbyte_hash256 (void* out, void const* in, std::size_t bytes)
     memcpy (out, hash.begin(), 4);
 }
 
-char const* Base58::getBitcoinAlphabet ()
+Base58::Alphabet const& Base58::getBitcoinAlphabet ()
 {
-    return "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz";
+    static Alphabet alphabet (
+        "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
+        );
+    return alphabet;
 }
 
-char const* Base58::getRippleAlphabet ()
+Base58::Alphabet const& Base58::getRippleAlphabet ()
 {
-    return "rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz";
+    static Alphabet alphabet (
+        "rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz"
+        );
+    return alphabet;
 }
 
-char const* Base58::getTestnetAlphabet ()
+Base58::Alphabet const& Base58::getTestnetAlphabet ()
 {
-    return "RPShNAF39wBUDnEGHJKLM4pQrsT7VWXYZ2bcdeCg65jkm8ofqi1tuvaxyz";
+    static Alphabet alphabet (
+        "RPShNAF39wBUDnEGHJKLM4pQrsT7VWXYZ2bcdeCg65jkm8ofqi1tuvaxyz"
+        );
+    return alphabet;
 }
 
 std::string Base58::raw_encode (
     unsigned char const* begin, unsigned char const* end,
-        char const* alphabet, bool withCheck)
+        Alphabet const& alphabet, bool withCheck)
 {
     CAutoBN_CTX pctx;
     CBigNum bn58 = 58;
@@ -88,22 +97,77 @@ std::string Base58::raw_encode (
     return str;
 }
 
-char const* Base58::getCurrentAlphabet ()
+Base58::Alphabet const& Base58::getCurrentAlphabet ()
 {
-    return s_currentAlphabet;
+    return *s_currentAlphabet;
 }
 
-void Base58::setCurrentAlphabet (char const* alphabet)
+void Base58::setCurrentAlphabet (Alphabet const& alphabet)
 {
-    s_currentAlphabet = alphabet;
+    s_currentAlphabet = &alphabet;
 }
 
 //------------------------------------------------------------------------------
 
-bool Base58::decode (const char* psz, Blob& vchRet, const char* pAlpha)
+bool Base58::raw_decode (char const* first, char const* last, void* dest,
+    std::size_t size, bool checked, Alphabet const& alphabet)
 {
-    assert (pAlpha != 0);
+    CAutoBN_CTX pctx;
+    CBigNum bn58 = 58;
+    CBigNum bn = 0;
+    CBigNum bnChar;
 
+    // Convert big endian string to bignum
+    for (char const* p = first; p != last; ++p)
+    {
+        int i (alphabet.from_char (*p));
+        if (i == -1)
+            return false;
+        bnChar.setuint ((unsigned int) i);
+
+        meets_invariant (BN_mul (&bn, &bn, &bn58, pctx));
+
+        bn += bnChar;
+    }
+
+    // Get bignum as little endian data
+    Blob vchTmp = bn.getvch ();
+
+    // Trim off sign byte if present
+    if (vchTmp.size () >= 2 && vchTmp.end ()[-1] == 0 && vchTmp.end ()[-2] >= 0x80)
+        vchTmp.erase (vchTmp.end () - 1);
+
+    char* const out (static_cast <char*> (dest));
+
+    // Count leading zeros
+    int nLeadingZeros = 0;
+    for (char const* p = first; p!=last && *p==alphabet[0]; p++)
+        nLeadingZeros++;
+
+    // Verify that the size is correct
+    if (vchTmp.size() + nLeadingZeros != size)
+        return false;
+
+    // Fill the leading zeros
+    memset (out, 0, nLeadingZeros);
+
+    // Copy little endian data to big endian
+    std::reverse_copy (vchTmp.begin (), vchTmp.end (),
+        out + nLeadingZeros);
+
+    if (checked)
+    {
+        char hash4 [4];
+        fourbyte_hash256 (hash4, out, size - 4);
+        if (memcmp (hash4, out + size - 4, 4) != 0)
+            return false;
+    }
+
+    return true;
+}
+
+bool Base58::decode (const char* psz, Blob& vchRet, Alphabet const& alphabet)
+{
     CAutoBN_CTX pctx;
     vchRet.clear ();
     CBigNum bn58 = 58;
@@ -116,7 +180,10 @@ bool Base58::decode (const char* psz, Blob& vchRet, const char* pAlpha)
     // Convert big endian string to bignum
     for (const char* p = psz; *p; p++)
     {
-        const char* p1 = strchr (pAlpha, *p);
+        // VFALCO TODO Make this use the inverse table!
+        //             Or better yet ditch this and call raw_decode
+        //
+        const char* p1 = strchr (alphabet.chars(), *p);
 
         if (p1 == NULL)
         {
@@ -129,7 +196,7 @@ bool Base58::decode (const char* psz, Blob& vchRet, const char* pAlpha)
             break;
         }
 
-        bnChar.setuint (p1 - pAlpha);
+        bnChar.setuint (p1 - alphabet.chars());
 
         if (!BN_mul (&bn, &bn, &bn58, pctx))
             throw bignum_error ("DecodeBase58 : BN_mul failed");
@@ -147,7 +214,7 @@ bool Base58::decode (const char* psz, Blob& vchRet, const char* pAlpha)
     // Restore leading zeros
     int nLeadingZeros = 0;
 
-    for (const char* p = psz; *p == pAlpha[0]; p++)
+    for (const char* p = psz; *p == alphabet.chars()[0]; p++)
         nLeadingZeros++;
 
     vchRet.assign (nLeadingZeros + vchTmp.size (), 0);
@@ -162,11 +229,9 @@ bool Base58::decode (const std::string& str, Blob& vchRet)
     return decode (str.c_str (), vchRet);
 }
 
-bool Base58::decodeWithCheck (const char* psz, Blob& vchRet, const char* pAlphabet)
+bool Base58::decodeWithCheck (const char* psz, Blob& vchRet, Alphabet const& alphabet)
 {
-    assert (pAlphabet != NULL);
+    if (!decode (psz, vchRet, alphabet))
-
-    if (!decode (psz, vchRet, pAlphabet))
         return false;
 
     if (vchRet.size () < 4)
@@ -187,9 +252,9 @@ bool Base58::decodeWithCheck (const char* psz, Blob& vchRet, const char* pAlphab
     return true;
 }
 
-bool Base58::decodeWithCheck (const std::string& str, Blob& vchRet, const char* pAlphabet)
+bool Base58::decodeWithCheck (const std::string& str, Blob& vchRet, Alphabet const& alphabet)
 {
-    return decodeWithCheck (str.c_str (), vchRet, pAlphabet);
+    return decodeWithCheck (str.c_str (), vchRet, alphabet);
 }
 
 }

src/ripple/validators/api/Source.h

@@ -44,10 +44,15 @@ public:
     */
     virtual ~Source () { }
 
+    /** The name of the source, used in diagnostic output. */
     virtual String name () = 0;
 
+    /** An identifier that uniquely describes the source.
+        This is used for identification in the database.
+    */
     virtual String uniqueID () = 0;
 
+    /** A string that is used to recreate the source from the database entry. */
     virtual String createParam () = 0;
 
     /** Fetch the most recent list from the Source.
@@ -61,14 +66,14 @@ public:
         bool success;
         String message;
         Time expirationTime;
-        Array <Info> list;
+        std::vector <Info> list;
     };
 
     /** Cancel any pending fetch.
         The default implementation does nothing.
     */
     virtual void cancel () { }
-    virtual Result fetch (Journal journal) = 0;
+    virtual void fetch (Result& result, Journal journal) = 0;
 };
 
 }

src/ripple/validators/impl/Logic.h

@@ -269,15 +269,19 @@ public:
     //
     void addStatic (Source* source)
     {
-        m_journal.info << "Addding static source '" << source->name() << "'";
+        m_journal.info << "Addding static " << source->name();
 
         ScopedPointer <Source> object (source);
-        Source::Result result (object->fetch (m_journal));
+        Source::Result result;
+        object->fetch (result, m_journal);
 
         if (result.success)
         {
             SharedState::Access state (m_state);
-            merge (result.list, source, state);
+            std::size_t const numAdded (
+                merge (result.list, source, state));
+            m_journal.info << "Added " << numAdded
+                           << " trusted validators from " << source->name();
         }
         else
         {
@@ -289,7 +293,7 @@ public:
     //
     void add (Source* source)
     {
-        m_journal.info << "Adding source '" << source->name() << "'";
+        m_journal.info << "Adding " << source->name();
 
         {
             SharedState::Access state (m_state);
@@ -303,13 +307,13 @@ public:
     // Add each entry in the list to the map, incrementing the
     // reference count if it already exists, and updating fields.
     //
-    void merge (Array <Source::Info> const& list,
+    std::size_t merge (std::vector <Source::Info> const& list,
         Source* source, SharedState::Access& state)
     {
         std::size_t numAdded (0);
         for (std::size_t i = 0; i < list.size (); ++i)
         {
-            Source::Info const& info (list.getReference (i));
+            Source::Info const& info (list [i]);
             std::pair <MapType::iterator, bool> result (
                 state->map.emplace (info.publicKey, Validator ()));
             Validator& validatorInfo (result.first->second);
@@ -322,20 +326,19 @@ public:
             }
         }
 
-        m_journal.info << "Added " << numAdded
+        return numAdded;
-                       << " trusted validators from '" << source->name() << "'";
     }
 
     // Decrement the reference count of each item in the list
     // in the map.
     //
-    void remove (Array <Source::Info> const& list,
+    std::size_t remove (std::vector <Source::Info> const& list,
         Source* source, SharedState::Access& state)
     {
         std::size_t numRemoved (0);
         for (std::size_t i = 0; i < list.size (); ++i)
         {
-            Source::Info const& info (list.getReference (i));
+            Source::Info const& info (list [i]);
             MapType::iterator iter (state->map.find (info.publicKey));
             bassert (iter != state->map.end ());
             Validator& validatorInfo (iter->second);
@@ -348,8 +351,7 @@ public:
             }
         }
 
-        m_journal.info << "Removed " << numRemoved
+        return numRemoved;
-                       << " trusted validators from '" << source->name() << "'";
     }
 
     //----------------------------------------------------------------------
@@ -410,9 +412,12 @@ public:
     /** Perform a fetch on the source. */
     void fetch (SourceDesc& desc)
     {
-        m_journal.info << "fetch " << desc.source->name();
+        Source* const source (desc.source);
 
-        Source::Result result (desc.source->fetch (m_journal));
+        m_journal.info << "fetch " << source->name();
+
+        Source::Result result;
+        source->fetch (result, m_journal);
 
         // Reset fetch timer for the source.
         desc.whenToFetch = Time::getCurrentTime () +
@@ -423,13 +428,22 @@ public:
             SharedState::Access state (m_state);
 
             // Add the new source info to the map
-            merge (result.list, desc.source, state);
+            std::size_t const numAdded (
+                merge (result.list, source, state));
 
             // Swap lists
             desc.result.swapWith (result);
 
             // Remove the old source info from the map
-            remove (result.list, desc.source, state);
+            std::size_t const numRemoved (
+                remove (result.list, source, state));
+
+            if (numAdded > numRemoved)
+                m_journal.info << "Added " << (numAdded - numRemoved) <<
+                    " trusted validators from " << source->name();
+            else if (numRemoved > numAdded)
+                m_journal.info << "Removed " << (numRemoved - numAdded) <<
+                    " trusted validators from " << source->name();
 
             // See if we need to rebuild
             checkChosen ();

src/ripple/validators/impl/Manager.cpp

@@ -188,14 +188,19 @@ public:
 
     void addStrings (String name, StringArray const& stringArray)
     {
-        addStaticSource (SourceStrings::New (
+        if (stringArray.size() > 0)
-            name, stringArray));
+        {
+            addStaticSource (SourceStrings::New (name, stringArray));
+        }
+        else
+        {
+            m_journal.debug << "Static source '" << name << "' is empty.";
+        }
     }
 
     void addFile (File const& file)
     {
-        //addStaticSource (SourceFile::New (file));
+        addStaticSource (SourceFile::New (file));
-        addSource (SourceFile::New (file));
     }
 
     void addURL (URL const& url)

src/ripple/validators/impl/Source.cpp

@@ -30,7 +30,7 @@ void Source::Result::swapWith (Result& other)
 {
     std::swap (success, other.success);
     std::swap (message, other.message);
-    list.swapWith (other.list);
+    list.swap (other.list);
 }
 
 }

src/ripple/validators/impl/SourceFile.cpp

@@ -36,7 +36,7 @@ public:
     
     String name ()
     {
-        return "File :'" + m_file.getFullPathName () + "'";
+        return "File: '" + m_file.getFullPathName () + "'";
     }
 
     String uniqueID ()
@@ -49,10 +49,8 @@ public:
         return m_file.getFullPathName ();
     }
     
-    Result fetch (Journal journal)
+    void fetch (Result& result, Journal journal)
     {
-        Result result;
-
         int64 const fileSize (m_file.getSize ());
 
         if (fileSize != 0)
@@ -68,6 +66,8 @@ public:
 
                 if (amountRead == fileSize)
                 {
+                    Utilities::ParseResultLine lineFunction (result, journal);
+                    Utilities::processLines (buffer.begin(), buffer.end(), lineFunction);
                 }
             }
             else
@@ -79,8 +79,6 @@ public:
         {
             // file doesn't exist
         }
-
-        return result;
     }
 
 private:

src/ripple/validators/impl/SourceStrings.cpp

@@ -51,11 +51,9 @@ public:
         return String::empty;
     }
 
-    Result fetch (Journal journal)
+    void fetch (Result& result, Journal journal)
     {
-        Result result;
+        result.list.reserve (m_strings.size ());
-
-        result.list.ensureStorageAllocated (m_strings.size ());
 
         for (int i = 0; i < m_strings.size (); ++i)
         {
@@ -65,7 +63,6 @@ public:
 
         result.success = result.list.size () > 0;
         result.expirationTime = Time::getCurrentTime () + RelativeTime::hours (24);
-        return result;
     }
 
 private:

src/ripple/validators/impl/SourceURL.cpp

@@ -49,17 +49,22 @@ public:
         return m_url.full();
     }
 
-    Result fetch (Journal journal)
+    void fetch (Result& result, Journal journal)
     {
-        Result result;
-
         ScopedPointer <HTTPClientBase> client (HTTPClientBase::New ());
 
         HTTPClientBase::result_type httpResult (client->get (m_url));
 
         if (httpResult.first == 0)
         {
-            //Logger::outputDebugString (httpResult.second->toString ());
+#if 0
+            journal.debug << std::endl << httpResult.second->toString ();
+            journal.debug << std::endl << httpResult.second->body().to_string();
+#endif
+
+            Utilities::ParseResultLine lineFunction (result, journal);
+            std::string const s (httpResult.second->body().to_string());
+            Utilities::processLines (s.begin(), s.end(), lineFunction);
         }
         else
         {
@@ -67,8 +72,6 @@ public:
                 "HTTP GET to " << m_url.full().toStdString() <<
                 " failed: '" << httpResult.first.message () << "'";
         }
-
-        return result;
     }
 
 private:
@@ -80,10 +83,7 @@ private:
 SourceURL* SourceURL::New (
     URL const& url)
 {
-    ScopedPointer <SourceURL> object (
+    return new SourceURLImp (url);
-        new SourceURLImp (url));
-
-    return object.release ();
 }
 
 }

src/ripple/validators/impl/StoreSqdb.cpp

@@ -36,6 +36,9 @@ Error StoreSqdb::open (File const& file)
     if (!error)
         error = init ();
 
+    if (!error)
+        error = update ();
+
     return error;
 }
 
@@ -116,20 +119,16 @@ void StoreSqdb::update (SourceDesc& desc, bool updateFetchResults)
     if (! error && updateFetchResults)
     {
         // Delete the previous data set
-        {
+        m_session.once (error) <<
-            sqdb::statement st = (m_session.prepare <<
             "DELETE FROM ValidatorsSourceInfo WHERE "
             "  sourceID = ?; "
             ,sqdb::use (sourceID)
-                );
+            ;
-
-            st.execute_and_fetch (error);
-        }
 
         // Insert the new data set
         if (! error)
         {
-            std::string publicKey;
+            std::string publicKeyString;
             String label;
 
             sqdb::statement st = (m_session.prepare <<
@@ -141,15 +140,15 @@ void StoreSqdb::update (SourceDesc& desc, bool updateFetchResults)
                 "  ?, ?, ? "
                 ");"
                 ,sqdb::use (sourceID)
-                ,sqdb::use (publicKey)
+                ,sqdb::use (publicKeyString)
                 ,sqdb::use (label)
                 );
 
-            Array <Source::Info>& list (desc.result.list);
+            std::vector <Source::Info>& list (desc.result.list);
             for (std::size_t i = 0; ! error && i < list.size(); ++i)
             {
-                Source::Info& info (list.getReference(i));
+                Source::Info& info (list [i]);
-                publicKey = Utilities::publicKeyToString (info.publicKey);
+                publicKeyString = info.publicKey.to_string ();
                 label = list[i].label;
                 st.execute_and_fetch (error);
             }
@@ -255,11 +254,11 @@ void StoreSqdb::selectList (SourceDesc& desc)
     bassert (desc.result.list.size() == 0);
 
     // Pre-allocate some storage
-    desc.result.list.ensureStorageAllocated (count);
+    desc.result.list.reserve (count);
 
     // Prepare the select
     {
-        std::string publicKey;
+        std::string publicKeyString;
         std::string label;
         sqdb::statement st = (m_session.prepare <<
             "SELECT "
@@ -267,7 +266,7 @@ void StoreSqdb::selectList (SourceDesc& desc)
             "  label "
             "FROM ValidatorsSourceInfo WHERE "
             "  sourceID = ? "
-            ,sqdb::into (publicKey)
+            ,sqdb::into (publicKeyString)
             ,sqdb::into (label)
             ,sqdb::use (sourceID)
             );
@@ -278,9 +277,20 @@ void StoreSqdb::selectList (SourceDesc& desc)
             do
             {
                 Source::Info info;
-                info.publicKey = Utilities::stringToPublicKey (publicKey);
+                std::pair <RipplePublicKey, bool> result (
+                    RipplePublicKey::from_string (publicKeyString));
+                if (result.second)
+                {
+                    bassert (result.first.to_string() == publicKeyString);
+                    info.publicKey = result.first;
                     info.label = label;
-                desc.result.list.add (info);
+                    desc.result.list.push_back (info);
+                }
+                else
+                {
+                    m_journal.error << "Invalid public key '" <<
+                        publicKeyString << "' found in database";
+                }
             }
             while (st.fetch (error));
         }
@@ -294,6 +304,81 @@ void StoreSqdb::selectList (SourceDesc& desc)
 
 //--------------------------------------------------------------------------
 
+// Update the database for the current schema
+Error StoreSqdb::update ()
+{
+    Error error;
+
+    sqdb::transaction tr (m_session);
+
+    // Get the version from the database
+    int version (0);
+    if (! error)
+    {
+        m_session.once (error) <<
+            "SELECT "
+            "  version "
+            "FROM SchemaVersion WHERE "
+            "  name = 'Validators' "
+            ,sqdb::into (version)
+            ;
+
+        if (! m_session.got_data ())
+        {
+            // pre-dates the "SchemaVersion" table
+            version = 0;
+        }
+    }
+
+    if (! error && version != currentSchemaVersion)
+    {
+        m_journal.info << "Updating old database version " << version;
+    }
+
+    // Update database based on version
+    if (! error && version < 1)
+    {
+        // Delete all the old data since its likely wrong
+        m_session.once (error) <<
+            "DELETE FROM ValidatorsSource";
+
+        if (! error)
+        {
+            m_session.once (error) <<
+                "DELETE FROM ValidatorsSourceInfo";
+        }
+    }
+
+    // Update the version to the current version
+    if (! error)
+    {
+        int const version (currentSchemaVersion);
+
+        m_session.once (error) <<
+            "INSERT OR REPLACE INTO SchemaVersion ( "
+            "  name, "
+            "  version "
+            ") VALUES ( "
+            "  'Validators', ? "
+            "); "
+            ,sqdb::use (version)
+            ;
+    }
+
+    if (! error)
+    {
+        error = tr.commit();
+    }
+
+    if (error)
+    {
+        tr.rollback ();
+        report (error, __FILE__, __LINE__);
+    }
+
+    return error;
+}
+
 Error StoreSqdb::init ()
 {
     Error error;
@@ -342,6 +427,23 @@ Error StoreSqdb::init ()
             ;
     }
 
+    if (! error)
+    {
+        // This table maps component names like "Validators" to their
+        // corresponding schema version number. This method allows us
+        // to keep all logic data in one database, or each in its own
+        // database, or in any grouping of databases, while still being
+        // able to let an individual component know what version of its
+        // schema it is opening.
+        //
+        m_session.once (error) <<
+            "CREATE TABLE IF NOT EXISTS SchemaVersion ( "
+            "  name             TEXT PRIMARY KEY, "
+            "  version          INTEGER"
+            ");"
+            ;
+    }
+
     if (! error)
     {
         error = tr.commit();

src/ripple/validators/impl/StoreSqdb.h

@@ -29,6 +29,12 @@ class StoreSqdb
     , public LeakChecked <StoreSqdb>
 {
 public:
+    enum
+    {
+        // This affects the format of the data!
+        currentSchemaVersion = 1
+    };
+
     explicit StoreSqdb (Journal journal = Journal());
 
     ~StoreSqdb ();
@@ -45,6 +51,7 @@ private:
     bool select (SourceDesc& desc);
     void selectList (SourceDesc& desc);
 
+    Error update ();
     Error init ();
 
     Journal m_journal;

src/ripple/validators/impl/Tests.cpp

@@ -121,23 +121,19 @@ public:
             return String::empty;
         }
 
-        Result fetch (Journal)
+        void fetch (Result& result, Journal)
         {
-            Result result;
-
             result.success = true;
             result.message = String::empty;
-            result.list.ensureStorageAllocated (numberOfTestValidators);
+            result.list.reserve (numberOfTestValidators);
 
             for (uint32 i = m_start ; i < m_end; ++i)
             {
                 Info info;
                 info.publicKey = RipplePublicKey::createFromInteger (i);
                 info.label = String::fromNumber (i);
-                result.list.add (info);
+                result.list.push_back (info);
             }
-
-            return result;
         }
 
         String m_name;

src/ripple/validators/impl/Utilities.cpp

@@ -24,22 +24,25 @@ struct Utilities::Helpers
 {
     // Matches a validator info line.
     //
-    static boost::regex& reInfo ()
+    static boost::regex const& reInfo ()
     {
         // e.g.
         //
         // n9KorY8QtTdRx7TVDpwnG9NvyxsDwHUKUEeDLY3AkiGncVaSXZi5 Comment Text
         //
         static boost::regex re (
-            "^"                 //     start of line
+            "\\G"               //      end of last match (or start)
-            "(?:\\h*)"          //     horiz-white (optional)
+            "(?:[\\v\\h]*)"     //      white (optional)
             "([^\\h\\v]+)"      // [1]  non-white run
             "(?:\\h*)"          //      horiz-white (optional)
-            "([^\\h\\v]*)"      // [2] any text (optional)
+            "([^\\v]*?)"        // [2]  non vert-white text (optional)
-            "$"                 //     end of line
+            "(?:\\h*)"          //      white run (optional)
+            "(?:\\v*)"          //      vert-white (optional)
 
-            , boost::regex::perl |
+            //"(?:\\')"           //      buffer boundary
-              boost::regex_constants::match_flags::match_not_dot_null
+
+            , boost::regex::perl
+              //| boost::regex_constants::match_flags::match_not_dot_newline
         );
 
         return re;
@@ -47,7 +50,7 @@ struct Utilities::Helpers
 
     // Matches a comment or whitespace line.
     //
-    static boost::regex& reComment ()
+    static boost::regex const& reComment ()
     {
         // e.g.
         //
@@ -87,23 +90,18 @@ bool Utilities::parseInfoLine (
 
         RippleAddress deprecatedPublicKey;
 
-        // VFALCO NOTE These bool return values are poorlydocumented
+        if (deprecatedPublicKey.setNodePublic (encodedKey))
-        //
-        if (deprecatedPublicKey.setSeedGeneric (encodedKey))
-        {
-            // expected a domain or public key but got a generic seed?
-            // log?
-        }
-        else if (deprecatedPublicKey.setNodePublic (encodedKey))
         {
             // We got a public key.
             RipplePublicKey publicKey (deprecatedPublicKey);
+            info.label = commentText;
+            info.publicKey = publicKey;
             success = true;
         }
         else
         {
             // Some other junk.
-            // log?
+            journal.error << "Invalid RipplePublicKey: '" << encodedKey << "'";
         }
     }
     else if (boost::regex_match (line, match, Helpers::reComment ()))
@@ -113,54 +111,9 @@ bool Utilities::parseInfoLine (
     else
     {
         // Log a warning about a parsing error
+        journal.error << "Invalid Validators source line:" << std::endl << line;
     }
 
-#if 0
-    static boost::regex reReferral ("\\`\\s*(\\S+)(?:\\s+(.+))?\\s*\\'");
-
-    if (!boost::regex_match (strReferral, smMatch, reReferral))
-    {
-        WriteLog (lsWARNING, UniqueNodeList) << str (boost::format ("Bad validator: syntax error: %s: %s") % strSite % strReferral);
-    }
-    else
-    {
-        std::string     strRefered  = smMatch[1];
-        std::string     strComment  = smMatch[2];
-        RippleAddress   naValidator;
-
-        if (naValidator.setSeedGeneric (strRefered))
-        {
-
-            WriteLog (lsWARNING, UniqueNodeList) << str (boost::format ("Bad validator: domain or public key required: %s %s") % strRefered % strComment);
-        }
-        else if (naValidator.setNodePublic (strRefered))
-        {
-            // A public key.
-            // XXX Schedule for CAS lookup.
-            nodeAddPublic (naValidator, vsWhy, strComment);
-
-            WriteLog (lsINFO, UniqueNodeList) << str (boost::format ("Node Public: %s %s") % strRefered % strComment);
-
-            if (naNodePublic.isValid ())
-                vstrValues.push_back (str (boost::format ("('%s',%d,'%s')") % strNodePublic % iValues % naValidator.humanNodePublic ()));
-
-            iValues++;
-        }
-        else
-        {
-            // A domain: need to look it up.
-            nodeAddDomain (strRefered, vsWhy, strComment);
-
-            WriteLog (lsINFO, UniqueNodeList) << str (boost::format ("Node Domain: %s %s") % strRefered % strComment);
-
-            if (naNodePublic.isValid ())
-                vstrValues.push_back (str (boost::format ("('%s',%d,%s)") % strNodePublic % iValues % sqlEscape (strRefered)));
-
-            iValues++;
-        }
-    }
-#endif
-
     return success;
 }
 
@@ -171,15 +124,13 @@ void Utilities::parseResultLine (
         std::string const& line,
             Journal journal)
 {
-    bool success = false;
-
-    if (! success)
-    {
     Source::Info info;
 
-        success = parseInfoLine (info, line, journal);
+    bool success = parseInfoLine (info, line, journal);
     if (success)
-            result.list.add (info);
+    {
+        result.list.push_back (info);
+        result.success = true;
     }
 }
 
@@ -249,19 +200,6 @@ Time Utilities::stringToTime (String s)
     return Time (0);
 }
 
-std::string Utilities::publicKeyToString (RipplePublicKey const& publicKey)
-{
-    std::string s (RipplePublicKey::size, ' ');
-    std::copy (publicKey.cbegin(), publicKey.cend(), s.begin());
-    return s;
-}
-
-RipplePublicKey Utilities::stringToPublicKey (std::string const& s)
-{
-    bassert (s.size() == RipplePublicKey::size);
-    return RipplePublicKey ();
-}
-
 //------------------------------------------------------------------------------
 
 }

src/ripple/validators/impl/Utilities.h

@@ -23,31 +23,93 @@
 namespace ripple {
 namespace Validators {
 
-/** Common code for Validators classes.
+/** Common code for Validators classes. */
-*/
 class Utilities
 {
 public:
     typedef std::vector <std::string> Strings;
 
-#if 0
+    /** A suitable LineFunction for parsing items into a fetch result. */
-    /** Parse a ConstBufferSequence of newline delimited text into strings.
+    class ParseResultLine
-        This works incrementally.
-    */
-    template <typename ConstBufferSequence>
-    static void parseLines (Strings& lines, ConstBufferSequence const& buffers)
     {
-        for (typename ConstBufferSequence::const_iterator iter = buffers.begin ();
+    public:
-            iter != buffers.end (); ++iter)
+        ParseResultLine (Source::Result& result, Journal journal)
-            parserLines (lines, *iter);
+            : m_result (&result)
+            , m_journal (journal)
+        { }
+
+        template <typename BidirectionalIterator>
+        void operator() (BidirectionalIterator first, BidirectionalIterator last)
+        {
+            std::string s (first, last);
+            Utilities::parseResultLine (*m_result, s, m_journal);
         }
 
-    /** Turn a linear buffer of newline delimited text into strings.
+    private:
-        This can be called incrementally, i.e. successive calls with
+        Source::Result* m_result;
-        multiple buffer segments.
+        Journal m_journal;
+    };
+
+    /** UnaryPredicate for breaking up lines.
+        This returns `true` for the first non-vertical whitespace character that
+        follows a vertical whitespace character.
     */
-    static void parseLines (Strings& lines, char const* buf, std::size_t bytes);
+    class FollowingVerticalWhite
-#endif
+    {
+    public:
+        FollowingVerticalWhite ()
+            : m_gotWhite (false)
+        {
+        }
+
+        template <typename CharT>
+        static bool isVerticalWhitespace (CharT c)
+        {
+            return c == '\r' || c == '\n';
+        }
+
+        template <typename CharT>
+        bool operator() (CharT c)
+        {
+            if (isVerticalWhitespace (c))
+            {
+                m_gotWhite = true;
+                return false;
+            }
+            else if (m_gotWhite)
+            {
+                m_gotWhite = false;
+                return true;
+            }
+            return false;
+        }
+
+    private:
+        bool m_gotWhite;
+    };
+
+    /** Call LineFunction for each newline-separated line in the input.
+        LineFunction will be called with this signature:
+        @code
+            void LineFunction (BidirectionalIterator first, BidirectionalIterator last)
+        @endcode
+        Where first and last mark the beginning and ending of the line.
+        The last line in the input may or may not contain the trailing newline.
+    */
+    template <typename BidirectionalIterator, typename LineFunction>
+    static void processLines (BidirectionalIterator first,
+        BidirectionalIterator last, LineFunction f)
+    {
+        for (;;)
+        {
+            BidirectionalIterator split (std::find_if (
+                first, last, FollowingVerticalWhite ()));
+            f (first, split);
+            if (split == last)
+                break;
+            first = split;
+        }
+    }
 
     /** Parse a string into the Source::Result.
         Invalid or comment lines will be skipped.
@@ -67,10 +129,6 @@ public:
     static String timeToString (Time const& t);
     static Time stringToTime (String s);
 
-    // conversion between RipplePublicKey and String
-    static std::string publicKeyToString (RipplePublicKey const& publicKey);
-    static RipplePublicKey stringToPublicKey (std::string const& s);
-
     struct Helpers;
 
     /** Parse a string into a Source::Info.

src/ripple_app/main/Application.cpp

@@ -614,21 +614,26 @@ public:
     // Initialize the Validators object with Config information.
     void prepareValidators ()
     {
+#if 0
         {
             std::vector <std::string> const& strings (getConfig().validators);
-            if (! strings.empty ())
             m_validators->addStrings ("rippled.cfg", strings);
         }
+#endif
 
+#if 1
         if (! getConfig().getValidatorsURL().empty())
         {
             m_validators->addURL (getConfig().getValidatorsURL());
         }
+#endif
 
+#if 0
         if (getConfig().getValidatorsFile() != File::nonexistent ())
         {
             m_validators->addFile (getConfig().getValidatorsFile());
         }
+#endif
     }
 
     //--------------------------------------------------------------------------

src/ripple_data/crypto/Base58Data.cpp

@@ -62,10 +62,10 @@ void CBase58Data::SetData (int nVersionIn, const unsigned char* pbegin, const un
     SetData (nVersionIn, (void*)pbegin, pend - pbegin);
 }
 
-bool CBase58Data::SetString (const char* psz, unsigned char version, const char* pAlphabet)
+bool CBase58Data::SetString (const char* psz, unsigned char version, Base58::Alphabet const& alphabet)
 {
     Blob vchTemp;
-    Base58::decodeWithCheck (psz, vchTemp, pAlphabet);
+    Base58::decodeWithCheck (psz, vchTemp, alphabet);
 
     if (vchTemp.empty () || vchTemp[0] != version)
     {

src/ripple_data/crypto/Base58Data.h

@@ -47,7 +47,7 @@ protected:
     void SetData (int nVersionIn, const unsigned char* pbegin, const unsigned char* pend);
 
 public:
-    bool SetString (const char* psz, unsigned char version, const char* pAlphabet = Base58::getCurrentAlphabet ());
+    bool SetString (const char* psz, unsigned char version, Base58::Alphabet const& alphabet = Base58::getCurrentAlphabet ());
     bool SetString (const std::string& str, unsigned char version);
 
     std::string ToString () const;

src/ripple_data/protocol/RippleAddress.cpp

@@ -348,7 +348,7 @@ std::string RippleAddress::humanAccountID () const
     }
 }
 
-bool RippleAddress::setAccountID (const std::string& strAccountID, const char* pAlphabet)
+bool RippleAddress::setAccountID (const std::string& strAccountID, Base58::Alphabet const& alphabet)
 {
     if (strAccountID.empty ())
     {
@@ -358,7 +358,7 @@ bool RippleAddress::setAccountID (const std::string& strAccountID, const char* p
     }
     else
     {
-        mIsValid    = SetString (strAccountID.c_str (), VER_ACCOUNT_ID, pAlphabet);
+        mIsValid    = SetString (strAccountID.c_str (), VER_ACCOUNT_ID, alphabet);
     }
 
     return mIsValid;

src/ripple_data/protocol/RippleAddress.h

@@ -95,7 +95,7 @@ public:
 
     std::string humanAccountID () const;
 
-    bool setAccountID (const std::string& strAccountID, const char* pAlphabet = Base58::getCurrentAlphabet ());
+    bool setAccountID (const std::string& strAccountID, Base58::Alphabet const& alphabet = Base58::getCurrentAlphabet ());
     void setAccountID (const uint160& hash160In);
 
     static RippleAddress createAccountID (const std::string& strAccountID)