From 66a272debd8fb7c6536817449b97f4b6e2929d94 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Thu, 3 Oct 2013 18:29:30 -0700 Subject: [PATCH] Alphabet class for base58 conversions, Validators work --- src/ripple/types/api/Base58.h | 58 ++++++-- src/ripple/types/api/CryptoIdentifier.h | 14 ++ src/ripple/types/api/IdentifierType.h | 8 ++ src/ripple/types/impl/Base58.cpp | 111 ++++++++++++--- src/ripple/validators/api/Source.h | 9 +- src/ripple/validators/impl/Logic.h | 46 +++--- src/ripple/validators/impl/Manager.cpp | 13 +- src/ripple/validators/impl/Source.cpp | 2 +- src/ripple/validators/impl/SourceFile.cpp | 12 +- src/ripple/validators/impl/SourceStrings.cpp | 7 +- src/ripple/validators/impl/SourceURL.cpp | 20 +-- src/ripple/validators/impl/StoreSqdb.cpp | 142 ++++++++++++++++--- src/ripple/validators/impl/StoreSqdb.h | 7 + src/ripple/validators/impl/Tests.cpp | 10 +- src/ripple/validators/impl/Utilities.cpp | 108 +++----------- src/ripple/validators/impl/Utilities.h | 100 ++++++++++--- src/ripple_app/main/Application.cpp | 9 +- src/ripple_data/crypto/Base58Data.cpp | 4 +- src/ripple_data/crypto/Base58Data.h | 2 +- src/ripple_data/protocol/RippleAddress.cpp | 4 +- src/ripple_data/protocol/RippleAddress.h | 2 +- 21 files changed, 468 insertions(+), 220 deletions(-) diff --git a/src/ripple/types/api/Base58.h b/src/ripple/types/api/Base58.h index 7372b63728..efc54fb864 100644 --- a/src/ripple/types/api/Base58.h +++ b/src/ripple/types/api/Base58.h @@ -43,18 +43,49 @@ namespace ripple { class Base58 { public: - static char const* getBitcoinAlphabet (); - static char const* getRippleAlphabet (); - static char const* getTestnetAlphabet (); + class Alphabet + { + public: + // chars may not contain high-ASCII values + explicit Alphabet (char const* chars) + : m_chars (chars) + { + std::fill (m_inverse, m_inverse + 128, -1); + int i (0); + for (char const* c (chars); *c; ++c) + m_inverse [*c] = i++; + } + + char const* chars () const + { return m_chars; } + + char to_char (int digit) const + { return m_chars [digit]; } + + char operator[] (int digit) const + { return to_char (digit); } + + int from_char (char c) const + { return m_inverse [c]; } + + private: + char const* m_chars; + int m_inverse [128]; + }; + + static Alphabet const& getBitcoinAlphabet (); + static Alphabet const& getRippleAlphabet (); + static Alphabet const& getTestnetAlphabet (); static std::string raw_encode ( unsigned char const* begin, unsigned char const* end, - char const* alphabet, bool withCheck); + Alphabet const& alphabet, bool withCheck); static void fourbyte_hash256 (void* out, void const* in, std::size_t bytes); template - static std::string encode (InputIt first, InputIt last, char const* alphabet, bool withCheck) + static std::string encode (InputIt first, InputIt last, + Alphabet const& alphabet, bool withCheck) { typedef typename std::iterator_traits::value_type value_type; std::vector ::type> v; @@ -87,8 +118,8 @@ public: // VFALCO NOTE Avoid this interface which uses globals, explicitly // pass the alphabet in the call to encode! // - static char const* getCurrentAlphabet (); - static void setCurrentAlphabet (char const* alphabet); + static Alphabet const& getCurrentAlphabet (); + static void setCurrentAlphabet (Alphabet const& alphabet); template static std::string encode (Container const& container) @@ -111,13 +142,18 @@ public: //-------------------------------------------------------------------------- - static bool decode (const char* psz, Blob& vchRet, const char* pAlphabet = getCurrentAlphabet ()); + // Raw decoder leaves the check bytes in place if present + + static bool raw_decode (char const* first, char const* last, + void* dest, std::size_t size, bool checked, Alphabet const& alphabet); + + static bool decode (const char* psz, Blob& vchRet, Alphabet const& alphabet = getCurrentAlphabet ()); static bool decode (const std::string& str, Blob& vchRet); - static bool decodeWithCheck (const char* psz, Blob& vchRet, const char* pAlphabet = getCurrentAlphabet ()); - static bool decodeWithCheck (const std::string& str, Blob& vchRet, const char* pAlphabet); + static bool decodeWithCheck (const char* psz, Blob& vchRet, Alphabet const& alphabet = getCurrentAlphabet()); + static bool decodeWithCheck (const std::string& str, Blob& vchRet, Alphabet const& alphabet = getCurrentAlphabet()); private: - static char const* s_currentAlphabet; + static Alphabet const* s_currentAlphabet; }; } diff --git a/src/ripple/types/api/CryptoIdentifier.h b/src/ripple/types/api/CryptoIdentifier.h index 45b1de1047..b05dadd2e5 100644 --- a/src/ripple/types/api/CryptoIdentifier.h +++ b/src/ripple/types/api/CryptoIdentifier.h @@ -112,6 +112,20 @@ public: return Base58::raw_encode (le.begin(), le.end(), Base58::getRippleAlphabet(), Checked); } + + /** Convert from std::string. */ + static std::pair from_string (std::string const& s) + { + value_type value; + bool success (! s.empty()); + if (success && !Base58::raw_decode (&s.front(), &s.back()+1, + value.storage().begin(), value_type::storage_size, Checked, + Base58::getRippleAlphabet())) + success = false; + if (success && value.storage()[0] != Token) + success = false; + return std::make_pair (value, success); + } }; } diff --git a/src/ripple/types/api/IdentifierType.h b/src/ripple/types/api/IdentifierType.h index c0ff0b7747..afed6d23c9 100644 --- a/src/ripple/types/api/IdentifierType.h +++ b/src/ripple/types/api/IdentifierType.h @@ -154,6 +154,14 @@ public: return Traits::to_string (m_value); } + /** Conversion from std::string. + The `bool` indicates the success of the conversion. + */ + static std::pair from_string (std::string const& s) + { + return Traits::from_string (s); + } + private: value_type m_value; }; diff --git a/src/ripple/types/impl/Base58.cpp b/src/ripple/types/impl/Base58.cpp index 1363a9553c..cecdd52786 100644 --- a/src/ripple/types/impl/Base58.cpp +++ b/src/ripple/types/impl/Base58.cpp @@ -25,7 +25,7 @@ namespace ripple { -char const* Base58::s_currentAlphabet = Base58::getRippleAlphabet (); +Base58::Alphabet const* Base58::s_currentAlphabet = &Base58::getRippleAlphabet (); void Base58::fourbyte_hash256 (void* out, void const* in, std::size_t bytes) { @@ -35,24 +35,33 @@ void Base58::fourbyte_hash256 (void* out, void const* in, std::size_t bytes) memcpy (out, hash.begin(), 4); } -char const* Base58::getBitcoinAlphabet () +Base58::Alphabet const& Base58::getBitcoinAlphabet () { - return "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; + static Alphabet alphabet ( + "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" + ); + return alphabet; } -char const* Base58::getRippleAlphabet () +Base58::Alphabet const& Base58::getRippleAlphabet () { - return "rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz"; + static Alphabet alphabet ( + "rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz" + ); + return alphabet; } -char const* Base58::getTestnetAlphabet () +Base58::Alphabet const& Base58::getTestnetAlphabet () { - return "RPShNAF39wBUDnEGHJKLM4pQrsT7VWXYZ2bcdeCg65jkm8ofqi1tuvaxyz"; + static Alphabet alphabet ( + "RPShNAF39wBUDnEGHJKLM4pQrsT7VWXYZ2bcdeCg65jkm8ofqi1tuvaxyz" + ); + return alphabet; } std::string Base58::raw_encode ( unsigned char const* begin, unsigned char const* end, - char const* alphabet, bool withCheck) + Alphabet const& alphabet, bool withCheck) { CAutoBN_CTX pctx; CBigNum bn58 = 58; @@ -88,22 +97,77 @@ std::string Base58::raw_encode ( return str; } -char const* Base58::getCurrentAlphabet () +Base58::Alphabet const& Base58::getCurrentAlphabet () { - return s_currentAlphabet; + return *s_currentAlphabet; } -void Base58::setCurrentAlphabet (char const* alphabet) +void Base58::setCurrentAlphabet (Alphabet const& alphabet) { - s_currentAlphabet = alphabet; + s_currentAlphabet = &alphabet; } //------------------------------------------------------------------------------ -bool Base58::decode (const char* psz, Blob& vchRet, const char* pAlpha) +bool Base58::raw_decode (char const* first, char const* last, void* dest, + std::size_t size, bool checked, Alphabet const& alphabet) { - assert (pAlpha != 0); + CAutoBN_CTX pctx; + CBigNum bn58 = 58; + CBigNum bn = 0; + CBigNum bnChar; + // Convert big endian string to bignum + for (char const* p = first; p != last; ++p) + { + int i (alphabet.from_char (*p)); + if (i == -1) + return false; + bnChar.setuint ((unsigned int) i); + + meets_invariant (BN_mul (&bn, &bn, &bn58, pctx)); + + bn += bnChar; + } + + // Get bignum as little endian data + Blob vchTmp = bn.getvch (); + + // Trim off sign byte if present + if (vchTmp.size () >= 2 && vchTmp.end ()[-1] == 0 && vchTmp.end ()[-2] >= 0x80) + vchTmp.erase (vchTmp.end () - 1); + + char* const out (static_cast (dest)); + + // Count leading zeros + int nLeadingZeros = 0; + for (char const* p = first; p!=last && *p==alphabet[0]; p++) + nLeadingZeros++; + + // Verify that the size is correct + if (vchTmp.size() + nLeadingZeros != size) + return false; + + // Fill the leading zeros + memset (out, 0, nLeadingZeros); + + // Copy little endian data to big endian + std::reverse_copy (vchTmp.begin (), vchTmp.end (), + out + nLeadingZeros); + + if (checked) + { + char hash4 [4]; + fourbyte_hash256 (hash4, out, size - 4); + if (memcmp (hash4, out + size - 4, 4) != 0) + return false; + } + + return true; +} + +bool Base58::decode (const char* psz, Blob& vchRet, Alphabet const& alphabet) +{ CAutoBN_CTX pctx; vchRet.clear (); CBigNum bn58 = 58; @@ -116,7 +180,10 @@ bool Base58::decode (const char* psz, Blob& vchRet, const char* pAlpha) // Convert big endian string to bignum for (const char* p = psz; *p; p++) { - const char* p1 = strchr (pAlpha, *p); + // VFALCO TODO Make this use the inverse table! + // Or better yet ditch this and call raw_decode + // + const char* p1 = strchr (alphabet.chars(), *p); if (p1 == NULL) { @@ -129,7 +196,7 @@ bool Base58::decode (const char* psz, Blob& vchRet, const char* pAlpha) break; } - bnChar.setuint (p1 - pAlpha); + bnChar.setuint (p1 - alphabet.chars()); if (!BN_mul (&bn, &bn, &bn58, pctx)) throw bignum_error ("DecodeBase58 : BN_mul failed"); @@ -147,7 +214,7 @@ bool Base58::decode (const char* psz, Blob& vchRet, const char* pAlpha) // Restore leading zeros int nLeadingZeros = 0; - for (const char* p = psz; *p == pAlpha[0]; p++) + for (const char* p = psz; *p == alphabet.chars()[0]; p++) nLeadingZeros++; vchRet.assign (nLeadingZeros + vchTmp.size (), 0); @@ -162,11 +229,9 @@ bool Base58::decode (const std::string& str, Blob& vchRet) return decode (str.c_str (), vchRet); } -bool Base58::decodeWithCheck (const char* psz, Blob& vchRet, const char* pAlphabet) +bool Base58::decodeWithCheck (const char* psz, Blob& vchRet, Alphabet const& alphabet) { - assert (pAlphabet != NULL); - - if (!decode (psz, vchRet, pAlphabet)) + if (!decode (psz, vchRet, alphabet)) return false; if (vchRet.size () < 4) @@ -187,9 +252,9 @@ bool Base58::decodeWithCheck (const char* psz, Blob& vchRet, const char* pAlphab return true; } -bool Base58::decodeWithCheck (const std::string& str, Blob& vchRet, const char* pAlphabet) +bool Base58::decodeWithCheck (const std::string& str, Blob& vchRet, Alphabet const& alphabet) { - return decodeWithCheck (str.c_str (), vchRet, pAlphabet); + return decodeWithCheck (str.c_str (), vchRet, alphabet); } } diff --git a/src/ripple/validators/api/Source.h b/src/ripple/validators/api/Source.h index fbafa669d5..1c1e7bb444 100644 --- a/src/ripple/validators/api/Source.h +++ b/src/ripple/validators/api/Source.h @@ -44,10 +44,15 @@ public: */ virtual ~Source () { } + /** The name of the source, used in diagnostic output. */ virtual String name () = 0; + /** An identifier that uniquely describes the source. + This is used for identification in the database. + */ virtual String uniqueID () = 0; + /** A string that is used to recreate the source from the database entry. */ virtual String createParam () = 0; /** Fetch the most recent list from the Source. @@ -61,14 +66,14 @@ public: bool success; String message; Time expirationTime; - Array list; + std::vector list; }; /** Cancel any pending fetch. The default implementation does nothing. */ virtual void cancel () { } - virtual Result fetch (Journal journal) = 0; + virtual void fetch (Result& result, Journal journal) = 0; }; } diff --git a/src/ripple/validators/impl/Logic.h b/src/ripple/validators/impl/Logic.h index 8a331ad624..5d1776f7d1 100644 --- a/src/ripple/validators/impl/Logic.h +++ b/src/ripple/validators/impl/Logic.h @@ -269,15 +269,19 @@ public: // void addStatic (Source* source) { - m_journal.info << "Addding static source '" << source->name() << "'"; + m_journal.info << "Addding static " << source->name(); ScopedPointer object (source); - Source::Result result (object->fetch (m_journal)); + Source::Result result; + object->fetch (result, m_journal); if (result.success) { SharedState::Access state (m_state); - merge (result.list, source, state); + std::size_t const numAdded ( + merge (result.list, source, state)); + m_journal.info << "Added " << numAdded + << " trusted validators from " << source->name(); } else { @@ -289,7 +293,7 @@ public: // void add (Source* source) { - m_journal.info << "Adding source '" << source->name() << "'"; + m_journal.info << "Adding " << source->name(); { SharedState::Access state (m_state); @@ -303,13 +307,13 @@ public: // Add each entry in the list to the map, incrementing the // reference count if it already exists, and updating fields. // - void merge (Array const& list, + std::size_t merge (std::vector const& list, Source* source, SharedState::Access& state) { std::size_t numAdded (0); for (std::size_t i = 0; i < list.size (); ++i) { - Source::Info const& info (list.getReference (i)); + Source::Info const& info (list [i]); std::pair result ( state->map.emplace (info.publicKey, Validator ())); Validator& validatorInfo (result.first->second); @@ -322,20 +326,19 @@ public: } } - m_journal.info << "Added " << numAdded - << " trusted validators from '" << source->name() << "'"; + return numAdded; } // Decrement the reference count of each item in the list // in the map. // - void remove (Array const& list, + std::size_t remove (std::vector const& list, Source* source, SharedState::Access& state) { std::size_t numRemoved (0); for (std::size_t i = 0; i < list.size (); ++i) { - Source::Info const& info (list.getReference (i)); + Source::Info const& info (list [i]); MapType::iterator iter (state->map.find (info.publicKey)); bassert (iter != state->map.end ()); Validator& validatorInfo (iter->second); @@ -348,8 +351,7 @@ public: } } - m_journal.info << "Removed " << numRemoved - << " trusted validators from '" << source->name() << "'"; + return numRemoved; } //---------------------------------------------------------------------- @@ -410,9 +412,12 @@ public: /** Perform a fetch on the source. */ void fetch (SourceDesc& desc) { - m_journal.info << "fetch " << desc.source->name(); + Source* const source (desc.source); - Source::Result result (desc.source->fetch (m_journal)); + m_journal.info << "fetch " << source->name(); + + Source::Result result; + source->fetch (result, m_journal); // Reset fetch timer for the source. desc.whenToFetch = Time::getCurrentTime () + @@ -423,13 +428,22 @@ public: SharedState::Access state (m_state); // Add the new source info to the map - merge (result.list, desc.source, state); + std::size_t const numAdded ( + merge (result.list, source, state)); // Swap lists desc.result.swapWith (result); // Remove the old source info from the map - remove (result.list, desc.source, state); + std::size_t const numRemoved ( + remove (result.list, source, state)); + + if (numAdded > numRemoved) + m_journal.info << "Added " << (numAdded - numRemoved) << + " trusted validators from " << source->name(); + else if (numRemoved > numAdded) + m_journal.info << "Removed " << (numRemoved - numAdded) << + " trusted validators from " << source->name(); // See if we need to rebuild checkChosen (); diff --git a/src/ripple/validators/impl/Manager.cpp b/src/ripple/validators/impl/Manager.cpp index 58c7e6c3a0..24f202ef5f 100644 --- a/src/ripple/validators/impl/Manager.cpp +++ b/src/ripple/validators/impl/Manager.cpp @@ -188,14 +188,19 @@ public: void addStrings (String name, StringArray const& stringArray) { - addStaticSource (SourceStrings::New ( - name, stringArray)); + if (stringArray.size() > 0) + { + addStaticSource (SourceStrings::New (name, stringArray)); + } + else + { + m_journal.debug << "Static source '" << name << "' is empty."; + } } void addFile (File const& file) { - //addStaticSource (SourceFile::New (file)); - addSource (SourceFile::New (file)); + addStaticSource (SourceFile::New (file)); } void addURL (URL const& url) diff --git a/src/ripple/validators/impl/Source.cpp b/src/ripple/validators/impl/Source.cpp index 7fab99a432..ba808df367 100644 --- a/src/ripple/validators/impl/Source.cpp +++ b/src/ripple/validators/impl/Source.cpp @@ -30,7 +30,7 @@ void Source::Result::swapWith (Result& other) { std::swap (success, other.success); std::swap (message, other.message); - list.swapWith (other.list); + list.swap (other.list); } } diff --git a/src/ripple/validators/impl/SourceFile.cpp b/src/ripple/validators/impl/SourceFile.cpp index 30b4e18427..67465e55ba 100644 --- a/src/ripple/validators/impl/SourceFile.cpp +++ b/src/ripple/validators/impl/SourceFile.cpp @@ -36,7 +36,7 @@ public: String name () { - return "File :'" + m_file.getFullPathName () + "'"; + return "File: '" + m_file.getFullPathName () + "'"; } String uniqueID () @@ -48,11 +48,9 @@ public: { return m_file.getFullPathName (); } - - Result fetch (Journal journal) + + void fetch (Result& result, Journal journal) { - Result result; - int64 const fileSize (m_file.getSize ()); if (fileSize != 0) @@ -68,6 +66,8 @@ public: if (amountRead == fileSize) { + Utilities::ParseResultLine lineFunction (result, journal); + Utilities::processLines (buffer.begin(), buffer.end(), lineFunction); } } else @@ -79,8 +79,6 @@ public: { // file doesn't exist } - - return result; } private: diff --git a/src/ripple/validators/impl/SourceStrings.cpp b/src/ripple/validators/impl/SourceStrings.cpp index c2485306de..9020f12e79 100644 --- a/src/ripple/validators/impl/SourceStrings.cpp +++ b/src/ripple/validators/impl/SourceStrings.cpp @@ -51,11 +51,9 @@ public: return String::empty; } - Result fetch (Journal journal) + void fetch (Result& result, Journal journal) { - Result result; - - result.list.ensureStorageAllocated (m_strings.size ()); + result.list.reserve (m_strings.size ()); for (int i = 0; i < m_strings.size (); ++i) { @@ -65,7 +63,6 @@ public: result.success = result.list.size () > 0; result.expirationTime = Time::getCurrentTime () + RelativeTime::hours (24); - return result; } private: diff --git a/src/ripple/validators/impl/SourceURL.cpp b/src/ripple/validators/impl/SourceURL.cpp index 85ff2950d3..bf00ad97c5 100644 --- a/src/ripple/validators/impl/SourceURL.cpp +++ b/src/ripple/validators/impl/SourceURL.cpp @@ -49,17 +49,22 @@ public: return m_url.full(); } - Result fetch (Journal journal) + void fetch (Result& result, Journal journal) { - Result result; - ScopedPointer client (HTTPClientBase::New ()); HTTPClientBase::result_type httpResult (client->get (m_url)); if (httpResult.first == 0) { - //Logger::outputDebugString (httpResult.second->toString ()); +#if 0 + journal.debug << std::endl << httpResult.second->toString (); + journal.debug << std::endl << httpResult.second->body().to_string(); +#endif + + Utilities::ParseResultLine lineFunction (result, journal); + std::string const s (httpResult.second->body().to_string()); + Utilities::processLines (s.begin(), s.end(), lineFunction); } else { @@ -67,8 +72,6 @@ public: "HTTP GET to " << m_url.full().toStdString() << " failed: '" << httpResult.first.message () << "'"; } - - return result; } private: @@ -80,10 +83,7 @@ private: SourceURL* SourceURL::New ( URL const& url) { - ScopedPointer object ( - new SourceURLImp (url)); - - return object.release (); + return new SourceURLImp (url); } } diff --git a/src/ripple/validators/impl/StoreSqdb.cpp b/src/ripple/validators/impl/StoreSqdb.cpp index 7ac83f2aca..229e2a0b84 100644 --- a/src/ripple/validators/impl/StoreSqdb.cpp +++ b/src/ripple/validators/impl/StoreSqdb.cpp @@ -36,6 +36,9 @@ Error StoreSqdb::open (File const& file) if (!error) error = init (); + if (!error) + error = update (); + return error; } @@ -116,20 +119,16 @@ void StoreSqdb::update (SourceDesc& desc, bool updateFetchResults) if (! error && updateFetchResults) { // Delete the previous data set - { - sqdb::statement st = (m_session.prepare << - "DELETE FROM ValidatorsSourceInfo WHERE " - " sourceID = ?; " - ,sqdb::use (sourceID) - ); - - st.execute_and_fetch (error); - } + m_session.once (error) << + "DELETE FROM ValidatorsSourceInfo WHERE " + " sourceID = ?; " + ,sqdb::use (sourceID) + ; // Insert the new data set if (! error) { - std::string publicKey; + std::string publicKeyString; String label; sqdb::statement st = (m_session.prepare << @@ -141,15 +140,15 @@ void StoreSqdb::update (SourceDesc& desc, bool updateFetchResults) " ?, ?, ? " ");" ,sqdb::use (sourceID) - ,sqdb::use (publicKey) + ,sqdb::use (publicKeyString) ,sqdb::use (label) ); - Array & list (desc.result.list); + std::vector & list (desc.result.list); for (std::size_t i = 0; ! error && i < list.size(); ++i) { - Source::Info& info (list.getReference(i)); - publicKey = Utilities::publicKeyToString (info.publicKey); + Source::Info& info (list [i]); + publicKeyString = info.publicKey.to_string (); label = list[i].label; st.execute_and_fetch (error); } @@ -255,11 +254,11 @@ void StoreSqdb::selectList (SourceDesc& desc) bassert (desc.result.list.size() == 0); // Pre-allocate some storage - desc.result.list.ensureStorageAllocated (count); + desc.result.list.reserve (count); // Prepare the select { - std::string publicKey; + std::string publicKeyString; std::string label; sqdb::statement st = (m_session.prepare << "SELECT " @@ -267,7 +266,7 @@ void StoreSqdb::selectList (SourceDesc& desc) " label " "FROM ValidatorsSourceInfo WHERE " " sourceID = ? " - ,sqdb::into (publicKey) + ,sqdb::into (publicKeyString) ,sqdb::into (label) ,sqdb::use (sourceID) ); @@ -278,9 +277,20 @@ void StoreSqdb::selectList (SourceDesc& desc) do { Source::Info info; - info.publicKey = Utilities::stringToPublicKey (publicKey); - info.label = label; - desc.result.list.add (info); + std::pair result ( + RipplePublicKey::from_string (publicKeyString)); + if (result.second) + { + bassert (result.first.to_string() == publicKeyString); + info.publicKey = result.first; + info.label = label; + desc.result.list.push_back (info); + } + else + { + m_journal.error << "Invalid public key '" << + publicKeyString << "' found in database"; + } } while (st.fetch (error)); } @@ -294,6 +304,81 @@ void StoreSqdb::selectList (SourceDesc& desc) //-------------------------------------------------------------------------- +// Update the database for the current schema +Error StoreSqdb::update () +{ + Error error; + + sqdb::transaction tr (m_session); + + // Get the version from the database + int version (0); + if (! error) + { + m_session.once (error) << + "SELECT " + " version " + "FROM SchemaVersion WHERE " + " name = 'Validators' " + ,sqdb::into (version) + ; + + if (! m_session.got_data ()) + { + // pre-dates the "SchemaVersion" table + version = 0; + } + } + + if (! error && version != currentSchemaVersion) + { + m_journal.info << "Updating old database version " << version; + } + + // Update database based on version + if (! error && version < 1) + { + // Delete all the old data since its likely wrong + m_session.once (error) << + "DELETE FROM ValidatorsSource"; + + if (! error) + { + m_session.once (error) << + "DELETE FROM ValidatorsSourceInfo"; + } + } + + // Update the version to the current version + if (! error) + { + int const version (currentSchemaVersion); + + m_session.once (error) << + "INSERT OR REPLACE INTO SchemaVersion ( " + " name, " + " version " + ") VALUES ( " + " 'Validators', ? " + "); " + ,sqdb::use (version) + ; + } + + if (! error) + { + error = tr.commit(); + } + + if (error) + { + tr.rollback (); + report (error, __FILE__, __LINE__); + } + + return error; +} + Error StoreSqdb::init () { Error error; @@ -342,6 +427,23 @@ Error StoreSqdb::init () ; } + if (! error) + { + // This table maps component names like "Validators" to their + // corresponding schema version number. This method allows us + // to keep all logic data in one database, or each in its own + // database, or in any grouping of databases, while still being + // able to let an individual component know what version of its + // schema it is opening. + // + m_session.once (error) << + "CREATE TABLE IF NOT EXISTS SchemaVersion ( " + " name TEXT PRIMARY KEY, " + " version INTEGER" + ");" + ; + } + if (! error) { error = tr.commit(); diff --git a/src/ripple/validators/impl/StoreSqdb.h b/src/ripple/validators/impl/StoreSqdb.h index 34ba4c3069..377dd29acc 100644 --- a/src/ripple/validators/impl/StoreSqdb.h +++ b/src/ripple/validators/impl/StoreSqdb.h @@ -29,6 +29,12 @@ class StoreSqdb , public LeakChecked { public: + enum + { + // This affects the format of the data! + currentSchemaVersion = 1 + }; + explicit StoreSqdb (Journal journal = Journal()); ~StoreSqdb (); @@ -45,6 +51,7 @@ private: bool select (SourceDesc& desc); void selectList (SourceDesc& desc); + Error update (); Error init (); Journal m_journal; diff --git a/src/ripple/validators/impl/Tests.cpp b/src/ripple/validators/impl/Tests.cpp index d4d2862d83..e1d0551bfa 100644 --- a/src/ripple/validators/impl/Tests.cpp +++ b/src/ripple/validators/impl/Tests.cpp @@ -121,23 +121,19 @@ public: return String::empty; } - Result fetch (Journal) + void fetch (Result& result, Journal) { - Result result; - result.success = true; result.message = String::empty; - result.list.ensureStorageAllocated (numberOfTestValidators); + result.list.reserve (numberOfTestValidators); for (uint32 i = m_start ; i < m_end; ++i) { Info info; info.publicKey = RipplePublicKey::createFromInteger (i); info.label = String::fromNumber (i); - result.list.add (info); + result.list.push_back (info); } - - return result; } String m_name; diff --git a/src/ripple/validators/impl/Utilities.cpp b/src/ripple/validators/impl/Utilities.cpp index a53d233569..ef3f85155f 100644 --- a/src/ripple/validators/impl/Utilities.cpp +++ b/src/ripple/validators/impl/Utilities.cpp @@ -24,22 +24,25 @@ struct Utilities::Helpers { // Matches a validator info line. // - static boost::regex& reInfo () + static boost::regex const& reInfo () { // e.g. // // n9KorY8QtTdRx7TVDpwnG9NvyxsDwHUKUEeDLY3AkiGncVaSXZi5 Comment Text // static boost::regex re ( - "^" // start of line - "(?:\\h*)" // horiz-white (optional) - "([^\\h\\v]+)" // [1] non-white run - "(?:\\h*)" // horiz-white (optional) - "([^\\h\\v]*)" // [2] any text (optional) - "$" // end of line + "\\G" // end of last match (or start) + "(?:[\\v\\h]*)" // white (optional) + "([^\\h\\v]+)" // [1] non-white run + "(?:\\h*)" // horiz-white (optional) + "([^\\v]*?)" // [2] non vert-white text (optional) + "(?:\\h*)" // white run (optional) + "(?:\\v*)" // vert-white (optional) - , boost::regex::perl | - boost::regex_constants::match_flags::match_not_dot_null + //"(?:\\')" // buffer boundary + + , boost::regex::perl + //| boost::regex_constants::match_flags::match_not_dot_newline ); return re; @@ -47,7 +50,7 @@ struct Utilities::Helpers // Matches a comment or whitespace line. // - static boost::regex& reComment () + static boost::regex const& reComment () { // e.g. // @@ -87,23 +90,18 @@ bool Utilities::parseInfoLine ( RippleAddress deprecatedPublicKey; - // VFALCO NOTE These bool return values are poorlydocumented - // - if (deprecatedPublicKey.setSeedGeneric (encodedKey)) - { - // expected a domain or public key but got a generic seed? - // log? - } - else if (deprecatedPublicKey.setNodePublic (encodedKey)) + if (deprecatedPublicKey.setNodePublic (encodedKey)) { // We got a public key. RipplePublicKey publicKey (deprecatedPublicKey); + info.label = commentText; + info.publicKey = publicKey; success = true; } else { // Some other junk. - // log? + journal.error << "Invalid RipplePublicKey: '" << encodedKey << "'"; } } else if (boost::regex_match (line, match, Helpers::reComment ())) @@ -113,54 +111,9 @@ bool Utilities::parseInfoLine ( else { // Log a warning about a parsing error + journal.error << "Invalid Validators source line:" << std::endl << line; } -#if 0 - static boost::regex reReferral ("\\`\\s*(\\S+)(?:\\s+(.+))?\\s*\\'"); - - if (!boost::regex_match (strReferral, smMatch, reReferral)) - { - WriteLog (lsWARNING, UniqueNodeList) << str (boost::format ("Bad validator: syntax error: %s: %s") % strSite % strReferral); - } - else - { - std::string strRefered = smMatch[1]; - std::string strComment = smMatch[2]; - RippleAddress naValidator; - - if (naValidator.setSeedGeneric (strRefered)) - { - - WriteLog (lsWARNING, UniqueNodeList) << str (boost::format ("Bad validator: domain or public key required: %s %s") % strRefered % strComment); - } - else if (naValidator.setNodePublic (strRefered)) - { - // A public key. - // XXX Schedule for CAS lookup. - nodeAddPublic (naValidator, vsWhy, strComment); - - WriteLog (lsINFO, UniqueNodeList) << str (boost::format ("Node Public: %s %s") % strRefered % strComment); - - if (naNodePublic.isValid ()) - vstrValues.push_back (str (boost::format ("('%s',%d,'%s')") % strNodePublic % iValues % naValidator.humanNodePublic ())); - - iValues++; - } - else - { - // A domain: need to look it up. - nodeAddDomain (strRefered, vsWhy, strComment); - - WriteLog (lsINFO, UniqueNodeList) << str (boost::format ("Node Domain: %s %s") % strRefered % strComment); - - if (naNodePublic.isValid ()) - vstrValues.push_back (str (boost::format ("('%s',%d,%s)") % strNodePublic % iValues % sqlEscape (strRefered))); - - iValues++; - } - } -#endif - return success; } @@ -171,15 +124,13 @@ void Utilities::parseResultLine ( std::string const& line, Journal journal) { - bool success = false; + Source::Info info; - if (! success) + bool success = parseInfoLine (info, line, journal); + if (success) { - Source::Info info; - - success = parseInfoLine (info, line, journal); - if (success) - result.list.add (info); + result.list.push_back (info); + result.success = true; } } @@ -249,19 +200,6 @@ Time Utilities::stringToTime (String s) return Time (0); } -std::string Utilities::publicKeyToString (RipplePublicKey const& publicKey) -{ - std::string s (RipplePublicKey::size, ' '); - std::copy (publicKey.cbegin(), publicKey.cend(), s.begin()); - return s; -} - -RipplePublicKey Utilities::stringToPublicKey (std::string const& s) -{ - bassert (s.size() == RipplePublicKey::size); - return RipplePublicKey (); -} - //------------------------------------------------------------------------------ } diff --git a/src/ripple/validators/impl/Utilities.h b/src/ripple/validators/impl/Utilities.h index 76cc928b8a..a0ac2c05e1 100644 --- a/src/ripple/validators/impl/Utilities.h +++ b/src/ripple/validators/impl/Utilities.h @@ -23,31 +23,93 @@ namespace ripple { namespace Validators { -/** Common code for Validators classes. -*/ +/** Common code for Validators classes. */ class Utilities { public: typedef std::vector Strings; -#if 0 - /** Parse a ConstBufferSequence of newline delimited text into strings. - This works incrementally. - */ - template - static void parseLines (Strings& lines, ConstBufferSequence const& buffers) + /** A suitable LineFunction for parsing items into a fetch result. */ + class ParseResultLine { - for (typename ConstBufferSequence::const_iterator iter = buffers.begin (); - iter != buffers.end (); ++iter) - parserLines (lines, *iter); - } + public: + ParseResultLine (Source::Result& result, Journal journal) + : m_result (&result) + , m_journal (journal) + { } - /** Turn a linear buffer of newline delimited text into strings. - This can be called incrementally, i.e. successive calls with - multiple buffer segments. + template + void operator() (BidirectionalIterator first, BidirectionalIterator last) + { + std::string s (first, last); + Utilities::parseResultLine (*m_result, s, m_journal); + } + + private: + Source::Result* m_result; + Journal m_journal; + }; + + /** UnaryPredicate for breaking up lines. + This returns `true` for the first non-vertical whitespace character that + follows a vertical whitespace character. */ - static void parseLines (Strings& lines, char const* buf, std::size_t bytes); -#endif + class FollowingVerticalWhite + { + public: + FollowingVerticalWhite () + : m_gotWhite (false) + { + } + + template + static bool isVerticalWhitespace (CharT c) + { + return c == '\r' || c == '\n'; + } + + template + bool operator() (CharT c) + { + if (isVerticalWhitespace (c)) + { + m_gotWhite = true; + return false; + } + else if (m_gotWhite) + { + m_gotWhite = false; + return true; + } + return false; + } + + private: + bool m_gotWhite; + }; + + /** Call LineFunction for each newline-separated line in the input. + LineFunction will be called with this signature: + @code + void LineFunction (BidirectionalIterator first, BidirectionalIterator last) + @endcode + Where first and last mark the beginning and ending of the line. + The last line in the input may or may not contain the trailing newline. + */ + template + static void processLines (BidirectionalIterator first, + BidirectionalIterator last, LineFunction f) + { + for (;;) + { + BidirectionalIterator split (std::find_if ( + first, last, FollowingVerticalWhite ())); + f (first, split); + if (split == last) + break; + first = split; + } + } /** Parse a string into the Source::Result. Invalid or comment lines will be skipped. @@ -67,10 +129,6 @@ public: static String timeToString (Time const& t); static Time stringToTime (String s); - // conversion between RipplePublicKey and String - static std::string publicKeyToString (RipplePublicKey const& publicKey); - static RipplePublicKey stringToPublicKey (std::string const& s); - struct Helpers; /** Parse a string into a Source::Info. diff --git a/src/ripple_app/main/Application.cpp b/src/ripple_app/main/Application.cpp index e50d506b54..174100db31 100644 --- a/src/ripple_app/main/Application.cpp +++ b/src/ripple_app/main/Application.cpp @@ -614,21 +614,26 @@ public: // Initialize the Validators object with Config information. void prepareValidators () { +#if 0 { std::vector const& strings (getConfig().validators); - if (! strings.empty ()) - m_validators->addStrings ("rippled.cfg", strings); + m_validators->addStrings ("rippled.cfg", strings); } +#endif +#if 1 if (! getConfig().getValidatorsURL().empty()) { m_validators->addURL (getConfig().getValidatorsURL()); } +#endif +#if 0 if (getConfig().getValidatorsFile() != File::nonexistent ()) { m_validators->addFile (getConfig().getValidatorsFile()); } +#endif } //-------------------------------------------------------------------------- diff --git a/src/ripple_data/crypto/Base58Data.cpp b/src/ripple_data/crypto/Base58Data.cpp index a818a2ee2e..daa3a7c958 100644 --- a/src/ripple_data/crypto/Base58Data.cpp +++ b/src/ripple_data/crypto/Base58Data.cpp @@ -62,10 +62,10 @@ void CBase58Data::SetData (int nVersionIn, const unsigned char* pbegin, const un SetData (nVersionIn, (void*)pbegin, pend - pbegin); } -bool CBase58Data::SetString (const char* psz, unsigned char version, const char* pAlphabet) +bool CBase58Data::SetString (const char* psz, unsigned char version, Base58::Alphabet const& alphabet) { Blob vchTemp; - Base58::decodeWithCheck (psz, vchTemp, pAlphabet); + Base58::decodeWithCheck (psz, vchTemp, alphabet); if (vchTemp.empty () || vchTemp[0] != version) { diff --git a/src/ripple_data/crypto/Base58Data.h b/src/ripple_data/crypto/Base58Data.h index 7d5465b380..c5cd2bad8a 100644 --- a/src/ripple_data/crypto/Base58Data.h +++ b/src/ripple_data/crypto/Base58Data.h @@ -47,7 +47,7 @@ protected: void SetData (int nVersionIn, const unsigned char* pbegin, const unsigned char* pend); public: - bool SetString (const char* psz, unsigned char version, const char* pAlphabet = Base58::getCurrentAlphabet ()); + bool SetString (const char* psz, unsigned char version, Base58::Alphabet const& alphabet = Base58::getCurrentAlphabet ()); bool SetString (const std::string& str, unsigned char version); std::string ToString () const; diff --git a/src/ripple_data/protocol/RippleAddress.cpp b/src/ripple_data/protocol/RippleAddress.cpp index faf8b85389..be4e4a1c09 100644 --- a/src/ripple_data/protocol/RippleAddress.cpp +++ b/src/ripple_data/protocol/RippleAddress.cpp @@ -348,7 +348,7 @@ std::string RippleAddress::humanAccountID () const } } -bool RippleAddress::setAccountID (const std::string& strAccountID, const char* pAlphabet) +bool RippleAddress::setAccountID (const std::string& strAccountID, Base58::Alphabet const& alphabet) { if (strAccountID.empty ()) { @@ -358,7 +358,7 @@ bool RippleAddress::setAccountID (const std::string& strAccountID, const char* p } else { - mIsValid = SetString (strAccountID.c_str (), VER_ACCOUNT_ID, pAlphabet); + mIsValid = SetString (strAccountID.c_str (), VER_ACCOUNT_ID, alphabet); } return mIsValid; diff --git a/src/ripple_data/protocol/RippleAddress.h b/src/ripple_data/protocol/RippleAddress.h index 88c41f50b2..9d27bf7965 100644 --- a/src/ripple_data/protocol/RippleAddress.h +++ b/src/ripple_data/protocol/RippleAddress.h @@ -95,7 +95,7 @@ public: std::string humanAccountID () const; - bool setAccountID (const std::string& strAccountID, const char* pAlphabet = Base58::getCurrentAlphabet ()); + bool setAccountID (const std::string& strAccountID, Base58::Alphabet const& alphabet = Base58::getCurrentAlphabet ()); void setAccountID (const uint160& hash160In); static RippleAddress createAccountID (const std::string& strAccountID)