diff --git a/Builds/VisualStudio2015/RippleD.vcxproj b/Builds/VisualStudio2015/RippleD.vcxproj index a33e3e9c0e..9e86841349 100644 --- a/Builds/VisualStudio2015/RippleD.vcxproj +++ b/Builds/VisualStudio2015/RippleD.vcxproj @@ -1532,6 +1532,10 @@ True True + + True + True + True True @@ -1552,18 +1556,14 @@ - - True - True - - - True True + + True True @@ -1740,6 +1740,10 @@ True True + + True + True + @@ -3371,22 +3375,6 @@ True True - - True - True - - - True - True - - - True - True - - - True - True - True True diff --git a/Builds/VisualStudio2015/RippleD.vcxproj.filters b/Builds/VisualStudio2015/RippleD.vcxproj.filters index bc549509ac..944b9f99f3 100644 --- a/Builds/VisualStudio2015/RippleD.vcxproj.filters +++ b/Builds/VisualStudio2015/RippleD.vcxproj.filters @@ -2271,6 +2271,9 @@ ripple\app\misc\impl + + ripple\app\misc\impl + ripple\app\misc @@ -2295,18 +2298,15 @@ ripple\app\misc - - ripple\app\misc - - - ripple\app\misc - ripple\app\misc ripple\app\misc + + ripple\app\misc + ripple\app\paths @@ -2463,6 +2463,9 @@ ripple\app\tests + + ripple\app\tests + ripple\app\tx @@ -4008,18 +4011,6 @@ ripple\rpc\handlers - - ripple\rpc\handlers - - - ripple\rpc\handlers - - - ripple\rpc\handlers - - - ripple\rpc\handlers - ripple\rpc\handlers diff --git a/doc/rippled-example.cfg b/doc/rippled-example.cfg index 47de42bf29..e122b00956 100644 --- a/doc/rippled-example.cfg +++ b/doc/rippled-example.cfg @@ -503,17 +503,11 @@ # # [validators] # -# List of nodes to always accept as validators. Nodes are specified by domain -# or public key. -# -# For domains, rippled will probe for https web servers at the specified -# domain in the following order: ripple.DOMAIN, www.DOMAIN, DOMAIN -# -# For public key entries, a comment may optionally be specified after adding -# a space to the public key. +# List of the validation public keys of nodes to always accept as validators. +# A comment may, optionally, be associated with each entry, separated by +# whitespace from the validation public key. # # Examples: -# ripple.com # n9KorY8QtTdRx7TVDpwnG9NvyxsDwHUKUEeDLY3AkiGncVaSXZi5 # n9MqiExBcoG19UXwoLjBJnhsxEhAZMuWwJDRdkyDz1EkEkwzQTNt John Doe # @@ -521,33 +515,27 @@ # # [validators_file] # -# Path to file contain a list of nodes to always accept as validators. Use -# this to specify a file other than this file to manage your validators list. +# Path to a file that contains the validation public keys of nodes to always +# accept as validators. A comment may, optionally, be associated with each +# entry, separated by whitespace from the validation public key. # -# If this entry is not present or empty and no nodes from previous runs were -# found in the database, rippled will look for a validators.txt in the config -# directory. If not found there, it will attempt to retrieve the file from -# the [validators_site] web site. -# -# After specifying a different [validators_file] or changing the contents of -# the validators file, issue a RPC unl_load command to have rippled load the -# file. +# The contents of the file should include a [validators] entry, followed by +# a list of validation public keys of nodes to always accept as validators, +# one per line, optionally followed by a comment separated by whitespace: # # Specify the file by specifying its full path. # # Examples: -# C:/home/johndoe/ripple/validators.txt -# /home/johndoe/ripple/validators.txt -# -# -# -# [validators_site] -# -# Specifies where to find validators.txt for UNL boostrapping and RPC -# unl_network command. -# -# Example: ripple.com +# /home/ripple/validators.txt +# C:/home/ripple/validators.txt # +# Example content: +# [validators] +# n949f75evCHwgyP4fPVgaHqNHxUVN15PsJEZ3B3HnXPcPjcZAoy7 RL1 +# n9MD5h24qrQqiyBC8aeqqCWvpiBiYQ3jxSr91uiDvmrkyHRdYLUj RL2 +# n9L81uNCaPgtUJfaHh89gmdvXKAmSt5Gdsw2g1iPWaPkAHW5Nm4C RL3 +# n9KiYM9CgngLvtRCQHZwgC2gjpdaZcCcbt3VboxiNFcKuwFVujzS RL4 +# n9LdgEtkmGB9E2h3K4Vp7iGUaKuq23Zr32ehxiU8FWY7xoxbWTSA RL5 # # # [path_search] diff --git a/src/ripple/app/main/Application.cpp b/src/ripple/app/main/Application.cpp index 4ed53bdddd..428d628774 100644 --- a/src/ripple/app/main/Application.cpp +++ b/src/ripple/app/main/Application.cpp @@ -42,9 +42,9 @@ #include #include #include +#include #include #include -#include #include #include #include @@ -323,7 +323,7 @@ public: TaggedCache m_acceptedLedgerCache; std::unique_ptr m_networkOPs; std::unique_ptr cluster_; - std::unique_ptr m_deprecatedUNL; + std::unique_ptr validators_; std::unique_ptr serverHandler_; std::unique_ptr m_amendmentTable; std::unique_ptr mFeeTrack; @@ -446,8 +446,11 @@ public: *m_jobQueue, *m_ledgerMaster, *m_jobQueue, logs_->journal("NetworkOPs"))) - // VFALCO NOTE LocalCredentials starts the deprecated UNL service - , m_deprecatedUNL (make_UniqueNodeList (*this, *m_jobQueue)) + , cluster_ (std::make_unique ( + logs_->journal("Overlay"))) + + , validators_ (std::make_unique ( + logs_->journal("UniqueNodeList"))) , serverHandler_ (make_ServerHandler (*this, *m_networkOPs, get_io_service (), *m_jobQueue, *m_networkOPs, *m_resourceManager, *m_collectorManager)) @@ -658,9 +661,9 @@ public: return *mValidations; } - UniqueNodeList& getUNL () override + ValidatorList& validators () override { - return *m_deprecatedUNL; + return *validators_; } Cluster& cluster () override @@ -1001,22 +1004,23 @@ void ApplicationImp::setup() m_orderBookDB.setup (getLedgerMaster ().getCurrentLedger ()); - cluster_ = make_Cluster (config (), logs_->journal("Overlay")); + if (!cluster_->load (config().section(SECTION_CLUSTER_NODES))) + { + m_journal.fatal << "Invalid entry in cluster configuration."; + throw std::exception(); + } + + if (!validators_->load (config().section (SECTION_VALIDATORS))) + { + m_journal.fatal << "Invalid entry in validator configuration."; + throw std::exception(); + } + + if (validators_->size () == 0 && !config_->RUN_STANDALONE) + m_journal.warning << "No validators are configured."; - // Begin validation and ip maintenance. - // - // - LocalCredentials maintains local information: including identity - // - and network connection persistence information. - // - // VFALCO NOTE this starts the UNL m_localCredentials.start (); - // - // Set up UNL. - // - if (!config_->RUN_STANDALONE) - getUNL ().nodeBootstrap (); - m_nodeStore->tune (config_->getSize (siNodeCacheSize), config_->getSize (siNodeCacheAge)); m_ledgerMaster->tune (config_->getSize (siLedgerSize), config_->getSize (siLedgerAge)); family().treecache().setTargetSize (config_->getSize (siTreeCacheSize)); diff --git a/src/ripple/app/main/Application.h b/src/ripple/app/main/Application.h index 9c5bdb510c..4a9645c3d5 100644 --- a/src/ripple/app/main/Application.h +++ b/src/ripple/app/main/Application.h @@ -45,7 +45,6 @@ class HashRouter; class Logs; class LoadFeeTrack; class LocalCredentials; -class UniqueNodeList; class JobQueue; class InboundLedgers; class InboundTransactions; @@ -64,6 +63,7 @@ class TimeKeeper; class TransactionMaster; class TxQ; class Validations; +class ValidatorList; class Cluster; class DatabaseCon; @@ -117,7 +117,7 @@ public: virtual LoadManager& getLoadManager () = 0; virtual Overlay& overlay () = 0; virtual TxQ& getTxQ() = 0; - virtual UniqueNodeList& getUNL () = 0; + virtual ValidatorList& validators () = 0; virtual Cluster& cluster () = 0; virtual Validations& getValidations () = 0; virtual NodeStore::Database& getNodeStore () = 0; diff --git a/src/ripple/app/main/LocalCredentials.cpp b/src/ripple/app/main/LocalCredentials.cpp index c17f8100f7..f52f2fcaea 100644 --- a/src/ripple/app/main/LocalCredentials.cpp +++ b/src/ripple/app/main/LocalCredentials.cpp @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include @@ -52,8 +51,6 @@ void LocalCredentials::start () if (!app_.config().QUIET) std::cerr << "NodeIdentity: " << mNodePublicKey.humanNodePublic () << std::endl; - - app_.getUNL ().start (); } // Retrieve network identity. diff --git a/src/ripple/app/misc/NetworkOPs.cpp b/src/ripple/app/misc/NetworkOPs.cpp index 3c3a6f62fe..ff3bd705a6 100644 --- a/src/ripple/app/misc/NetworkOPs.cpp +++ b/src/ripple/app/misc/NetworkOPs.cpp @@ -37,11 +37,11 @@ #include #include #include +#include #include #include -#include +#include #include -#include #include #include #include diff --git a/src/ripple/app/misc/UniqueNodeList.cpp b/src/ripple/app/misc/UniqueNodeList.cpp deleted file mode 100644 index 5b5a046771..0000000000 --- a/src/ripple/app/misc/UniqueNodeList.cpp +++ /dev/null @@ -1,2289 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ripple { - -// XXX Dynamically limit fetching by distance. -// XXX Want a limit of 2000 validators. - -// Guarantees minimum throughput of 1 node per second. -#define NODE_FETCH_JOBS 10 -#define NODE_FETCH_SECONDS 10 -#define NODE_FILE_BYTES_MAX (50<<10) // 50k - -// Wait for validation information to be stable before scoring. -// #define SCORE_DELAY_SECONDS 20 -#define SCORE_DELAY_SECONDS 5 - -// Don't bother propagating past this number of rounds. -#define SCORE_ROUNDS 10 - -// VFALCO TODO Replace macros with language constructs -#define VALIDATORS_FETCH_SECONDS 30 -#define VALIDATORS_FILE_BYTES_MAX (50 << 10) - -// Gather string constants. -#define SECTION_CURRENCIES "currencies" -#define SECTION_DOMAIN "domain" -#define SECTION_IPS "ips" -#define SECTION_IPS_URL "ips_url" -#define SECTION_PUBLIC_KEY "validation_public_key" -#define SECTION_VALIDATORS "validators" -#define SECTION_VALIDATORS_URL "validators_url" - -// Limit pollution of database. -// YYY Move to config file. -#define REFERRAL_VALIDATORS_MAX 50 -#define REFERRAL_IPS_MAX 50 - -template -static -std::string -strJoin (Iterator first, Iterator last, std::string strSeparator) -{ - std::ostringstream ossValues; - - for (Iterator start = first; first != last; ++first) - { - ossValues << str (boost::format ("%s%s") % (start == first ? "" : strSeparator) % *first); - } - - return ossValues.str (); -} - -template -void selectBlobsIntoStrings ( - soci::session& s, - String&& sql, - std::vector, I>>& columns) -{ - columns.clear (); - columns.reserve (32); - - std::vector blobs; - blobs.reserve (I); - for (int i = 0; i < I; ++i) - blobs.emplace_back (s); - std::array indicators; - std::string str; - soci::statement st = [&] - { - auto&& tmp = s.prepare << sql; - for (int i = 0; i < I; ++i) - tmp.operator, (soci::into (blobs[i], indicators[i])); - return tmp; - }(); - - st.execute (); - while (st.fetch ()) - { - columns.emplace_back (); - for (int i = 0; i < I; ++i) - { - if (soci::i_ok == indicators[i]) - { - convert (blobs[i], str); - columns.back ()[i] = str; - } - } - } -} - -template -void selectBlobsIntoStrings ( - soci::session& s, - String&& sql, - std::vector, boost::optional>>& columns) -{ - columns.clear (); - columns.reserve (32); - - soci::blob blob(s); - soci::indicator ind; - boost::optional other; - std::string str; - soci::statement st = - (s.prepare << sql, soci::into(blob, ind), soci::into(other)); - - st.execute (); - while (st.fetch ()) - { - columns.emplace_back (); - if (soci::i_ok == ind) - { - convert (blob, str); - get<0>(columns.back ()) = str; - } - get<1>(columns.back ()) = other; - } -} - -//------------------------------------------------------------------------------ - -class UniqueNodeListImp - : public UniqueNodeList - , public beast::DeadlineTimer::Listener -{ -private: - struct seedDomain - { - std::string strDomain; - RippleAddress naPublicKey; - ValidatorSource vsSource; - boost::posix_time::ptime tpNext; - boost::posix_time::ptime tpScan; - boost::posix_time::ptime tpFetch; - uint256 iSha256; - std::string strComment; - }; - - struct seedNode - { - RippleAddress naPublicKey; - ValidatorSource vsSource; - boost::posix_time::ptime tpNext; - boost::posix_time::ptime tpScan; - boost::posix_time::ptime tpFetch; - uint256 iSha256; - std::string strComment; - }; - - // Used to distribute scores. - struct scoreNode - { - int iScore; - int iRoundScore; - int iRoundSeed; - int iSeen; - std::string strValidator; // The public key. - std::vector viReferrals; - }; - -private: - Application& app_; - - std::mutex mFetchLock; - std::recursive_mutex mUNLLock; - - // VFALCO TODO Replace ptime with beast::Time - // Misc persistent information - boost::posix_time::ptime mtpScoreUpdated; - boost::posix_time::ptime mtpFetchUpdated; - - // XXX Make this faster, make this the contents vector unsigned char or raw public key. - // XXX Contents needs to based on score. - hash_set mUNL; - hash_map ephemeralValidatorKeys_; - - boost::posix_time::ptime mtpScoreNext; // When to start scoring. - boost::posix_time::ptime mtpScoreStart; // Time currently started scoring. - beast::DeadlineTimer m_scoreTimer; // Timer to start scoring. - - int mFetchActive; // Count of active fetches. - - boost::posix_time::ptime mtpFetchNext; // Time of to start next fetch. - beast::DeadlineTimer m_fetchTimer; // Timer to start fetching. - - std::string node_file_name_; - std::string node_file_path_; - - beast::Journal j_; -public: - UniqueNodeListImp (Application& app, Stoppable& parent); - - void onStop(); - - void doScore(); - - void doFetch(); - - void onDeadlineTimer (beast::DeadlineTimer& timer); - - // This is called when the application is started. - // Get update times and start fetching and scoring as needed. - void start(); - - void insertEphemeralKey (PublicKey pk, std::string comment); - void deleteEphemeralKey (PublicKey const& pk); - - // Add a trusted node. Called by RPC or other source. - void nodeAddPublic (RippleAddress const& naNodePublic, ValidatorSource vsWhy, std::string const& strComment); - - // Queue a domain for a single attempt fetch a ripple.txt. - // --> strComment: only used on vsManual - // YYY As a lot of these may happen at once, would be nice to wrap multiple calls in a transaction. - void nodeAddDomain (std::string strDomain, ValidatorSource vsWhy, std::string const& strComment); - - void nodeRemovePublic (RippleAddress const& naNodePublic); - - void nodeRemoveDomain (std::string strDomain); - - void nodeReset(); - - // For debugging, schedule forced scoring. - void nodeScore(); - - bool nodeInUNL (RippleAddress const& naNodePublic); - - void nodeBootstrap(); - - bool nodeLoad (boost::filesystem::path pConfig); - - void nodeNetwork(); - - Json::Value getUnlJson(); - - // For each kind of source, have a starting number of points to be distributed. - int iSourceScore (ValidatorSource vsWhy); - - //-------------------------------------------------------------------------- -private: - // Load information about when we last updated. - bool miscLoad(); - - // Persist update information. - bool miscSave(); - - void trustedLoad(); - - // For a round of scoring we destribute points from a node to nodes it refers to. - // Returns true, iff scores were distributed. - // - bool scoreRound (std::vector& vsnNodes); - - // From SeedDomains and ValidatorReferrals compute scores and update TrustedNodes. - // - // VFALCO TODO Shrink this function, break it up - // - void scoreCompute(); - - // Start a timer to update scores. - // <-- bNow: true, to force scoring for debugging. - void scoreNext (bool bNow); - - // Given a ripple.txt, process it. - // - // VFALCO TODO Can't we take a filename or stream instead of a string? - // - bool responseFetch (std::string const& strDomain, const boost::system::error_code& err, int iStatus, std::string const& strSiteFile); - - // Try to process the next fetch of a ripple.txt. - void fetchNext(); - - // Called when we need to update scores. - void fetchDirty(); - - - void fetchFinish(); - - // Get the ripple.txt and process it. - void fetchProcess (std::string strDomain); - - // Process IniFileSections [validators_url]. - void getValidatorsUrl (RippleAddress const& naNodePublic, - IniFileSections secSite); - - // Process IniFileSections [ips_url]. - // If we have a IniFileSections with a single entry, fetch the url and process it. - void getIpsUrl (RippleAddress const& naNodePublic, IniFileSections secSite); - - - // Given a IniFileSections with IPs, parse and persist it for a validator. - bool responseIps (std::string const& strSite, RippleAddress const& naNodePublic, const boost::system::error_code& err, int iStatus, std::string const& strIpsFile);; - - // After fetching a ripple.txt from a web site, given a IniFileSections with validators, parse and persist it. - bool responseValidators (std::string const& strValidatorsUrl, RippleAddress const& naNodePublic, IniFileSections secSite, std::string const& strSite, const boost::system::error_code& err, int iStatus, std::string const& strValidatorsFile); - - - // Persist the IPs refered to by a Validator. - // --> strSite: source of the IPs (for debugging) - // --> naNodePublic: public key of the validating node. - void processIps (std::string const& strSite, RippleAddress const& naNodePublic, IniFileSections::mapped_type* pmtVecStrIps); - - // Persist ValidatorReferrals. - // --> strSite: source site for display - // --> strValidatorsSrc: source details for display - // --> naNodePublic: remote source public key - not valid for local - // --> vsWhy: reason for adding validator to SeedDomains or SeedNodes. - int processValidators (std::string const& strSite, std::string const& strValidatorsSrc, RippleAddress const& naNodePublic, ValidatorSource vsWhy, IniFileSections::mapped_type const* pmtVecStrValidators); - - // Process a ripple.txt. - void processFile (std::string const& strDomain, RippleAddress const& naNodePublic, IniFileSections secSite); - - // Retrieve a SeedDomain from DB. - bool getSeedDomains (std::string const& strDomain, seedDomain& dstSeedDomain); - - // Persist a SeedDomain. - void setSeedDomains (const seedDomain& sdSource, bool bNext); - - - // Retrieve a SeedNode from DB. - bool getSeedNodes (RippleAddress const& naNodePublic, seedNode& dstSeedNode); - - // Persist a SeedNode. - // <-- bNext: true, to do fetching if needed. - void setSeedNodes (const seedNode& snSource, bool bNext); - - bool validatorsResponse (const boost::system::error_code& err, int iStatus, std::string strResponse); - - // Process a validators.txt. - // --> strSite: source of validators - // --> strValidators: contents of a validators.txt - // - // VFALCO TODO Can't we name this processValidatorList? - // - void nodeProcess (std::string const& strSite, std::string const& strValidators, std::string const& strSource); -}; - -//------------------------------------------------------------------------------ - -UniqueNodeList::UniqueNodeList (Stoppable& parent) - : Stoppable ("UniqueNodeList", parent) -{ -} - -//------------------------------------------------------------------------------ - -UniqueNodeListImp::UniqueNodeListImp (Application& app, Stoppable& parent) - : UniqueNodeList (parent) - , app_ (app) - , m_scoreTimer (this) - , mFetchActive (0) - , m_fetchTimer (this) - , j_ (app.journal ("UniqueNodeList")) -{ - node_file_name_ = std::string (systemName ()) + ".txt"; - node_file_path_ = "/" + node_file_name_; -} - -void UniqueNodeListImp::onStop() -{ - m_fetchTimer.cancel (); - m_scoreTimer.cancel (); - - stopped (); -} - -void UniqueNodeListImp::doScore() -{ - mtpScoreNext = boost::posix_time::ptime (boost::posix_time::not_a_date_time); // Timer not set. - mtpScoreStart = boost::posix_time::second_clock::universal_time (); // Scoring. - - JLOG (j_.trace) << "Scoring: Start"; - - scoreCompute (); - - JLOG (j_.trace) << "Scoring: End"; - - // Save update time. - mtpScoreUpdated = mtpScoreStart; - miscSave (); - - mtpScoreStart = boost::posix_time::ptime (boost::posix_time::not_a_date_time); // Not scoring. - - // Score again if needed. - scoreNext (false); -} - -void UniqueNodeListImp::doFetch() -{ - // Time to check for another fetch. - JLOG (j_.trace) << "fetchTimerHandler"; - fetchNext (); -} - -void UniqueNodeListImp::onDeadlineTimer (beast::DeadlineTimer& timer) -{ - if (timer == m_scoreTimer) - { - app_.getJobQueue ().addJob ( - jtUNL, "UNL.score", - [this] (Job&) { doScore(); }); - } - else if (timer == m_fetchTimer) - { - app_.getJobQueue ().addJob (jtUNL, "UNL.fetch", - [this] (Job&) { doFetch(); }); - } -} - -// This is called when the application is started. -// Get update times and start fetching and scoring as needed. -void UniqueNodeListImp::start() -{ - miscLoad (); - - JLOG (j_.debug) << "Validator fetch updated: " << mtpFetchUpdated; - JLOG (j_.debug) << "Validator score updated: " << mtpScoreUpdated; - - fetchNext (); // Start fetching. - scoreNext (false); // Start scoring. -} - -//-------------------------------------------------------------------------- - -void UniqueNodeListImp::insertEphemeralKey (PublicKey pk, std::string comment) -{ - std::lock_guard sl (mUNLLock); - - ephemeralValidatorKeys_.insert (std::make_pair(std::move(pk), std::move(comment))); -} - -void UniqueNodeListImp::deleteEphemeralKey (PublicKey const& pk) -{ - std::lock_guard sl (mUNLLock); - - ephemeralValidatorKeys_.erase (pk); -} - -//-------------------------------------------------------------------------- - -// Add a trusted node. Called by RPC or other source. -void UniqueNodeListImp::nodeAddPublic (RippleAddress const& naNodePublic, ValidatorSource vsWhy, std::string const& strComment) -{ - seedNode snCurrent; - - bool bFound = getSeedNodes (naNodePublic, snCurrent); - bool bChanged = false; - - if (!bFound) - { - snCurrent.naPublicKey = naNodePublic; - snCurrent.tpNext = boost::posix_time::second_clock::universal_time (); - } - - // Promote source, if needed. - if (!bFound /*|| iSourceScore (vsWhy) >= iSourceScore (snCurrent.vsSource)*/) - { - snCurrent.vsSource = vsWhy; - snCurrent.strComment = strComment; - bChanged = true; - } - - if (vsManual == vsWhy) - { - // A manual add forces immediate scan. - snCurrent.tpNext = boost::posix_time::second_clock::universal_time (); - bChanged = true; - } - - if (bChanged) - setSeedNodes (snCurrent, true); -} - -//-------------------------------------------------------------------------- - -// Queue a domain for a single attempt fetch a ripple.txt. -// --> strComment: only used on vsManual -// YYY As a lot of these may happen at once, would be nice to wrap multiple calls in a transaction. -void UniqueNodeListImp::nodeAddDomain (std::string strDomain, ValidatorSource vsWhy, std::string const& strComment) -{ - boost::trim (strDomain); - boost::to_lower (strDomain); - - // YYY Would be best to verify strDomain is a valid domain. - // JLOG (lsTRACE) << str(boost::format("nodeAddDomain: '%s' %c '%s'") - // % strDomain - // % vsWhy - // % strComment); - - seedDomain sdCurrent; - - bool bFound = getSeedDomains (strDomain, sdCurrent); - bool bChanged = false; - - if (!bFound) - { - sdCurrent.strDomain = strDomain; - sdCurrent.tpNext = boost::posix_time::second_clock::universal_time (); - } - - // Promote source, if needed. - if (!bFound || iSourceScore (vsWhy) >= iSourceScore (sdCurrent.vsSource)) - { - sdCurrent.vsSource = vsWhy; - sdCurrent.strComment = strComment; - bChanged = true; - } - - if (vsManual == vsWhy) - { - // A manual add forces immediate scan. - sdCurrent.tpNext = boost::posix_time::second_clock::universal_time (); - bChanged = true; - } - - if (bChanged) - setSeedDomains (sdCurrent, true); -} - -//-------------------------------------------------------------------------- - -void UniqueNodeListImp::nodeRemovePublic (RippleAddress const& naNodePublic) -{ - { - auto db = app_.getWalletDB ().checkoutDb (); - - *db << str ( - boost::format ("DELETE FROM SeedNodes WHERE PublicKey=%s;") % - sqlEscape (naNodePublic.humanNodePublic ())); - *db << str ( - boost::format ("DELETE FROM TrustedNodes WHERE PublicKey=%s;") % - sqlEscape (naNodePublic.humanNodePublic ())); - } - - // YYY Only dirty on successful delete. - fetchDirty (); - - std::lock_guard sl (mUNLLock); - mUNL.erase (naNodePublic.humanNodePublic ()); -} - -//-------------------------------------------------------------------------- - -void UniqueNodeListImp::nodeRemoveDomain (std::string strDomain) -{ - boost::trim (strDomain); - boost::to_lower (strDomain); - - { - auto db = app_.getWalletDB ().checkoutDb (); - - *db << str (boost::format ("DELETE FROM SeedDomains WHERE Domain=%s;") % sqlEscape (strDomain)); - } - - // YYY Only dirty on successful delete. - fetchDirty (); -} - -//-------------------------------------------------------------------------- - -void UniqueNodeListImp::nodeReset() -{ - { - auto db = app_.getWalletDB ().checkoutDb (); - - *db << "DELETE FROM SeedDomains;"; - *db << "DELETE FROM SeedNodes;"; - } - - fetchDirty (); -} - -//-------------------------------------------------------------------------- - -// For debugging, schedule forced scoring. -void UniqueNodeListImp::nodeScore() -{ - scoreNext (true); -} - -//-------------------------------------------------------------------------- - -bool UniqueNodeListImp::nodeInUNL (RippleAddress const& naNodePublic) -{ - auto const& blob = naNodePublic.getNodePublic(); - PublicKey const pk (Slice(blob.data(), blob.size())); - - std::lock_guard sl (mUNLLock); - - if (ephemeralValidatorKeys_.find (pk) != ephemeralValidatorKeys_.end()) - { - return true; - } - - return mUNL.find (naNodePublic.humanNodePublic()) != mUNL.end(); -} - -//-------------------------------------------------------------------------- - -void UniqueNodeListImp::nodeBootstrap() -{ - int iDomains = 0; - int iNodes = 0; - -#if 0 - { - auto sl (app_.getWalletDB ().lock ()); - auto db = app_.getWalletDB ().getDB (); - - if (db->executeSQL (str (boost::format ("SELECT COUNT(*) AS Count FROM SeedDomains WHERE Source='%s' OR Source='%c';") % vsManual % vsValidator)) && db->startIterRows ()) - iDomains = db->getInt ("Count"); - - db->endIterRows (); - - if (db->executeSQL (str (boost::format ("SELECT COUNT(*) AS Count FROM SeedNodes WHERE Source='%s' OR Source='%c';") % vsManual % vsValidator)) && db->startIterRows ()) - iNodes = db->getInt ("Count"); - - db->endIterRows (); - } -#endif - - bool bLoaded = iDomains || iNodes; - - // Always merge in the file specified in the config. - if (!app_.config().VALIDATORS_FILE.empty ()) - { - JLOG (j_.info) << "Bootstrapping UNL: loading from unl_default."; - - bLoaded = nodeLoad (app_.config().VALIDATORS_FILE); - } - - // If never loaded anything try the current directory. - if (!bLoaded && app_.config().VALIDATORS_FILE.empty ()) - { - JLOG (j_.info) << boost::str (boost::format ("Bootstrapping UNL: loading from '%s'.") - % app_.config().VALIDATORS_BASE); - - bLoaded = nodeLoad (app_.config().VALIDATORS_BASE); - } - - // Always load from rippled.cfg - if (!app_.config().validators.empty ()) - { - RippleAddress naInvalid; // Don't want a referrer on added entries. - - JLOG (j_.info) << boost::str (boost::format ("Bootstrapping UNL: loading from '%s'.") - % app_.config().CONFIG_FILE); - - if (processValidators ("local", - app_.config().CONFIG_FILE.string (), naInvalid, - vsConfig, &(app_.config().validators))) - bLoaded = true; - } - - if (!bLoaded) - { - JLOG (j_.info) << boost::str (boost::format ("Bootstrapping UNL: loading from '%s'.") - % app_.config().VALIDATORS_SITE); - - nodeNetwork (); - } -} - -//-------------------------------------------------------------------------- - -bool UniqueNodeListImp::nodeLoad (boost::filesystem::path pConfig) -{ - if (pConfig.empty ()) - { - JLOG (j_.info) << Config::Helpers::getValidatorsFileName() << - " path not specified."; - - return false; - } - - if (!boost::filesystem::exists (pConfig)) - { - JLOG (j_.warning) << Config::Helpers::getValidatorsFileName() << - " not found: " << pConfig; - - return false; - } - - if (!boost::filesystem::is_regular_file (pConfig)) - { - JLOG (j_.warning) << Config::Helpers::getValidatorsFileName() << - " not regular file: " << pConfig; - - return false; - } - - std::ifstream ifsDefault (pConfig.native ().c_str (), std::ios::in); - - if (!ifsDefault) - { - JLOG (j_.fatal) << Config::Helpers::getValidatorsFileName() << - " failed to open: " << pConfig; - - return false; - } - - std::string strValidators; - - strValidators.assign ((std::istreambuf_iterator (ifsDefault)), - std::istreambuf_iterator ()); - - if (ifsDefault.bad ()) - { - JLOG (j_.fatal) << Config::Helpers::getValidatorsFileName() << - "Failed to read: " << pConfig; - - return false; - } - - nodeProcess ("local", strValidators, pConfig.string ()); - - JLOG (j_.trace) << str (boost::format ("Processing: %s") % pConfig); - - return true; -} - -//-------------------------------------------------------------------------- - -void UniqueNodeListImp::nodeNetwork() -{ - if (!app_.config().VALIDATORS_SITE.empty ()) - { - HTTPClient::get ( - true, - app_.getIOService (), - app_.config().VALIDATORS_SITE, - 443, - app_.config().VALIDATORS_URI, - VALIDATORS_FILE_BYTES_MAX, - boost::posix_time::seconds (VALIDATORS_FETCH_SECONDS), - std::bind (&UniqueNodeListImp::validatorsResponse, this, - std::placeholders::_1, - std::placeholders::_2, - std::placeholders::_3), - app_.logs()); - } -} - -//-------------------------------------------------------------------------- - -Json::Value UniqueNodeListImp::getUnlJson() -{ - - Json::Value ret (Json::arrayValue); - - auto db = app_.getWalletDB ().checkoutDb (); - - - std::vector, 2>> columns; - selectBlobsIntoStrings(*db, - "SELECT PublicKey, Comment FROM TrustedNodes;", - columns); - for(auto const& strArray : columns) - { - Json::Value node (Json::objectValue); - - node["publicKey"] = strArray[0].value_or(""); - node["comment"] = strArray[1].value_or(""); - - ret.append (node); - } - - std::lock_guard sl (mUNLLock); - - for (auto const& key : ephemeralValidatorKeys_) - { - Json::Value node (Json::objectValue); - - node["publicKey"] = toBase58(TokenType::TOKEN_NODE_PUBLIC, key.first); - node["comment"] = key.second; - - ret.append (node); - } - - return ret; -} - -//-------------------------------------------------------------------------- - -// For each kind of source, have a starting number of points to be distributed. -int UniqueNodeListImp::iSourceScore (ValidatorSource vsWhy) -{ - int iScore = 0; - - switch (vsWhy) - { - case vsConfig: - iScore = 1500; - break; - - case vsInbound: - iScore = 0; - break; - - case vsManual: - iScore = 1500; - break; - - case vsReferral: - iScore = 0; - break; - - case vsTold: - iScore = 0; - break; - - case vsValidator: - iScore = 1000; - break; - - case vsWeb: - iScore = 200; - break; - - default: - Throw ("Internal error: bad ValidatorSource."); - } - - return iScore; -} - -// Load information about when we last updated. -bool UniqueNodeListImp::miscLoad() -{ - auto db = app_.getWalletDB ().checkoutDb (); - - boost::optional suO, fuO; - - *db << "SELECT ScoreUpdated, FetchUpdated FROM Misc WHERE Magic=1;", - soci::into(suO), soci::into(fuO); - - if (!db->got_data() ) - return false; - - mtpFetchUpdated = ptFromSeconds (fuO.value_or(-1)); - mtpScoreUpdated = ptFromSeconds (suO.value_or(-1)); - - trustedLoad (); - - return true; -} - -// Persist update information. -bool UniqueNodeListImp::miscSave() -{ - auto db = app_.getWalletDB ().checkoutDb (); - - *db << str (boost::format ("REPLACE INTO Misc (Magic,FetchUpdated,ScoreUpdated) VALUES (1,%d,%d);") - % iToSeconds (mtpFetchUpdated) - % iToSeconds (mtpScoreUpdated)); - - return true; -} - -//-------------------------------------------------------------------------- - -void UniqueNodeListImp::trustedLoad() -{ - auto db = app_.getWalletDB ().checkoutDb (); - std::lock_guard slUNL (mUNLLock); - - mUNL.clear (); - - std::vector, 1>> columns; - selectBlobsIntoStrings(*db, - "SELECT PublicKey FROM TrustedNodes WHERE Score != 0;", - columns); - for(auto const& strArray : columns) - { - mUNL.insert (strArray[0].value_or("")); - } -} - -//-------------------------------------------------------------------------- - -// For a round of scoring we destribute points from a node to nodes it refers to. -// Returns true, iff scores were distributed. -// -bool UniqueNodeListImp::scoreRound (std::vector& vsnNodes) -{ - bool bDist = false; - - // For each node, distribute roundSeed to roundScores. - for (auto& sn : vsnNodes) - { - int iEntries = sn.viReferrals.size (); - - if (sn.iRoundSeed && iEntries) - { - score iTotal = (iEntries + 1) * iEntries / 2; - score iBase = sn.iRoundSeed * iEntries / iTotal; - - // Distribute the current entires' seed score to validators - // prioritized by mention order. - for (int i = 0; i != iEntries; i++) - { - score iPoints = iBase * (iEntries - i) / iEntries; - - vsnNodes[sn.viReferrals[i]].iRoundScore += iPoints; - } - } - } - - if (ShouldLog (lsTRACE, UniqueNodeList)) - { - JLOG (j_.trace) << "midway: "; - for (auto& sn : vsnNodes) - { - JLOG (j_.trace) << str (boost::format ("%s| %d, %d, %d: [%s]") - % sn.strValidator - % sn.iScore - % sn.iRoundScore - % sn.iRoundSeed - % strJoin (sn.viReferrals.begin (), sn.viReferrals.end (), ",")); - } - } - - // Add roundScore to score. - // Make roundScore new roundSeed. - for (auto& sn : vsnNodes) - { - if (!bDist && sn.iRoundScore) - bDist = true; - - sn.iScore += sn.iRoundScore; - sn.iRoundSeed = sn.iRoundScore; - sn.iRoundScore = 0; - } - - if (ShouldLog (lsTRACE, UniqueNodeList)) - { - JLOG (j_.trace) << "finish: "; - for (auto& sn : vsnNodes) - { - JLOG (j_.trace) << str (boost::format ("%s| %d, %d, %d: [%s]") - % sn.strValidator - % sn.iScore - % sn.iRoundScore - % sn.iRoundSeed - % strJoin (sn.viReferrals.begin (), sn.viReferrals.end (), ",")); - } - } - - return bDist; -} - -//-------------------------------------------------------------------------- - -// From SeedDomains and ValidatorReferrals compute scores and update TrustedNodes. -// -// VFALCO TODO Shrink this function, break it up -// -void UniqueNodeListImp::scoreCompute() -{ - hash_map umPulicIdx; // Map of public key to index. - hash_map umDomainIdx; // Map of domain to index. - std::vector vsnNodes; // Index to scoring node. - - // For each entry in SeedDomains with a PublicKey: - // - Add an entry in umPulicIdx, umDomainIdx, and vsnNodes. - { - auto db = app_.getWalletDB ().checkoutDb (); - - std::vector, 3>> columns; - selectBlobsIntoStrings(*db, - "SELECT Domain,PublicKey,Source FROM SeedDomains;", - columns); - for(auto const& strArray : columns) - { - if (!strArray[1]) - // We ignore entries we don't have public keys for. - continue; - - std::string const strDomain = strArray[0].value_or(""); - std::string const strPublicKey = *strArray[1]; - std::string const strSource = strArray[2].value_or(""); - - assert (!strSource.empty ()); - - int const iScore = iSourceScore (static_cast (strSource[0])); - auto siOld = umPulicIdx.find (strPublicKey); - - if (siOld == umPulicIdx.end ()) - { - // New node - int iNode = vsnNodes.size (); - - umPulicIdx[strPublicKey] = iNode; - umDomainIdx[strDomain] = iNode; - - scoreNode snCurrent; - - snCurrent.strValidator = strPublicKey; - snCurrent.iScore = iScore; - snCurrent.iRoundSeed = snCurrent.iScore; - snCurrent.iRoundScore = 0; - snCurrent.iSeen = -1; - - vsnNodes.push_back (snCurrent); - } - else - { - scoreNode& snOld = vsnNodes[siOld->second]; - - if (snOld.iScore < iScore) - { - // Update old node - - snOld.iScore = iScore; - snOld.iRoundSeed = snOld.iScore; - } - } - } - } - - // For each entry in SeedNodes: - // - Add an entry in umPulicIdx, umDomainIdx, and vsnNodes. - { - auto db = app_.getWalletDB ().checkoutDb (); - - std::vector, 2>> columns; - selectBlobsIntoStrings(*db, - "SELECT PublicKey,Source FROM SeedNodes;", - columns); - for(auto const& strArray : columns) - { - std::string strPublicKey = strArray[0].value_or(""); - std::string strSource = strArray[1].value_or(""); - assert (!strSource.empty ()); - int iScore = iSourceScore (static_cast (strSource[0])); - auto siOld = umPulicIdx.find (strPublicKey); - - if (siOld == umPulicIdx.end ()) - { - // New node - int iNode = vsnNodes.size (); - - umPulicIdx[strPublicKey] = iNode; - - scoreNode snCurrent; - - snCurrent.strValidator = strPublicKey; - snCurrent.iScore = iScore; - snCurrent.iRoundSeed = snCurrent.iScore; - snCurrent.iRoundScore = 0; - snCurrent.iSeen = -1; - - vsnNodes.push_back (snCurrent); - } - else - { - scoreNode& snOld = vsnNodes[siOld->second]; - - if (snOld.iScore < iScore) - { - // Update old node - - snOld.iScore = iScore; - snOld.iRoundSeed = snOld.iScore; - } - } - } - } - - // For debugging, print out initial scores. - if (ShouldLog (lsTRACE, UniqueNodeList)) - { - for (auto& sn : vsnNodes) - { - JLOG (j_.trace) << str (boost::format ("%s| %d, %d, %d") - % sn.strValidator - % sn.iScore - % sn.iRoundScore - % sn.iRoundSeed); - } - } - - // JLOG (j_.trace) << str(boost::format("vsnNodes.size=%d") % vsnNodes.size()); - - // Step through growing list of nodes adding each validation list. - // - Each validator may have provided referals. Add those referals as validators. - for (int iNode = 0; iNode != vsnNodes.size (); ++iNode) - { - scoreNode& sn = vsnNodes[iNode]; - std::string& strValidator = sn.strValidator; - std::vector& viReferrals = sn.viReferrals; - - auto db = app_.getWalletDB ().checkoutDb (); - - std::vector, 1>> columns; - selectBlobsIntoStrings(*db, - boost::str (boost::format ( - "SELECT Referral FROM ValidatorReferrals " - "WHERE Validator=%s ORDER BY Entry;") % - sqlEscape (strValidator)), - columns); - std::string strReferral; - for(auto const& strArray : columns) - { - strReferral = strArray[0].value_or(""); - - int iReferral; - - RippleAddress na; - - if (na.setNodePublic (strReferral)) - { - // Referring a public key. - auto itEntry = umPulicIdx.find (strReferral); - - if (itEntry == umPulicIdx.end ()) - { - // Not found add public key to list of nodes. - iReferral = vsnNodes.size (); - - umPulicIdx[strReferral] = iReferral; - - scoreNode snCurrent; - - snCurrent.strValidator = strReferral; - snCurrent.iScore = iSourceScore (vsReferral); - snCurrent.iRoundSeed = snCurrent.iScore; - snCurrent.iRoundScore = 0; - snCurrent.iSeen = -1; - - vsnNodes.push_back (snCurrent); - } - else - { - iReferral = itEntry->second; - } - } - else - { - // Referring a domain. - auto itEntry = umDomainIdx.find (strReferral); - iReferral = itEntry == umDomainIdx.end () - ? -1 // We ignore domains we can't find entires for. - : itEntry->second; - } - - if (iReferral >= 0 && iNode != iReferral) - viReferrals.push_back (iReferral); - } - } - - // - // Distribute the points from the seeds. - // - bool bDist = true; - - for (int i = SCORE_ROUNDS; bDist && i--;) - bDist = scoreRound (vsnNodes); - - if (ShouldLog (lsTRACE, UniqueNodeList)) - { - JLOG (j_.trace) << "Scored:"; - for (auto& sn : vsnNodes) - { - JLOG (j_.trace) << str (boost::format ("%s| %d, %d, %d: [%s]") - % sn.strValidator - % sn.iScore - % sn.iRoundScore - % sn.iRoundSeed - % strJoin (sn.viReferrals.begin (), sn.viReferrals.end (), ",")); - } - } - - // Persist validator scores. - auto db = app_.getWalletDB ().checkoutDb (); - - soci::transaction tr(*db); - *db << "UPDATE TrustedNodes SET Score = 0 WHERE Score != 0;"; - - if (!vsnNodes.empty ()) - { - // Load existing Seens from DB. - std::vector vstrPublicKeys; - - vstrPublicKeys.resize (vsnNodes.size ()); - - for (int iNode = vsnNodes.size (); iNode--;) - { - vstrPublicKeys[iNode] = sqlEscape (vsnNodes[iNode].strValidator); - } - - // Iterate through the result rows with a fectch b/c putting a - // column of type DATETIME into a boost::tuple can throw when the - // datetime column is invalid (even if the value as int is valid). - std::vector, - boost::optional>> columns; - selectBlobsIntoStrings ( - *db, - str (boost::format ( - "SELECT PublicKey,Seen FROM TrustedNodes WHERE " - "PublicKey IN (%s);") % - strJoin ( - vstrPublicKeys.begin (), vstrPublicKeys.end (), ",")), - columns); - std::string pk; - for(auto const& col : columns) - { - pk = get<0>(col).value_or (""); - - vsnNodes[umPulicIdx[pk]].iSeen = get<1>(col).value_or (-1); - } - } - - hash_set usUNL; - - if (!vsnNodes.empty ()) - { - // Update the score old entries and add new entries as needed. - std::vector vstrValues; - - vstrValues.resize (vsnNodes.size ()); - - for (int iNode = vsnNodes.size (); iNode--;) - { - scoreNode& sn = vsnNodes[iNode]; - std::string strSeen = sn.iSeen >= 0 ? str (boost::format ("%d") % sn.iSeen) : "NULL"; - - vstrValues[iNode] = str (boost::format ("(%s,%s,%s)") - % sqlEscape (sn.strValidator) - % sn.iScore - % strSeen); - - usUNL.insert (sn.strValidator); - } - - *db << str (boost::format ("REPLACE INTO TrustedNodes (PublicKey,Score,Seen) VALUES %s;") - % strJoin (vstrValues.begin (), vstrValues.end (), ",")); - } - - { - std::lock_guard sl (mUNLLock); - - // XXX Should limit to scores above a certain minimum and limit to a certain number. - mUNL.swap (usUNL); - } - - hash_map umValidators; - - if (!vsnNodes.empty ()) - { - // For every IpReferral add a score for the IP and PORT. - std::vector, - boost::optional>> columns; - selectBlobsIntoStrings ( - *db, - "SELECT Validator,COUNT(*) AS Count FROM " - "IpReferrals GROUP BY Validator;", - columns); - for(auto const& col : columns) - { - umValidators[get<0>(col).value_or("")] = get<1>(col).value_or(0); - - // JLOG (j_.trace) << strValidator << ":" << db->getInt("Count"); - } - } - - // For each validator, get each referral and add its score to ip's score. - // map of pair :: score - hash_map, score> umScore; - - for (auto& vc : umValidators) - { - std::string strValidator = vc.first; - - auto itIndex = umPulicIdx.find (strValidator); - - if (itIndex != umPulicIdx.end ()) - { - int iSeed = vsnNodes[itIndex->second].iScore; - int iEntries = vc.second; - score iTotal = (iEntries + 1) * iEntries / 2; - score iBase = iSeed * iEntries / iTotal; - int iEntry = 0; - - std::vector, - boost::optional>> columns; - selectBlobsIntoStrings ( - *db, - str (boost::format ( - "SELECT IP,Port FROM IpReferrals WHERE " - "Validator=%s ORDER BY Entry;") % - sqlEscape (strValidator)), - columns); - for(auto const& col : columns) - { - score iPoints = iBase * (iEntries - iEntry) / iEntries; - int iPort; - - iPort = get<1>(col).value_or(0); - - std::pair< std::string, int> ep = std::make_pair (get<0>(col).value_or(""), iPort); - - auto itEp = umScore.find (ep); - - umScore[ep] = itEp == umScore.end () ? iPoints : itEp->second + iPoints; - iEntry++; - } - } - } - - tr.commit (); -} - -//-------------------------------------------------------------------------- - -// Start a timer to update scores. -// <-- bNow: true, to force scoring for debugging. -void UniqueNodeListImp::scoreNext (bool bNow) -{ - // JLOG (j_.trace) << str(boost::format("scoreNext: mtpFetchUpdated=%s mtpScoreStart=%s mtpScoreUpdated=%s mtpScoreNext=%s") % mtpFetchUpdated % mtpScoreStart % mtpScoreUpdated % mtpScoreNext); - bool bCanScore = mtpScoreStart.is_not_a_date_time () // Not scoring. - && !mtpFetchUpdated.is_not_a_date_time (); // Something to score. - - bool bDirty = - (mtpScoreUpdated.is_not_a_date_time () || mtpScoreUpdated <= mtpFetchUpdated) // Not already scored. - && (mtpScoreNext.is_not_a_date_time () // Timer is not fine. - || mtpScoreNext < mtpFetchUpdated + boost::posix_time::seconds (SCORE_DELAY_SECONDS)); - - if (bCanScore && (bNow || bDirty)) - { - // Need to update or set timer. - double const secondsFromNow = bNow ? 0 : SCORE_DELAY_SECONDS; - mtpScoreNext = boost::posix_time::second_clock::universal_time () // Past now too. - + boost::posix_time::seconds (secondsFromNow); - - // JLOG (j_.trace) << str(boost::format("scoreNext: @%s") % mtpScoreNext); - m_scoreTimer.setExpiration (secondsFromNow); - } -} - -//-------------------------------------------------------------------------- - -// Given a ripple.txt, process it. -// -// VFALCO TODO Can't we take a filename or stream instead of a string? -// -bool UniqueNodeListImp::responseFetch (std::string const& strDomain, const boost::system::error_code& err, int iStatus, std::string const& strSiteFile) -{ - bool bReject = !err && iStatus != 200; - - if (!bReject) - { - IniFileSections secSite = parseIniFile (strSiteFile, true); - bool bGood = !err; - - if (bGood) - { - JLOG (j_.trace) << strDomain - << ": retrieved configuration"; - } - else - { - JLOG (j_.trace) << strDomain - << ": unable to retrieve configuration: " - << err.message (); - } - - // - // Verify file domain - // - std::string strSite; - - if (bGood && !getSingleSection (secSite, SECTION_DOMAIN, strSite, j_)) - { - bGood = false; - - JLOG (j_.trace) << strDomain - << ": " << SECTION_DOMAIN - << "entry missing."; - } - - if (bGood && strSite != strDomain) - { - bGood = false; - - JLOG (j_.trace) << strDomain - << ": " << SECTION_DOMAIN << " does not match " << strSite; - } - - // - // Process public key - // - std::string strNodePublicKey; - - if (bGood && !getSingleSection (secSite, SECTION_PUBLIC_KEY, strNodePublicKey, j_)) - { - // Bad [validation_public_key] IniFileSections. - bGood = false; - - JLOG (j_.trace) << strDomain - << ": " << SECTION_PUBLIC_KEY << " entry missing."; - } - - RippleAddress naNodePublic; - - if (bGood && !naNodePublic.setNodePublic (strNodePublicKey)) - { - // Bad public key. - bGood = false; - - JLOG (j_.trace) << strDomain - << ": " << SECTION_PUBLIC_KEY << " is not a public key: " - << strNodePublicKey; - } - - if (bGood) - { - seedDomain sdCurrent; - - bool bFound = getSeedDomains (strDomain, sdCurrent); - assert (bFound); - (void) bFound; - - uint256 iSha256 = - sha512Half(makeSlice(strSiteFile)); - bool bChangedB = - sdCurrent.iSha256 != iSha256; - - sdCurrent.strDomain = strDomain; - // XXX If the node public key is changing, delete old public key information? - // XXX Only if no other refs to keep it arround, other wise we have an attack vector. - sdCurrent.naPublicKey = naNodePublic; - - // JLOG (j_.trace) << boost::format("sdCurrent.naPublicKey: '%s'") % sdCurrent.naPublicKey.humanNodePublic(); - - sdCurrent.tpFetch = boost::posix_time::second_clock::universal_time (); - sdCurrent.iSha256 = iSha256; - - setSeedDomains (sdCurrent, true); - - if (bChangedB) - { - JLOG (j_.trace) << strDomain - << ": processing new " << node_file_name_ << "."; - processFile (strDomain, naNodePublic, secSite); - } - else - { - JLOG (j_.trace) << strDomain - << ": no change in " << node_file_name_ << "."; - fetchFinish (); - } - } - else - { - // Failed: Update - - // XXX If we have public key, perhaps try look up in CAS? - fetchFinish (); - } - } - - return bReject; -} - -//-------------------------------------------------------------------------- - -// Try to process the next fetch of a ripple.txt. -void UniqueNodeListImp::fetchNext() -{ - bool bFull; - - { - std::lock_guard sl (mFetchLock); - - bFull = (mFetchActive == NODE_FETCH_JOBS); - } - - if (!bFull) - { - // Determine next scan. - std::string strDomain; - boost::posix_time::ptime tpNext (boost::posix_time::min_date_time); - boost::posix_time::ptime tpNow (boost::posix_time::second_clock::universal_time ()); - - auto db = app_.getWalletDB ().checkoutDb (); - - - { - soci::blob b(*db); - soci::indicator ind; - boost::optional nO; - *db << "SELECT Domain,Next FROM SeedDomains INDEXED BY SeedDomainNext ORDER BY Next LIMIT 1;", - soci::into(b, ind), - soci::into(nO); - if (nO) - { - int iNext (*nO); - - tpNext = ptFromSeconds (iNext); - - JLOG (j_.trace) << str (boost::format ("fetchNext: iNext=%s tpNext=%s tpNow=%s") % iNext % tpNext % tpNow); - if (soci::i_ok == ind) - convert (b, strDomain); - else - strDomain.clear (); - } - } - - if (!strDomain.empty ()) - { - std::lock_guard sl (mFetchLock); - - bFull = (mFetchActive == NODE_FETCH_JOBS); - - if (!bFull && tpNext <= tpNow) - { - mFetchActive++; - } - } - - if (strDomain.empty () || bFull) - { - JLOG (j_.trace) << str (boost::format ("fetchNext: strDomain=%s bFull=%d") % strDomain % bFull); - } - else if (tpNext > tpNow) - { - JLOG (j_.trace) << str (boost::format ("fetchNext: set timer : strDomain=%s") % strDomain); - // Fetch needs to happen in the future. Set a timer to wake us. - mtpFetchNext = tpNext; - - double seconds = (tpNext - tpNow).seconds (); - - // VFALCO check this. - if (seconds == 0) - seconds = 1; - - m_fetchTimer.setExpiration (seconds); - } - else - { - JLOG (j_.trace) << str (boost::format ("fetchNext: fetch now: strDomain=%s tpNext=%s tpNow=%s") % strDomain % tpNext % tpNow); - // Fetch needs to happen now. - mtpFetchNext = boost::posix_time::ptime (boost::posix_time::not_a_date_time); - - seedDomain sdCurrent; - bool bFound = getSeedDomains (strDomain, sdCurrent); - assert (bFound); - (void) bFound; - - // Update time of next fetch and this scan attempt. - sdCurrent.tpScan = tpNow; - - // XXX Use a longer duration if we have lots of validators. - sdCurrent.tpNext = sdCurrent.tpScan + boost::posix_time::hours (7 * 24); - - setSeedDomains (sdCurrent, false); - - JLOG (j_.trace) << strDomain - << " fetching " << node_file_name_ << "."; - - fetchProcess (strDomain); // Go get it. - - fetchNext (); // Look for more. - } - } -} - -//-------------------------------------------------------------------------- - -// Called when we need to update scores. -void UniqueNodeListImp::fetchDirty() -{ - // Note update. - mtpFetchUpdated = boost::posix_time::second_clock::universal_time (); - miscSave (); - - // Update scores. - scoreNext (false); -} - - -//-------------------------------------------------------------------------- - -void UniqueNodeListImp::fetchFinish() -{ - { - std::lock_guard sl (mFetchLock); - mFetchActive--; - } - - fetchNext (); -} - -//-------------------------------------------------------------------------- - -// Get the ripple.txt and process it. -void UniqueNodeListImp::fetchProcess (std::string strDomain) -{ - JLOG (j_.trace) << strDomain - << ": fetching " << node_file_name_ << "."; - - std::deque deqSites; - - // Order searching from most specifically for purpose to generic. - // This order allows the client to take the most burden rather than the servers. - deqSites.push_back (systemName () + strDomain); - deqSites.push_back ("www." + strDomain); - deqSites.push_back (strDomain); - - HTTPClient::get ( - true, - app_.getIOService (), - deqSites, - 443, - node_file_path_, - NODE_FILE_BYTES_MAX, - boost::posix_time::seconds (NODE_FETCH_SECONDS), - std::bind (&UniqueNodeListImp::responseFetch, this, strDomain, - std::placeholders::_1, std::placeholders::_2, - std::placeholders::_3), - app_.logs ()); -} - -// Process IniFileSections [validators_url]. -void UniqueNodeListImp::getValidatorsUrl (RippleAddress const& naNodePublic, - IniFileSections secSite) -{ - std::string strValidatorsUrl; - std::string strScheme; - std::string strDomain; - int iPort; - std::string strPath; - - if (getSingleSection (secSite, SECTION_VALIDATORS_URL, strValidatorsUrl, j_) - && !strValidatorsUrl.empty () - && parseUrl (strValidatorsUrl, strScheme, strDomain, iPort, strPath) - && -1 == iPort - && strScheme == "https") - { - HTTPClient::get ( - true, - app_.getIOService (), - strDomain, - 443, - strPath, - NODE_FILE_BYTES_MAX, - boost::posix_time::seconds (NODE_FETCH_SECONDS), - std::bind (&UniqueNodeListImp::responseValidators, this, - strValidatorsUrl, naNodePublic, secSite, strDomain, - std::placeholders::_1, std::placeholders::_2, - std::placeholders::_3), - app_.logs ()); - } - else - { - getIpsUrl (naNodePublic, secSite); - } -} - -//-------------------------------------------------------------------------- - -// Process IniFileSections [ips_url]. -// If we have a IniFileSections with a single entry, fetch the url and process it. -void UniqueNodeListImp::getIpsUrl (RippleAddress const& naNodePublic, IniFileSections secSite) -{ - std::string strIpsUrl; - std::string strScheme; - std::string strDomain; - int iPort; - std::string strPath; - - if (getSingleSection (secSite, SECTION_IPS_URL, strIpsUrl, j_) - && !strIpsUrl.empty () - && parseUrl (strIpsUrl, strScheme, strDomain, iPort, strPath) - && -1 == iPort - && strScheme == "https") - { - HTTPClient::get ( - true, - app_.getIOService (), - strDomain, - 443, - strPath, - NODE_FILE_BYTES_MAX, - boost::posix_time::seconds (NODE_FETCH_SECONDS), - std::bind (&UniqueNodeListImp::responseIps, this, strDomain, - naNodePublic, std::placeholders::_1, - std::placeholders::_2, std::placeholders::_3), - app_.logs ()); - } - else - { - fetchFinish (); - } -} - - -//-------------------------------------------------------------------------- - -// Given a IniFileSections with IPs, parse and persist it for a validator. -bool UniqueNodeListImp::responseIps (std::string const& strSite, RippleAddress const& naNodePublic, const boost::system::error_code& err, int iStatus, std::string const& strIpsFile) -{ - bool bReject = !err && iStatus != 200; - - if (!bReject) - { - if (!err) - { - IniFileSections secFile = parseIniFile (strIpsFile, true); - - processIps (strSite, naNodePublic, getIniFileSection (secFile, SECTION_IPS)); - } - - fetchFinish (); - } - - return bReject; -} - -// After fetching a ripple.txt from a web site, given a IniFileSections with validators, parse and persist it. -bool UniqueNodeListImp::responseValidators (std::string const& strValidatorsUrl, RippleAddress const& naNodePublic, IniFileSections secSite, std::string const& strSite, const boost::system::error_code& err, int iStatus, std::string const& strValidatorsFile) -{ - bool bReject = !err && iStatus != 200; - - if (!bReject) - { - if (!err) - { - IniFileSections secFile = parseIniFile (strValidatorsFile, true); - - processValidators (strSite, strValidatorsUrl, naNodePublic, vsValidator, getIniFileSection (secFile, SECTION_VALIDATORS)); - } - - getIpsUrl (naNodePublic, secSite); - } - - return bReject; -} - - -//-------------------------------------------------------------------------- - -// Persist the IPs refered to by a Validator. -// --> strSite: source of the IPs (for debugging) -// --> naNodePublic: public key of the validating node. -void UniqueNodeListImp::processIps (std::string const& strSite, RippleAddress const& naNodePublic, IniFileSections::mapped_type* pmtVecStrIps) -{ - std::string strEscNodePublic = sqlEscape (naNodePublic.humanNodePublic ()); - - JLOG (j_.debug) - << str (boost::format ("Validator: '%s' processing %d ips.") - % strSite % ( pmtVecStrIps ? pmtVecStrIps->size () : 0)); - - // Remove all current Validator's entries in IpReferrals - { - auto db = app_.getWalletDB ().checkoutDb (); - *db << str (boost::format ("DELETE FROM IpReferrals WHERE Validator=%s;") % strEscNodePublic); - } - - // Add new referral entries. - if (pmtVecStrIps && !pmtVecStrIps->empty ()) - { - std::vector vstrValues; - - vstrValues.resize (std::min ((int) pmtVecStrIps->size (), REFERRAL_IPS_MAX)); - - int iValues = 0; - for (auto const& strReferral : *pmtVecStrIps) - { - if (iValues == REFERRAL_VALIDATORS_MAX) - break; - - std::string strIP; - int iPort; - bool bValid = parseIpPort (strReferral, strIP, iPort); - - // XXX Filter out private network ips. - // XXX http://en.wikipedia.org/wiki/Private_network - - if (bValid) - { - vstrValues[iValues] = str (boost::format ("(%s,%d,%s,%d)") - % strEscNodePublic % iValues % sqlEscape (strIP) % iPort); - iValues++; - } - else - { - JLOG (j_.trace) - << str (boost::format ("Validator: '%s' [" SECTION_IPS "]: rejecting '%s'") - % strSite % strReferral); - } - } - - if (iValues) - { - vstrValues.resize (iValues); - - auto db = app_.getWalletDB ().checkoutDb (); - *db << str (boost::format ("INSERT INTO IpReferrals (Validator,Entry,IP,Port) VALUES %s;") - % strJoin (vstrValues.begin (), vstrValues.end (), ",")); - // XXX Check result. - } - } - - fetchDirty (); -} - -//-------------------------------------------------------------------------- - -// Persist ValidatorReferrals. -// --> strSite: source site for display -// --> strValidatorsSrc: source details for display -// --> naNodePublic: remote source public key - not valid for local -// --> vsWhy: reason for adding validator to SeedDomains or SeedNodes. -int UniqueNodeListImp::processValidators (std::string const& strSite, std::string const& strValidatorsSrc, RippleAddress const& naNodePublic, ValidatorSource vsWhy, IniFileSections::mapped_type const* pmtVecStrValidators) -{ - std::string strNodePublic = naNodePublic.isValid () ? naNodePublic.humanNodePublic () : strValidatorsSrc; - int iValues = 0; - - JLOG (j_.trace) - << str (boost::format ("Validator: '%s' : '%s' : processing %d validators.") - % strSite - % strValidatorsSrc - % ( pmtVecStrValidators ? pmtVecStrValidators->size () : 0)); - - // Remove all current Validator's entries in ValidatorReferrals - { - auto db = app_.getWalletDB ().checkoutDb (); - - *db << str (boost::format ("DELETE FROM ValidatorReferrals WHERE Validator='%s';") % strNodePublic); - // XXX Check result. - } - - // Add new referral entries. - if (pmtVecStrValidators && pmtVecStrValidators->size ()) - { - std::vector vstrValues; - - vstrValues.reserve (std::min ((int) pmtVecStrValidators->size (), REFERRAL_VALIDATORS_MAX)); - - for (auto const& strReferral : *pmtVecStrValidators) - { - if (iValues == REFERRAL_VALIDATORS_MAX) - break; - - boost::smatch smMatch; - - // domain comment? - // public_key comment? - static boost::regex reReferral ("\\`\\s*(\\S+)(?:\\s+(.+))?\\s*\\'"); - - if (!boost::regex_match (strReferral, smMatch, reReferral)) - { - JLOG (j_.warning) << str (boost::format ("Bad validator: syntax error: %s: %s") % strSite % strReferral); - } - else - { - std::string strRefered = smMatch[1]; - std::string strComment = smMatch[2]; - RippleAddress naValidator; - - if (naValidator.setSeedGeneric (strRefered)) - { - JLOG (j_.warning) << str (boost::format ("Bad validator: domain or public key required: %s %s") % strRefered % strComment); - } - else if (naValidator.setNodePublic (strRefered)) - { - // A public key. - // XXX Schedule for CAS lookup. - nodeAddPublic (naValidator, vsWhy, strComment); - - JLOG (j_.info) << str (boost::format ("Node Public: %s %s") % strRefered % strComment); - - if (naNodePublic.isValid ()) - vstrValues.push_back (str (boost::format ("('%s',%d,'%s')") % strNodePublic % iValues % naValidator.humanNodePublic ())); - - iValues++; - } - else - { - // A domain: need to look it up. - nodeAddDomain (strRefered, vsWhy, strComment); - - JLOG (j_.info) << str (boost::format ("Node Domain: %s %s") % strRefered % strComment); - - if (naNodePublic.isValid ()) - vstrValues.push_back (str (boost::format ("('%s',%d,%s)") % strNodePublic % iValues % sqlEscape (strRefered))); - - iValues++; - } - } - } - - if (!vstrValues.empty ()) - { - std::string strSql = str (boost::format ("INSERT INTO ValidatorReferrals (Validator,Entry,Referral) VALUES %s;") - % strJoin (vstrValues.begin (), vstrValues.end (), ",")); - - auto db = app_.getWalletDB ().checkoutDb (); - - *db << strSql; - // XXX Check result. - } - } - - fetchDirty (); - - return iValues; -} - -//-------------------------------------------------------------------------- - -// Process a ripple.txt. -void UniqueNodeListImp::processFile (std::string const& strDomain, RippleAddress const& naNodePublic, IniFileSections secSite) -{ - // - // Process Validators - // - processValidators (strDomain, node_file_name_, naNodePublic, - vsReferral, getIniFileSection (secSite, SECTION_VALIDATORS)); - - // - // Process ips - // - processIps (strDomain, naNodePublic, getIniFileSection (secSite, SECTION_IPS)); - - // - // Process currencies - // - IniFileSections::mapped_type* pvCurrencies; - - if ((pvCurrencies = getIniFileSection (secSite, SECTION_CURRENCIES)) && pvCurrencies->size ()) - { - // XXX Process currencies. - JLOG (j_.warning) << "Ignoring currencies: not implemented."; - } - - getValidatorsUrl (naNodePublic, secSite); -} - -//-------------------------------------------------------------------------- - -// Retrieve a SeedDomain from DB. -bool UniqueNodeListImp::getSeedDomains (std::string const& strDomain, seedDomain& dstSeedDomain) -{ - bool bResult = false; - - std::string strSql = boost::str ( - boost::format ( - "SELECT Domain, PublicKey, Source, Next, Scan, Fetch, Sha256, " - "Comment FROM SeedDomains WHERE Domain=%s;") % - sqlEscape (strDomain)); - - auto db = app_.getWalletDB ().checkoutDb (); - - // Iterate through the result rows with a fectch b/c putting a - // column of type DATETIME into a boost::tuple can throw when the - // datetime column is invalid (even if the value as int is valid). - soci::blob domainBlob(*db); - soci::indicator di; - boost::optional strPublicKey; - soci:: blob sourceBlob(*db); - soci::indicator si; - std::string strSource; - boost::optional iNext; - boost::optional iScan; - boost::optional iFetch; - boost::optional strSha256; - soci::blob commentBlob(*db); - soci::indicator ci; - boost::optional strComment; - - soci::statement st = (db->prepare << strSql, - soci::into (domainBlob, di), - soci::into (strPublicKey), - soci::into (sourceBlob, si), - soci::into (iNext), - soci::into (iScan), - soci::into (iFetch), - soci::into (strSha256), - soci::into (commentBlob, ci)); - - st.execute (); - while (st.fetch ()) - { - bResult = true; - - if (soci::i_ok == di) - convert (domainBlob, dstSeedDomain.strDomain); - - if (strPublicKey && !strPublicKey->empty ()) - dstSeedDomain.naPublicKey.setNodePublic (*strPublicKey); - else - dstSeedDomain.naPublicKey.clear (); - - if (soci::i_ok == si) - { - convert (sourceBlob, strSource); - dstSeedDomain.vsSource = static_cast (strSource[0]); - } - else - { - assert (0); - } - - dstSeedDomain.tpNext = ptFromSeconds (iNext.value_or (0)); - dstSeedDomain.tpScan = ptFromSeconds (iScan.value_or (0)); - dstSeedDomain.tpFetch = ptFromSeconds (iFetch.value_or (0)); - - if (strSha256 && !strSha256->empty ()) - dstSeedDomain.iSha256.SetHex (*strSha256); - else - dstSeedDomain.iSha256.zero (); - - if (soci::i_ok == ci) - convert (commentBlob, dstSeedDomain.strComment); - else - dstSeedDomain.strComment.clear (); - } - - return bResult; -} - -//-------------------------------------------------------------------------- - -// Persist a SeedDomain. -void UniqueNodeListImp::setSeedDomains (const seedDomain& sdSource, bool bNext) -{ - int iNext = iToSeconds (sdSource.tpNext); - int iScan = iToSeconds (sdSource.tpScan); - int iFetch = iToSeconds (sdSource.tpFetch); - - // JLOG (lsTRACE) << str(boost::format("setSeedDomains: iNext=%s tpNext=%s") % iNext % sdSource.tpNext); - - std::string strSql = boost::str (boost::format ("REPLACE INTO SeedDomains (Domain,PublicKey,Source,Next,Scan,Fetch,Sha256,Comment) VALUES (%s, %s, %s, %d, %d, %d, '%s', %s);") - % sqlEscape (sdSource.strDomain) - % (sdSource.naPublicKey.isValid () ? sqlEscape (sdSource.naPublicKey.humanNodePublic ()) : "NULL") - % sqlEscape (std::string (1, static_cast (sdSource.vsSource))) - % iNext - % iScan - % iFetch - % to_string (sdSource.iSha256) - % sqlEscape (sdSource.strComment) - ); - - auto db = app_.getWalletDB ().checkoutDb (); - - try - { - *db << strSql; - } - catch (soci::soci_error& e) - { - // XXX Check result. - JLOG (j_.warning) << "setSeedDomains: failed. Error: " << e.what(); - } - - if (bNext && (mtpFetchNext.is_not_a_date_time () || mtpFetchNext > sdSource.tpNext)) - { - // Schedule earlier wake up. - fetchNext (); - } -} - - -//-------------------------------------------------------------------------- - -// Retrieve a SeedNode from DB. -bool UniqueNodeListImp::getSeedNodes (RippleAddress const& naNodePublic, seedNode& dstSeedNode) -{ - std::string strSql = - str (boost::format ( - "SELECT PublicKey, Source, Next, Scan, Fetch, Sha256, " - "Comment FROM SeedNodes WHERE PublicKey='%s';") % - naNodePublic.humanNodePublic ()); - - auto db = app_.getWalletDB ().checkoutDb (); - - std::string strPublicKey; - std::string strSource; - soci::blob sourceBlob(*db); - soci::indicator si; - boost::optional iNext; - boost::optional iScan; - boost::optional iFetch; - boost::optional strSha256; - soci::blob commentBlob(*db); - soci::indicator ci; - - *db << strSql, - soci::into (strPublicKey), - soci::into (sourceBlob, si), - soci::into (iNext), - soci::into (iScan), - soci::into (iFetch), - soci::into (strSha256), - soci::into (commentBlob, ci); - - if (!db->got_data ()) - return false; - - if (!strPublicKey.empty ()) - dstSeedNode.naPublicKey.setNodePublic (strPublicKey); - else - dstSeedNode.naPublicKey.clear (); - - if (soci::i_ok == si) - { - convert (sourceBlob, strSource); - dstSeedNode.vsSource = static_cast (strSource[0]); - } - else - assert (0); - - dstSeedNode.tpNext = ptFromSeconds (iNext.value_or(0)); - dstSeedNode.tpScan = ptFromSeconds (iScan.value_or(0)); - dstSeedNode.tpFetch = ptFromSeconds (iFetch.value_or(0)); - - if (strSha256 && !strSha256->empty ()) - dstSeedNode.iSha256.SetHex (*strSha256); - else - dstSeedNode.iSha256.zero (); - - if (soci::i_ok == ci) - convert (commentBlob, dstSeedNode.strComment); - else - dstSeedNode.strComment.clear (); - - return true; -} - -//-------------------------------------------------------------------------- - -// Persist a SeedNode. -// <-- bNext: true, to do fetching if needed. -void UniqueNodeListImp::setSeedNodes (const seedNode& snSource, bool bNext) -{ - int iNext = iToSeconds (snSource.tpNext); - int iScan = iToSeconds (snSource.tpScan); - int iFetch = iToSeconds (snSource.tpFetch); - - // JLOG (lsTRACE) << str(boost::format("setSeedNodes: iNext=%s tpNext=%s") % iNext % sdSource.tpNext); - - assert (snSource.naPublicKey.isValid ()); - - std::string strSql = str (boost::format ("REPLACE INTO SeedNodes (PublicKey,Source,Next,Scan,Fetch,Sha256,Comment) VALUES ('%s', '%c', %d, %d, %d, '%s', %s);") - % snSource.naPublicKey.humanNodePublic () - % static_cast (snSource.vsSource) - % iNext - % iScan - % iFetch - % to_string (snSource.iSha256) - % sqlEscape (snSource.strComment) - ); - - { - auto db = app_.getWalletDB ().checkoutDb (); - - try - { - *db << strSql; - } - catch(soci::soci_error& e) - { - JLOG (j_.trace) << "setSeedNodes: failed. Error: " << e.what (); - } - } - -#if 0 - - // YYY When we have a cas schedule lookups similar to this. - if (bNext && (mtpFetchNext.is_not_a_date_time () || mtpFetchNext > snSource.tpNext)) - { - // Schedule earlier wake up. - fetchNext (); - } - -#else - fetchDirty (); -#endif -} - -//-------------------------------------------------------------------------- - -bool UniqueNodeListImp::validatorsResponse (const boost::system::error_code& err, int iStatus, std::string strResponse) -{ - bool bReject = !err && iStatus != 200; - - if (!bReject) - { - JLOG (j_.trace) << - "Fetch '" << - Config::Helpers::getValidatorsFileName () << - "' complete."; - - if (!err) - { - nodeProcess ("network", strResponse, app_.config().VALIDATORS_SITE); - } - else - { - JLOG (j_.warning) << "Error: " << err.message (); - } - } - return bReject; -} - -//-------------------------------------------------------------------------- - -// Process a validators.txt. -// --> strSite: source of validators -// --> strValidators: contents of a validators.txt -// -// VFALCO TODO Can't we name this processValidatorList? -// -void UniqueNodeListImp::nodeProcess (std::string const& strSite, std::string const& strValidators, std::string const& strSource) -{ - IniFileSections secValidators = parseIniFile (strValidators, true); - - IniFileSections::mapped_type* pmtEntries = getIniFileSection (secValidators, SECTION_VALIDATORS); - - if (pmtEntries) - { - RippleAddress naInvalid; // Don't want a referrer on added entries. - - // YYY Unspecified might be bootstrap or rpc command - processValidators (strSite, strSource, naInvalid, vsValidator, pmtEntries); - } - else - { - JLOG (j_.warning) << boost::str (boost::format ("'%s' missing [" SECTION_VALIDATORS "].") - % app_.config().VALIDATORS_BASE); - } -} - -//------------------------------------------------------------------------------ - -std::unique_ptr -make_UniqueNodeList (Application& app, beast::Stoppable& parent) -{ - return std::make_unique (app, parent); -} - -} // ripple diff --git a/src/ripple/app/misc/UniqueNodeList.h b/src/ripple/app/misc/UniqueNodeList.h deleted file mode 100644 index 370ae2e425..0000000000 --- a/src/ripple/app/misc/UniqueNodeList.h +++ /dev/null @@ -1,86 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_APP_PEERS_UNIQUENODELIST_H_INCLUDED -#define RIPPLE_APP_PEERS_UNIQUENODELIST_H_INCLUDED - -#include -#include -#include -#include -#include -#include // - -namespace ripple { - -class UniqueNodeList : public beast::Stoppable -{ -protected: - explicit UniqueNodeList (Stoppable& parent); - -public: - enum ValidatorSource - { - vsConfig = 'C', // rippled.cfg - vsInbound = 'I', - vsManual = 'M', - vsReferral = 'R', - vsTold = 'T', - vsValidator = 'V', // validators.txt - vsWeb = 'W', - }; - - // VFALCO TODO rename this to use the right coding style - using score = long; - -public: - virtual ~UniqueNodeList () { } - - // VFALCO TODO Roll this into the constructor so there is one less state. - virtual void start () = 0; - - virtual void insertEphemeralKey (PublicKey pk, std::string comment) = 0; - virtual void deleteEphemeralKey (PublicKey const& pk) = 0; - - // VFALCO TODO rename all these, the "node" prefix is redundant (lol) - virtual void nodeAddPublic (RippleAddress const& naNodePublic, ValidatorSource vsWhy, std::string const& strComment) = 0; - virtual void nodeAddDomain (std::string strDomain, ValidatorSource vsWhy, std::string const& strComment = "") = 0; - virtual void nodeRemovePublic (RippleAddress const& naNodePublic) = 0; - virtual void nodeRemoveDomain (std::string strDomain) = 0; - virtual void nodeReset () = 0; - - virtual void nodeScore () = 0; - - virtual bool nodeInUNL (RippleAddress const& naNodePublic) = 0; - - virtual void nodeBootstrap () = 0; - virtual bool nodeLoad (boost::filesystem::path pConfig) = 0; - virtual void nodeNetwork () = 0; - - virtual Json::Value getUnlJson () = 0; - - virtual int iSourceScore (ValidatorSource vsWhy) = 0; -}; - -std::unique_ptr -make_UniqueNodeList (Application& app, beast::Stoppable& parent); - -} // ripple - -#endif diff --git a/src/ripple/app/misc/Validations.cpp b/src/ripple/app/misc/Validations.cpp index 7a619b483d..84aef2fc7a 100644 --- a/src/ripple/app/misc/Validations.cpp +++ b/src/ripple/app/misc/Validations.cpp @@ -24,7 +24,7 @@ #include #include #include -#include +#include #include #include #include @@ -89,7 +89,7 @@ private: RippleAddress signer = val->getSignerPublic (); bool isCurrent = current (val); - if (! val->isTrusted() && app_.getUNL().nodeInUNL (signer)) + if (!val->isTrusted() && app_.validators().trusted (signer)) val->setTrusted(); if (!val->isTrusted ()) diff --git a/src/ripple/app/misc/ValidatorList.h b/src/ripple/app/misc/ValidatorList.h new file mode 100644 index 0000000000..84e17f898d --- /dev/null +++ b/src/ripple/app/misc/ValidatorList.h @@ -0,0 +1,121 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2015 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_APP_MISC_VALIDATORLIST_H_INCLUDED +#define RIPPLE_APP_MISC_VALIDATORLIST_H_INCLUDED + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ripple { + +class ValidatorList +{ +private: + /** The non-ephemeral public keys from the configuration file. */ + hash_map permanent_; + + /** The ephemeral public keys from manifests. */ + hash_map ephemeral_; + + std::mutex mutable mutex_; + beast::Journal mutable j_; + +public: + explicit + ValidatorList (beast::Journal j); + + /** Determines whether a node is in the UNL + @return boost::none if the node isn't a member, + otherwise, the comment associated with the + node (which may be an empty string). + */ + boost::optional + member ( + RippleAddress const& identity) const; + + /** Determines whether a node is in the UNL */ + bool + trusted ( + RippleAddress const& identity) const; + + /** Insert a short-term validator key published in a manifest. */ + bool + insertEphemeralKey ( + PublicKey const& identity, + std::string const& comment); + + /** Remove a short-term validator revoked in a manifest. */ + bool + removeEphemeralKey ( + PublicKey const& identity); + + /** Insert a long-term validator key. */ + bool + insertPermanentKey ( + RippleAddress const& identity, + std::string const& comment); + + /** Remove a long-term validator key. */ + bool + removePermanentKey ( + RippleAddress const& identity); + + /** The number of installed permanent and ephemeral keys */ + std::size_t + size () const; + + /** Invokes the callback once for every node in the UNL. + @note You are not allowed to insert or remove any + nodes in the UNL from within the callback. + + The arguments passed into the lambda are: + - The public key of the validator; + - A (possibly empty) comment. + - A boolean indicating whether this is a + permanent or ephemeral key; + */ + void + for_each ( + std::function func) const; + + /** Load the list of trusted validators. + + The section contains entries consisting of a base58 + encoded validator public key, optionally followed by + a comment. + + @return false if an entry could not be parsed or + contained an invalid validator public key, + true otherwise. + */ + bool + load ( + Section const& validators); +}; + +} + +#endif diff --git a/src/ripple/app/misc/impl/ValidatorList.cpp b/src/ripple/app/misc/impl/ValidatorList.cpp new file mode 100644 index 0000000000..5c3f4b2c6c --- /dev/null +++ b/src/ripple/app/misc/impl/ValidatorList.cpp @@ -0,0 +1,208 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2015 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include +#include +#include + +namespace ripple { + +static +PublicKey +asPublicKey(RippleAddress const& raPublicKey) +{ + auto const& blob = raPublicKey.getNodePublic(); + + if (blob.empty()) + LogicError ("Can't convert invalid RippleAddress to PublicKey"); + + return PublicKey(Slice(blob.data(), blob.size())); +} + +ValidatorList::ValidatorList (beast::Journal j) + : j_ (j) +{ +} + +boost::optional +ValidatorList::member (RippleAddress const& identity) const +{ + std::lock_guard sl (mutex_); + + auto const publicKey = asPublicKey (identity); + + auto ret = ephemeral_.find (publicKey); + + if (ret != ephemeral_.end()) + return ret->second; + + ret = permanent_.find (publicKey); + + if (ret != permanent_.end()) + return ret->second; + + return boost::none; +} + +bool +ValidatorList::trusted (RippleAddress const& identity) const +{ + return static_cast (member(identity)); +} + +bool +ValidatorList::insertEphemeralKey ( + PublicKey const& identity, + std::string const& comment) +{ + std::lock_guard sl (mutex_); + + if (permanent_.find (identity) != permanent_.end()) + { + JLOG (j_.error) << + toBase58 (TokenType::TOKEN_NODE_PUBLIC, identity) << + ": ephemeral key exists in permanent table!"; + return false; + } + + return ephemeral_.emplace (identity, comment).second; +} + +bool +ValidatorList::removeEphemeralKey ( + PublicKey const& identity) +{ + std::lock_guard sl (mutex_); + return ephemeral_.erase (identity); +} + +bool +ValidatorList::insertPermanentKey ( + RippleAddress const& identity, + std::string const& comment) +{ + std::lock_guard sl (mutex_); + + auto const publicKey = asPublicKey (identity); + + if (ephemeral_.find (publicKey) != ephemeral_.end()) + { + JLOG (j_.error) << + toBase58 (TokenType::TOKEN_NODE_PUBLIC, publicKey) << + ": permanent key exists in ephemeral table!"; + return false; + } + + return permanent_.emplace (publicKey, comment).second; +} + +bool +ValidatorList::removePermanentKey ( + RippleAddress const& identity) +{ + std::lock_guard sl (mutex_); + return permanent_.erase (asPublicKey (identity)); +} + +std::size_t +ValidatorList::size () const +{ + std::lock_guard sl (mutex_); + return permanent_.size () + ephemeral_.size (); +} + +void +ValidatorList::for_each ( + std::function func) const +{ + std::lock_guard sl (mutex_); + + for (auto const& v : permanent_) + func (v.first, v.second, false); + for (auto const& v : ephemeral_) + func (v.first, v.second, true); +} + +bool +ValidatorList::load ( + Section const& validators) +{ + static boost::regex const re ( + "[[:space:]]*" // skip leading whitespace + "([[:alnum:]]+)" // node identity + "(?:" // begin optional comment block + "[[:space:]]+" // (skip all leading whitespace) + "(?:" // begin optional comment + "(.*[^[:space:]]+)" // the comment + "[[:space:]]*" // (skip all trailing whitespace) + ")?" // end optional comment + ")?" // end optional comment block + ); + + JLOG (j_.debug) << + "Loading configured validators"; + + std::size_t count = 0; + + for (auto const& n : validators.values ()) + { + JLOG (j_.trace) << + "Processing '" << n << "'"; + + boost::smatch match; + + if (!boost::regex_match (n, match, re)) + { + JLOG (j_.error) << + "Malformed entry: '" << n << "'"; + return false; + } + + auto const id = parseBase58( + TokenType::TOKEN_NODE_PUBLIC, match[1]); + + if (!id) + { + JLOG (j_.error) << + "Invalid node identity: " << match[1]; + return false; + } + + auto const ra = RippleAddress::createNodePublic (match[1]); + + if (trusted (ra)) + { + JLOG (j_.warning) << + "Duplicate node identity: " << match[1]; + continue; + } + + if (insertPermanentKey(ra, trim_whitespace (match[2]))) + ++count; + } + + JLOG (j_.debug) << + "Loaded " << count << " entries"; + + return true; +} + +} diff --git a/src/ripple/app/tests/ValidatorList_test.cpp b/src/ripple/app/tests/ValidatorList_test.cpp new file mode 100644 index 0000000000..3fdf39bc03 --- /dev/null +++ b/src/ripple/app/tests/ValidatorList_test.cpp @@ -0,0 +1,280 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright 2015 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include + +namespace ripple { +namespace tests { + +class ValidatorList_test : public ripple::TestSuite +{ +private: + static + PublicKey + asPublicKey(RippleAddress const& raPublicKey) + { + auto const& blob = raPublicKey.getNodePublic(); + + if (blob.empty()) + LogicError ("Can't convert invalid RippleAddress to PublicKey"); + + return PublicKey(Slice(blob.data(), blob.size())); + } + + static + RippleAddress + randomNode () + { + return RippleAddress::createNodePublic ( + RippleAddress::createSeedRandom ()); + } + + static + bool + isPresent ( + std::vector container, + RippleAddress const& item) + { + auto found = std::find ( + std::begin (container), + std::end (container), + item); + + return (found != std::end (container)); + } + + void + testConfigLoad () + { + testcase ("Config Load"); + + auto validators = std::make_unique (beast::Journal ()); + + std::vector network; + + while (network.size () != 8) + network.push_back (randomNode()); + + Section s1; + + // Correct (empty) configuration + expect (validators->load (s1)); + expect (validators->size() == 0); + + // Correct configuration + s1.append (network[0].humanNodePublic()); + s1.append (network[1].humanNodePublic() + " Comment"); + s1.append (network[2].humanNodePublic() + " Multi Word Comment"); + s1.append (network[3].humanNodePublic() + " Leading Whitespace"); + s1.append (network[4].humanNodePublic() + " Trailing Whitespace "); + s1.append (network[5].humanNodePublic() + " Leading & Trailing Whitespace "); + s1.append (network[6].humanNodePublic() + " Leading, Trailing & Internal Whitespace "); + s1.append (network[7].humanNodePublic() + " "); + + expect (validators->load (s1)); + + for (auto const& n : network) + expect (validators->trusted (n)); + + // Incorrect configurations: + Section s2; + s2.append ("NotAPublicKey"); + + expect (!validators->load (s2)); + + Section s3; + s3.append ("@" + network[0].humanNodePublic()); + expect (!validators->load (s3)); + + Section s4; + s4.append (network[0].humanNodePublic() + "!"); + expect (!validators->load (s4)); + + Section s5; + s5.append (network[0].humanNodePublic() + "! Comment"); + expect (!validators->load (s5)); + + // Check if we properly terminate when we encounter + // a malformed or unparseable entry: + auto const badNode = randomNode(); + auto const goodNode = randomNode (); + + Section s6; + s6.append (badNode.humanNodePublic() + "XXX"); + s6.append (goodNode.humanNodePublic()); + + expect (!validators->load (s6)); + expect (!validators->trusted (badNode)); + expect (!validators->trusted (goodNode)); + } + + void + testMembership () + { + // The servers on the permanentValidators + std::vector permanentValidators; + std::vector ephemeralValidators; + + while (permanentValidators.size () != 64) + permanentValidators.push_back (randomNode()); + + while (ephemeralValidators.size () != 64) + ephemeralValidators.push_back (randomNode()); + + { + testcase ("Membership: No Validators"); + + auto vl = std::make_unique (beast::Journal ()); + + for (auto const& v : permanentValidators) + expect (!vl->trusted (v)); + + for (auto const& v : ephemeralValidators) + expect (!vl->trusted (v)); + } + + { + testcase ("Membership: Non-Empty, Some Present, Some Not Present"); + + std::vector p ( + permanentValidators.begin (), + permanentValidators.begin () + 16); + + while (p.size () != 32) + p.push_back (randomNode()); + + std::vector e ( + ephemeralValidators.begin (), + ephemeralValidators.begin () + 16); + + while (e.size () != 32) + e.push_back (randomNode()); + + auto vl = std::make_unique (beast::Journal ()); + + for (auto const& v : p) + vl->insertPermanentKey (v, v.ToString()); + + for (auto const& v : e) + vl->insertEphemeralKey (asPublicKey (v), v.ToString()); + + for (auto const& v : p) + expect (vl->trusted (v)); + + for (auto const& v : e) + expect (vl->trusted (v)); + + for (auto const& v : permanentValidators) + expect (static_cast(vl->trusted (v)) == isPresent (p, v)); + + for (auto const& v : ephemeralValidators) + expect (static_cast(vl->trusted (v)) == isPresent (e, v)); + } + } + + void + testModification () + { + testcase ("Insertion and Removal"); + + auto vl = std::make_unique (beast::Journal ()); + + auto const v = randomNode (); + + // Inserting a new permanent key succeeds + expect (vl->insertPermanentKey (v, "Permanent")); + { + auto member = vl->member (v); + expect (static_cast(member)); + expect (member->compare("Permanent") == 0); + } + // Inserting the same permanent key fails: + expect (!vl->insertPermanentKey (v, "")); + { + auto member = vl->member (v); + expect (static_cast(member)); + expect (member->compare("Permanent") == 0); + } + // Inserting the same key as ephemeral fails: + expect (!vl->insertEphemeralKey (asPublicKey(v), "Ephemeral")); + { + auto member = vl->member (v); + expect (static_cast(member)); + expect (member->compare("Permanent") == 0); + } + // Removing the key as ephemeral fails: + expect (!vl->removeEphemeralKey (asPublicKey(v))); + { + auto member = vl->member (v); + expect (static_cast(member)); + expect (member->compare("Permanent") == 0); + } + // Deleting the key as permanent succeeds: + expect (vl->removePermanentKey (v)); + expect (!static_cast(vl->trusted (v))); + + // Insert an ephemeral validator key + expect (vl->insertEphemeralKey (asPublicKey(v), "Ephemeral")); + { + auto member = vl->member (v); + expect (static_cast(member)); + expect (member->compare("Ephemeral") == 0); + } + // Inserting the same ephemeral key fails + expect (!vl->insertEphemeralKey (asPublicKey(v), "")); + { + auto member = vl->member (v); + expect (static_cast(member)); + expect (member->compare("Ephemeral") == 0); + } + // Inserting the same key as permanent fails: + expect (!vl->insertPermanentKey (v, "Permanent")); + { + auto member = vl->member (v); + expect (static_cast(member)); + expect (member->compare("Ephemeral") == 0); + } + // Deleting the key as permanent fails: + expect (!vl->removePermanentKey (v)); + { + auto member = vl->member (v); + expect (static_cast(member)); + expect (member->compare("Ephemeral") == 0); + } + // Deleting the key as ephemeral succeeds: + expect (vl->removeEphemeralKey (asPublicKey(v))); + expect (!vl->trusted(v)); + } + +public: + void + run() override + { + testConfigLoad(); + testMembership (); + testModification (); + } +}; + +BEAST_DEFINE_TESTSUITE(ValidatorList, app, ripple); + +} // tests +} // ripple diff --git a/src/ripple/basics/StringUtilities.h b/src/ripple/basics/StringUtilities.h index bac014d853..258336f3a1 100644 --- a/src/ripple/basics/StringUtilities.h +++ b/src/ripple/basics/StringUtilities.h @@ -93,6 +93,8 @@ bool parseIpPort (std::string const& strSource, std::string& strIP, int& iPort); bool parseUrl (std::string const& strUrl, std::string& strScheme, std::string& strDomain, int& iPort, std::string& strPath); +std::string trim_whitespace (std::string str); + } // ripple #endif diff --git a/src/ripple/basics/impl/StringUtilities.cpp b/src/ripple/basics/impl/StringUtilities.cpp index 1966aa7c93..d05996d8b8 100644 --- a/src/ripple/basics/impl/StringUtilities.cpp +++ b/src/ripple/basics/impl/StringUtilities.cpp @@ -189,4 +189,11 @@ bool parseUrl (std::string const& strUrl, std::string& strScheme, std::string& s return bMatch; } + +std::string trim_whitespace (std::string str) +{ + boost::trim (str); + return str; +} + } // ripple diff --git a/src/ripple/core/Config.h b/src/ripple/core/Config.h index ac5d0cd77e..98e6c5bdeb 100644 --- a/src/ripple/core/Config.h +++ b/src/ripple/core/Config.h @@ -43,19 +43,6 @@ namespace ripple { -IniFileSections -parseIniFile (std::string const& strInput, const bool bTrim); - -bool -getSingleSection (IniFileSections& secSource, - std::string const& strSection, std::string& strValue, beast::Journal j); - -int -countSectionEntries (IniFileSections& secSource, std::string const& strSection); - -IniFileSections::mapped_type* -getIniFileSection (IniFileSections& secSource, std::string const& strSection); - class Rules; //------------------------------------------------------------------------------ @@ -91,31 +78,10 @@ struct SizedItem class Config : public BasicConfig { public: - struct Helpers - { - // This replaces CONFIG_FILE_NAME - static char const* getConfigFileName () - { - return "rippled.cfg"; - } - - static char const* getDatabaseDirName () - { - return "db"; - } - - static char const* getValidatorsFileName () - { - return "validators.txt"; - } - }; - - //-------------------------------------------------------------------------- - // Settings related to the configuration file location and directories - - /** Returns the directory from which the configuration file was loaded. */ - beast::File getConfigDir () const; + static char const* const configFileName; + static char const* const databaseDirName; + static char const* const validatorsFileName; /** Returns the full path and filename of the debug log file. */ boost::filesystem::path getDebugLogFile () const; @@ -123,31 +89,13 @@ public: /** Returns the full path and filename of the entropy seed file. */ boost::filesystem::path getEntropyFile () const; - // DEPRECATED - boost::filesystem::path CONFIG_FILE; // used by UniqueNodeList private: + boost::filesystem::path CONFIG_FILE; boost::filesystem::path CONFIG_DIR; boost::filesystem::path DEBUG_LOGFILE; void load (); beast::Journal j_; -public: - - //-------------------------------------------------------------------------- - - // Settings related to validators - - /** Return the path to the separate, optional validators file. */ - beast::File getValidatorsFile () const; - - /** Returns the optional URL to a trusted network source of validators. */ - beast::URL getValidatorsURL () const; - - // DEPRECATED - boost::filesystem::path VALIDATORS_FILE; // As specifed in rippled.cfg. - - /** List of Validators entries from rippled.cfg */ - std::vector validators; public: //-------------------------------------------------------------------------- @@ -164,9 +112,6 @@ public: bool SILENT = false; // No output to console after startup. bool ELB_SUPPORT = false; - std::string VALIDATORS_SITE; // Where to find validators.txt on the Internet. - std::string VALIDATORS_URI; // URI of validators.txt. - std::string VALIDATORS_BASE; // Name std::vector IPS; // Peer IPs from rippled.cfg. std::vector IPS_FIXED; // Fixed Peer IPs from rippled.cfg. std::vector SNTP_SERVERS; // SNTP servers from rippled.cfg. @@ -225,8 +170,7 @@ public: RippleAddress VALIDATION_PUB; RippleAddress VALIDATION_PRIV; - // Node/Cluster - std::vector CLUSTER_NODES; + // Node RippleAddress NODE_SEED; RippleAddress NODE_PUB; RippleAddress NODE_PRIV; diff --git a/src/ripple/core/ConfigSections.h b/src/ripple/core/ConfigSections.h index 0d1685b67d..b6b53107d4 100644 --- a/src/ripple/core/ConfigSections.h +++ b/src/ripple/core/ConfigSections.h @@ -64,7 +64,6 @@ struct ConfigSection #define SECTION_VALIDATION_SEED "validation_seed" #define SECTION_WEBSOCKET_PING_FREQ "websocket_ping_frequency" #define SECTION_VALIDATORS "validators" -#define SECTION_VALIDATORS_SITE "validators_site" } // ripple diff --git a/src/ripple/core/impl/Config.cpp b/src/ripple/core/impl/Config.cpp index a5819e9c68..2594c819eb 100644 --- a/src/ripple/core/impl/Config.cpp +++ b/src/ripple/core/impl/Config.cpp @@ -35,10 +35,7 @@ #include #include #include - -#ifndef DUMP_CONFIG -#define DUMP_CONFIG 0 -#endif +#include namespace ripple { @@ -110,15 +107,6 @@ getIniFileSection (IniFileSections& secSource, std::string const& strSection) return smtResult; } -int -countSectionEntries (IniFileSections& secSource, std::string const& strSection) -{ - IniFileSections::mapped_type* pmtEntries = - getIniFileSection (secSource, strSection); - - return pmtEntries ? pmtEntries->size () : 0; -} - bool getSingleSection (IniFileSections& secSource, std::string const& strSection, std::string& strValue, beast::Journal j) { @@ -140,52 +128,16 @@ bool getSingleSection (IniFileSections& secSource, return bSingle; } -/** Parses a set of strings into IP::Endpoint - Strings which fail to parse are not included in the output. If a stream is - provided, human readable diagnostic error messages are written for each - failed parse. - @param out An OutputSequence to store the IP::Endpoint list - @param first The begining of the string input sequence - @param last The one-past-the-end of the string input sequence -*/ -template -void -parseAddresses (OutputSequence& out, InputIterator first, InputIterator last, - beast::Journal::Stream stream = beast::Journal::Stream ()) -{ - while (first != last) - { - auto const str (*first); - ++first; - { - beast::IP::Endpoint const addr ( - beast::IP::Endpoint::from_string (str)); - if (! is_unspecified (addr)) - { - out.push_back (addr); - continue; - } - } - { - beast::IP::Endpoint const addr ( - beast::IP::Endpoint::from_string_altform (str)); - if (! is_unspecified (addr)) - { - out.push_back (addr); - continue; - } - } - if (stream) stream << - "Config: \"" << str << "\" is not a valid IP address."; - } -} - //------------------------------------------------------------------------------ // // Config (DEPRECATED) // //------------------------------------------------------------------------------ +char const* const Config::configFileName = "rippled.cfg"; +char const* const Config::databaseDirName = "db"; +char const* const Config::validatorsFileName = "validators.txt"; + static std::string getEnvVar (char const* name) @@ -214,12 +166,12 @@ void Config::setup (std::string const& strConf, bool bQuiet) QUIET = bQuiet; - strDbPath = Helpers::getDatabaseDirName (); - strConfFile = strConf.empty () ? Helpers::getConfigFileName () : strConf; + strDbPath = databaseDirName; - VALIDATORS_BASE = Helpers::getValidatorsFileName (); - - VALIDATORS_URI = boost::str (boost::format ("/%s") % VALIDATORS_BASE); + if (!strConf.empty()) + strConfFile = strConf; + else + strConfFile = configFileName; if (!strConf.empty ()) { @@ -328,12 +280,6 @@ void Config::loadFromString (std::string const& fileContents) build (secConfig); - if (auto s = getIniFileSection (secConfig, SECTION_VALIDATORS)) - validators = *s; - - if (auto s = getIniFileSection (secConfig, SECTION_CLUSTER_NODES)) - CLUSTER_NODES = *s; - if (auto s = getIniFileSection (secConfig, SECTION_IPS)) IPS = *s; @@ -372,8 +318,6 @@ void Config::loadFromString (std::string const& fileContents) } } - (void) getSingleSection (secConfig, SECTION_VALIDATORS_SITE, VALIDATORS_SITE, j_); - std::string strTemp; if (getSingleSection (secConfig, SECTION_PEER_PRIVATE, strTemp, j_)) @@ -489,9 +433,84 @@ void Config::loadFromString (std::string const& fileContents) if (getSingleSection (secConfig, SECTION_PATH_SEARCH_MAX, strTemp, j_)) PATH_SEARCH_MAX = beast::lexicalCastThrow (strTemp); + // If a file was explicitly specified, then warn if the path is malformed + // or the file does not exist or is not a file. + // If no path was specified, then look for validators.txt in the same path + // as the config file - don't complain if we can't find it. + boost::filesystem::path validatorsFile; + if (getSingleSection (secConfig, SECTION_VALIDATORS_FILE, strTemp, j_)) { - VALIDATORS_FILE = strTemp; + validatorsFile = strTemp; + + if (validatorsFile.empty ()) + { + JLOG (j_.error) << + "[" SECTION_VALIDATORS_FILE "]" << + ": " << strTemp << + " is not a valid path"; + validatorsFile.clear (); + } + else if (!boost::filesystem::exists (validatorsFile)) + { + JLOG (j_.error) << + "[" SECTION_VALIDATORS_FILE "]" << + ": the file " << validatorsFile << + " does not exist"; + validatorsFile.clear (); + } + else if (!boost::filesystem::is_regular_file (validatorsFile)) + { + JLOG (j_.error) << + "[" SECTION_VALIDATORS_FILE "]" << + ": the file " << validatorsFile << + " is not a regular file"; + validatorsFile.clear (); + } + } + else + { + validatorsFile = CONFIG_DIR / validatorsFileName; + + if (!validatorsFile.empty ()) + { + if(!boost::filesystem::exists (validatorsFile)) + validatorsFile.clear(); + else if (!boost::filesystem::is_regular_file (validatorsFile)) + validatorsFile.clear(); + } + } + + if (!validatorsFile.empty () && + boost::filesystem::exists (validatorsFile) && + boost::filesystem::is_regular_file (validatorsFile)) + { + std::ifstream ifsDefault (validatorsFile.native().c_str()); + + std::string data; + + data.assign ( + std::istreambuf_iterator(ifsDefault), + std::istreambuf_iterator()); + + auto iniFile = parseIniFile (data, true); + + auto entries = getIniFileSection ( + iniFile, + SECTION_VALIDATORS); + + if (!entries) + { + JLOG (j_.error) << + "[" SECTION_VALIDATORS_FILE "]" << + ": the file " << validatorsFile << + " does not contain a [" SECTION_VALIDATORS << + "] section"; + } + else + { + section (SECTION_VALIDATORS).append (*entries); + } } if (getSingleSection (secConfig, SECTION_DEBUG_LOGFILE, strTemp, j_)) @@ -575,27 +594,6 @@ boost::filesystem::path Config::getDebugLogFile () const return log_file; } -beast::File Config::getConfigDir () const -{ - beast::String const s (CONFIG_FILE.native().c_str ()); - if (s.isNotEmpty ()) - return beast::File (s).getParentDirectory (); - return beast::File::nonexistent (); -} - -beast::File Config::getValidatorsFile () const -{ - beast::String const s (VALIDATORS_FILE.native().c_str()); - if (s.isNotEmpty() && getConfigDir() != beast::File::nonexistent()) - return getConfigDir().getChildFile (s); - return beast::File::nonexistent (); -} - -beast::URL Config::getValidatorsURL () const -{ - return beast::parse_URL (VALIDATORS_SITE).second; -} - beast::File Config::getModuleDatabasePath () const { boost::filesystem::path dbPath (legacy ("database_path")); diff --git a/src/ripple/core/tests/Config.test.cpp b/src/ripple/core/tests/Config.test.cpp index 3e51e65e1a..b21fdca2a4 100644 --- a/src/ripple/core/tests/Config.test.cpp +++ b/src/ripple/core/tests/Config.test.cpp @@ -156,10 +156,10 @@ public: if (dbPath.empty ()) { - dataDir_ = subDir_ / path (Config::Helpers::getDatabaseDirName ()); + dataDir_ = subDir_ / path (Config::databaseDirName); } - configFile_ = subDir_ / path (Config::Helpers::getConfigFileName ()); + configFile_ = subDir_ / path (Config::configFileName); { if (!exists (subDir_)) { @@ -338,7 +338,7 @@ port_wss_admin auto& c (g.config ()); std::string const nativeDbPath = absolute (path ("test_db") / - path (Config::Helpers::getDatabaseDirName ())) + path (Config::databaseDirName)) .string (); expect (g.dataDirExists ()); expect (g.configFileExists ()); diff --git a/src/ripple/net/impl/RPCCall.cpp b/src/ripple/net/impl/RPCCall.cpp index 50635f96ee..1609cbf1aa 100644 --- a/src/ripple/net/impl/RPCCall.cpp +++ b/src/ripple/net/impl/RPCCall.cpp @@ -953,10 +953,6 @@ public: { "unl_add", &RPCParser::parseUnlAdd, 1, 2 }, { "unl_delete", &RPCParser::parseUnlDelete, 1, 1 }, { "unl_list", &RPCParser::parseAsIs, 0, 0 }, - { "unl_load", &RPCParser::parseAsIs, 0, 0 }, - { "unl_network", &RPCParser::parseAsIs, 0, 0 }, - { "unl_reset", &RPCParser::parseAsIs, 0, 0 }, - { "unl_score", &RPCParser::parseAsIs, 0, 0 }, { "validation_create", &RPCParser::parseValidationCreate, 0, 1 }, { "validation_seed", &RPCParser::parseValidationSeed, 0, 1 }, { "version", &RPCParser::parseAsIs, 0, 0 }, diff --git a/src/ripple/overlay/Cluster.h b/src/ripple/overlay/Cluster.h index b83ca8f291..1685c0f418 100644 --- a/src/ripple/overlay/Cluster.h +++ b/src/ripple/overlay/Cluster.h @@ -23,7 +23,7 @@ #include #include #include -#include +#include #include #include #include @@ -76,9 +76,9 @@ public: Cluster (beast::Journal j); /** Determines whether a node belongs in the cluster - @return empty optional if the node isn't a member, - otherwise, the node's name (which may be - empty). + @return boost::none if the node isn't a member, + otherwise, the comment associated with the + node (which may be an empty string). */ boost::optional member (RippleAddress const& node) const; @@ -106,10 +106,20 @@ public: void for_each ( std::function func) const; -}; -std::unique_ptr -make_Cluster (Config const& config, beast::Journal j); + /** Load the list of cluster nodes. + + The section contains entries consisting of a base58 + encoded node public key, optionally followed by + a comment. + + @return false if an entry could not be parsed or + contained an invalid node public key, + true otherwise. + */ + bool + load (Section const& nodes); +}; } // ripple diff --git a/src/ripple/overlay/impl/Cluster.cpp b/src/ripple/overlay/impl/Cluster.cpp index 4e80b7fbda..064a4ae7ac 100644 --- a/src/ripple/overlay/impl/Cluster.cpp +++ b/src/ripple/overlay/impl/Cluster.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -93,51 +94,52 @@ Cluster::for_each ( func (ni); } -std::unique_ptr -make_Cluster (Config const& config, beast::Journal j) +bool +Cluster::load (Section const& nodes) { static boost::regex const re ( - "^" // start of line - "(?:\\s*)" // whitespace (optional) - "([a-zA-Z0-9]*)" // Node identity - "(?:\\s*)" // whitespace (optional) - "(.*\\S*)" // - "(?:\\s*)" // whitespace (optional) + "[[:space:]]*" // skip leading whitespace + "([[:alnum:]]+)" // node identity + "(?:" // begin optional comment block + "[[:space:]]+" // (skip all leading whitespace) + "(?:" // begin optional comment + "(.*[^[:space:]]+)" // the comment + "[[:space:]]*" // (skip all trailing whitespace) + ")?" // end optional comment + ")?" // end optional comment block ); - auto cluster = std::make_unique (j); - - for (auto const& n : config.CLUSTER_NODES) + for (auto const& n : nodes.values()) { boost::smatch match; if (!boost::regex_match (n, match, re)) { - JLOG (j.error) << + JLOG (j_.error) << "Malformed entry: '" << n << "'"; - continue; + return false; } auto const nid = RippleAddress::createNodePublic (match[1]); if (!nid.isValid()) { - JLOG (j.error) << + JLOG (j_.error) << "Invalid node identity: " << match[1]; - continue; + return false; } - if (cluster->member (nid)) + if (member (nid)) { - JLOG (j.warning) << + JLOG (j_.warning) << "Duplicate node identity: " << match[1]; continue; } - cluster->update(nid, match[2]); + update(nid, trim_whitespace(match[2])); } - return cluster; + return true; } } // ripple diff --git a/src/ripple/overlay/impl/ConnectAttempt.h b/src/ripple/overlay/impl/ConnectAttempt.h index 66ea7a3c77..a40670d617 100644 --- a/src/ripple/overlay/impl/ConnectAttempt.h +++ b/src/ripple/overlay/impl/ConnectAttempt.h @@ -26,7 +26,6 @@ #include #include #include -#include // move to .cpp #include #include #include diff --git a/src/ripple/overlay/impl/Manifest.cpp b/src/ripple/overlay/impl/Manifest.cpp index 5dd8647e8e..de770e45f4 100644 --- a/src/ripple/overlay/impl/Manifest.cpp +++ b/src/ripple/overlay/impl/Manifest.cpp @@ -18,8 +18,8 @@ //============================================================================== #include -#include #include +#include #include #include #include @@ -158,7 +158,7 @@ ManifestCache::configValidatorKey( void ManifestCache::configManifest ( - Manifest m, UniqueNodeList& unl, beast::Journal journal) + Manifest m, ValidatorList& unl, beast::Journal journal) { if (! m.verify()) { @@ -228,7 +228,7 @@ ManifestCache::canApply (PublicKey const& pk, std::uint32_t seq, ManifestDisposition ManifestCache::applyManifest ( - Manifest m, UniqueNodeList& unl, beast::Journal journal) + Manifest m, ValidatorList& unl, beast::Journal journal) { { std::lock_guard lock (mutex_); @@ -307,7 +307,7 @@ ManifestCache::applyManifest ( m.masterKey, m.sequence, old->sequence); } - unl.deleteEphemeralKey (old->signingKey); + unl.removeEphemeralKey (old->signingKey); } if (m.revoked ()) @@ -335,7 +335,7 @@ ManifestCache::applyManifest ( } void ManifestCache::load ( - DatabaseCon& dbCon, UniqueNodeList& unl, beast::Journal journal) + DatabaseCon& dbCon, ValidatorList& unl, beast::Journal journal) { static const char* const sql = "SELECT RawData FROM ValidatorManifests;"; diff --git a/src/ripple/overlay/impl/Manifest.h b/src/ripple/overlay/impl/Manifest.h index 75f2943ebb..2e5a39660a 100644 --- a/src/ripple/overlay/impl/Manifest.h +++ b/src/ripple/overlay/impl/Manifest.h @@ -20,7 +20,7 @@ #ifndef RIPPLE_OVERLAY_MANIFEST_H_INCLUDED #define RIPPLE_OVERLAY_MANIFEST_H_INCLUDED -#include +#include #include #include #include @@ -162,16 +162,16 @@ public: ~ManifestCache() = default; void configValidatorKey(std::string const& line, beast::Journal journal); - void configManifest (Manifest m, UniqueNodeList& unl, beast::Journal journal); + void configManifest (Manifest m, ValidatorList& unl, beast::Journal journal); void addTrustedKey (PublicKey const& pk, std::string comment); ManifestDisposition applyManifest ( - Manifest m, UniqueNodeList& unl, beast::Journal journal); + Manifest m, ValidatorList& unl, beast::Journal journal); void load ( - DatabaseCon& dbCon, UniqueNodeList& unl, beast::Journal journal); + DatabaseCon& dbCon, ValidatorList& unl, beast::Journal journal); void save (DatabaseCon& dbCon) const; // A "for_each" for populated manifests only diff --git a/src/ripple/overlay/impl/OverlayImpl.cpp b/src/ripple/overlay/impl/OverlayImpl.cpp index e7c6bdabde..b2ebee5060 100644 --- a/src/ripple/overlay/impl/OverlayImpl.cpp +++ b/src/ripple/overlay/impl/OverlayImpl.cpp @@ -463,7 +463,10 @@ OverlayImpl::setupValidatorKeyManifests (BasicConfig const& config, s = beast::base64_decode(s); if (auto mo = make_Manifest (std::move (s))) { - manifestCache_.configManifest (std::move (*mo), app_.getUNL (), journal_); + manifestCache_.configManifest ( + std::move (*mo), + app_.validators(), + journal_); } else { @@ -476,7 +479,10 @@ OverlayImpl::setupValidatorKeyManifests (BasicConfig const& config, journal_.warning << "No [validation_manifest] section in config"; } - manifestCache_.load (db, app_.getUNL(), journal_); + manifestCache_.load ( + db, + app_.validators(), + journal_); } void @@ -685,8 +691,10 @@ OverlayImpl::onManifests ( continue; auto const serialized = mo->serialized; - auto const result = - manifestCache_.applyManifest (std::move(*mo), app_.getUNL(), journal); + auto const result = manifestCache_.applyManifest ( + std::move(*mo), + app_.validators(), + journal); if (result == ManifestDisposition::accepted) { diff --git a/src/ripple/overlay/impl/PeerImp.cpp b/src/ripple/overlay/impl/PeerImp.cpp index ec71fc3a3b..958e4f06fc 100644 --- a/src/ripple/overlay/impl/PeerImp.cpp +++ b/src/ripple/overlay/impl/PeerImp.cpp @@ -28,8 +28,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -1265,7 +1265,7 @@ PeerImp::onMessage (std::shared_ptr const& m) return; } - bool isTrusted = app_.getUNL ().nodeInUNL (signerPublic); + auto const isTrusted = app_.validators().trusted (signerPublic); if (!isTrusted) { @@ -1594,7 +1594,8 @@ PeerImp::onMessage (std::shared_ptr const& m) return; } - bool isTrusted = app_.getUNL ().nodeInUNL (val->getSignerPublic ()); + auto const isTrusted = + app_.validators().trusted(val->getSignerPublic ()); if (!isTrusted && (sanity_.load () == Sanity::insane)) { p_journal_.debug << @@ -1606,9 +1607,13 @@ PeerImp::onMessage (std::shared_ptr const& m) app_.getJobQueue ().addJob ( isTrusted ? jtVALIDATION_t : jtVALIDATION_ut, "recvValidation->checkValidation", - [weak, val, isTrusted, m] (Job&) { + [weak, val, isTrusted, m] (Job&) + { if (auto peer = weak.lock()) - peer->checkValidation(val, isTrusted, m); + peer->checkValidation( + val, + isTrusted, + m); }); } else diff --git a/src/ripple/overlay/tests/cluster_test.cpp b/src/ripple/overlay/tests/cluster_test.cpp index b2de18a3fe..661e12838a 100644 --- a/src/ripple/overlay/tests/cluster_test.cpp +++ b/src/ripple/overlay/tests/cluster_test.cpp @@ -18,6 +18,7 @@ //============================================================================== #include +#include #include #include #include @@ -29,7 +30,7 @@ class cluster_test : public ripple::TestSuite { public: std::unique_ptr - make_Cluster (std::vector const& nodes) + create (std::vector const& nodes) { auto cluster = std::make_unique (beast::Journal ()); @@ -58,7 +59,7 @@ public: { testcase ("Membership: Empty cluster"); - auto c = make_Cluster ({}); + auto c = create ({}); for (auto const& n : network) expect (!c->member (n)); @@ -71,7 +72,7 @@ public: while (cluster.size () != 32) cluster.push_back (randomNode()); - auto c = make_Cluster (cluster); + auto c = create (cluster); for (auto const& n : network) expect (!c->member (n)); @@ -86,7 +87,7 @@ public: while (cluster.size () != 32) cluster.push_back (randomNode()); - auto c = make_Cluster (cluster); + auto c = create (cluster); for (auto const& n : cluster) expect (c->member (n)); @@ -106,7 +107,7 @@ public: std::vector cluster ( network.begin (), network.begin () + 32); - auto c = make_Cluster (cluster); + auto c = create (cluster); for (auto const& n : cluster) expect (c->member (n)); @@ -126,7 +127,7 @@ public: { testcase ("Updating"); - auto c = make_Cluster ({}); + auto c = create ({}); auto const node = randomNode (); std::uint32_t load = 0; @@ -177,11 +178,78 @@ public: } } + void + testConfigLoad () + { + testcase ("Config Load"); + + auto c = std::make_unique (beast::Journal ()); + + // The servers on the network + std::vector network; + + while (network.size () != 8) + network.push_back (randomNode()); + + Section s1; + + // Correct (empty) configuration + expect (c->load (s1)); + expect (c->size() == 0); + + // Correct configuration + s1.append (network[0].humanNodePublic()); + s1.append (network[1].humanNodePublic() + " Comment"); + s1.append (network[2].humanNodePublic() + " Multi Word Comment"); + s1.append (network[3].humanNodePublic() + " Leading Whitespace"); + s1.append (network[4].humanNodePublic() + " Trailing Whitespace "); + s1.append (network[5].humanNodePublic() + " Leading & Trailing Whitespace "); + s1.append (network[6].humanNodePublic() + " Leading, Trailing & Internal Whitespace "); + s1.append (network[7].humanNodePublic() + " "); + + expect (c->load (s1)); + + for (auto const& n : network) + expect (c->member (n)); + + // Incorrect configurations + Section s2; + s2.append ("NotAPublicKey"); + + expect (!c->load (s2)); + + Section s3; + s3.append ("@" + network[0].humanNodePublic()); + expect (!c->load (s3)); + + Section s4; + s4.append (network[0].humanNodePublic() + "!"); + expect (!c->load (s4)); + + Section s5; + s5.append (network[0].humanNodePublic() + "! Comment"); + expect (!c->load (s5)); + + // Check if we properly terminate when we encounter + // a malformed or unparseable entry: + auto const badNode = randomNode(); + auto const goodNode = randomNode (); + + Section s6; + s6.append (badNode.humanNodePublic() + "XXX"); + s6.append (goodNode.humanNodePublic()); + + expect (!c->load (s6)); + expect (!c->member (badNode)); + expect (!c->member (goodNode)); + } + void run() override { testMembership (); testUpdating (); + testConfigLoad (); } }; diff --git a/src/ripple/overlay/tests/manifest_test.cpp b/src/ripple/overlay/tests/manifest_test.cpp index 92c57fb8bb..188debb449 100644 --- a/src/ripple/overlay/tests/manifest_test.cpp +++ b/src/ripple/overlay/tests/manifest_test.cpp @@ -26,7 +26,6 @@ #include #include #include -#include #include #include @@ -122,7 +121,7 @@ public: return Manifest (m.serialized, m.masterKey, m.signingKey, m.sequence); } - void testLoadStore (ManifestCache const& m, UniqueNodeList& unl) + void testLoadStore (ManifestCache const& m, ValidatorList& unl) { testcase ("load/store"); @@ -184,8 +183,8 @@ public: run() override { ManifestCache cache; - test::jtx::Env env(*this); - auto& unl = env.app().getUNL(); + beast::Journal journal; + auto unl = std::make_unique (journal); { testcase ("apply"); auto const accepted = ManifestDisposition::accepted; @@ -193,8 +192,6 @@ public: auto const stale = ManifestDisposition::stale; auto const invalid = ManifestDisposition::invalid; - beast::Journal journal; - auto const sk_a = randomSecretKey(); auto const pk_a = derivePublicKey(KeyType::ed25519, sk_a); auto const kp_a = randomKeyPair(KeyType::secp256k1); @@ -210,26 +207,26 @@ public: make_Manifest (KeyType::ed25519, sk_b, kp_b.first, 2, true); // broken auto const fake = s_b1.serialized + '\0'; - expect (cache.applyManifest (clone (s_a0), unl, journal) == untrusted, + expect (cache.applyManifest (clone (s_a0), *unl, journal) == untrusted, "have to install a trusted key first"); cache.addTrustedKey (pk_a, "a"); cache.addTrustedKey (pk_b, "b"); - expect (cache.applyManifest (clone (s_a0), unl, journal) == accepted); - expect (cache.applyManifest (clone (s_a0), unl, journal) == stale); + expect (cache.applyManifest (clone (s_a0), *unl, journal) == accepted); + expect (cache.applyManifest (clone (s_a0), *unl, journal) == stale); - expect (cache.applyManifest (clone (s_a1), unl, journal) == accepted); - expect (cache.applyManifest (clone (s_a1), unl, journal) == stale); - expect (cache.applyManifest (clone (s_a0), unl, journal) == stale); + expect (cache.applyManifest (clone (s_a1), *unl, journal) == accepted); + expect (cache.applyManifest (clone (s_a1), *unl, journal) == stale); + expect (cache.applyManifest (clone (s_a0), *unl, journal) == stale); - expect (cache.applyManifest (clone (s_b0), unl, journal) == accepted); - expect (cache.applyManifest (clone (s_b0), unl, journal) == stale); + expect (cache.applyManifest (clone (s_b0), *unl, journal) == accepted); + expect (cache.applyManifest (clone (s_b0), *unl, journal) == stale); expect (!ripple::make_Manifest(fake)); - expect (cache.applyManifest (clone (s_b2), unl, journal) == invalid); + expect (cache.applyManifest (clone (s_b2), *unl, journal) == invalid); } - testLoadStore (cache, unl); + testLoadStore (cache, *unl); } }; diff --git a/src/ripple/protocol/JsonFields.h b/src/ripple/protocol/JsonFields.h index 7e0f8284a1..04f2631da4 100644 --- a/src/ripple/protocol/JsonFields.h +++ b/src/ripple/protocol/JsonFields.h @@ -75,7 +75,7 @@ JSS ( action ); JSS ( acquiring ); // out: LedgerRequest JSS ( address ); // out: PeerImp JSS ( affected ); // out: AcceptedLedgerTx -JSS ( age ); // out: UniqueNodeList, NetworkOPs, Peers +JSS ( age ); // out: NetworkOPs, Peers JSS ( alternatives ); // out: PathRequest, RipplePathFind JSS ( amendment_blocked ); // out: NetworkOPs JSS ( asks ); // out: Subscribe diff --git a/src/ripple/rpc/handlers/UnlAdd.cpp b/src/ripple/rpc/handlers/UnlAdd.cpp index 9bf263b2fa..d6b7e66a8e 100644 --- a/src/ripple/rpc/handlers/UnlAdd.cpp +++ b/src/ripple/rpc/handlers/UnlAdd.cpp @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include @@ -31,32 +31,28 @@ namespace ripple { // { -// node: |, +// node: , // comment: // optional // } Json::Value doUnlAdd (RPC::Context& context) { auto lock = beast::make_lock(context.app.getMasterMutex()); - std::string strNode = context.params.isMember (jss::node) - ? context.params[jss::node].asString () : ""; - std::string strComment = context.params.isMember (jss::comment) - ? context.params[jss::comment].asString () : ""; + if (!context.params.isMember (jss::node)) + return rpcError (rpcINVALID_PARAMS); RippleAddress raNodePublic; - - if (raNodePublic.setNodePublic (strNode)) + if (raNodePublic.setNodePublic (context.params[jss::node].asString ())) { - context.app.getUNL ().nodeAddPublic ( - raNodePublic, UniqueNodeList::vsManual, strComment); + context.app.validators().insertPermanentKey ( + raNodePublic, + context.params.isMember (jss::comment) + ? context.params[jss::comment].asString () + : ""); return RPC::makeObjectValue ("adding node by public key"); } - else - { - context.app.getUNL ().nodeAddDomain ( - strNode, UniqueNodeList::vsManual, strComment); - return RPC::makeObjectValue ("adding node by domain"); - } + + return rpcError (rpcINVALID_PARAMS); } } // ripple diff --git a/src/ripple/rpc/handlers/UnlDelete.cpp b/src/ripple/rpc/handlers/UnlDelete.cpp index a760538b4e..c52ccf958b 100644 --- a/src/ripple/rpc/handlers/UnlDelete.cpp +++ b/src/ripple/rpc/handlers/UnlDelete.cpp @@ -19,7 +19,7 @@ #include #include -#include +#include #include #include #include @@ -39,19 +39,14 @@ Json::Value doUnlDelete (RPC::Context& context) if (!context.params.isMember (jss::node)) return rpcError (rpcINVALID_PARAMS); - auto strNode = context.params[jss::node].asString (); RippleAddress raNodePublic; - - if (raNodePublic.setNodePublic (strNode)) + if (raNodePublic.setNodePublic (context.params[jss::node].asString ())) { - context.app.getUNL ().nodeRemovePublic (raNodePublic); + context.app.validators().removePermanentKey (raNodePublic); return RPC::makeObjectValue ("removing node by public key"); } - else - { - context.app.getUNL ().nodeRemoveDomain (strNode); - return RPC::makeObjectValue ("removing node by domain"); - } + + return rpcError (rpcINVALID_PARAMS); } } // ripple diff --git a/src/ripple/rpc/handlers/UnlList.cpp b/src/ripple/rpc/handlers/UnlList.cpp index ef5424264f..4c6ac4b431 100644 --- a/src/ripple/rpc/handlers/UnlList.cpp +++ b/src/ripple/rpc/handlers/UnlList.cpp @@ -19,7 +19,7 @@ #include #include -#include +#include #include #include #include @@ -31,7 +31,22 @@ Json::Value doUnlList (RPC::Context& context) auto lock = beast::make_lock(context.app.getMasterMutex()); Json::Value obj (Json::objectValue); - obj[jss::unl] = context.app.getUNL ().getUnlJson (); + context.app.validators().for_each ( + [&unl = obj[jss::unl]]( + PublicKey const& publicKey, + std::string const& comment, + bool ephemeral) + { + Json::Value node (Json::objectValue); + + node["publicKey"] = toBase58( + TokenType::TOKEN_NODE_PUBLIC, publicKey); + node["ephemeral"] = ephemeral; + if (!comment.empty()) + node["comment"] = comment; + + unl.append (node); + }); return obj; } diff --git a/src/ripple/rpc/handlers/UnlLoad.cpp b/src/ripple/rpc/handlers/UnlLoad.cpp deleted file mode 100644 index 6ba0dd39d7..0000000000 --- a/src/ripple/rpc/handlers/UnlLoad.cpp +++ /dev/null @@ -1,45 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012-2014 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ripple { - -// Populate the UNL from a local validators.txt file. -Json::Value doUnlLoad (RPC::Context& context) -{ - auto lock = beast::make_lock(context.app.getMasterMutex()); - - if (context.app.config().VALIDATORS_FILE.empty () - || !context.app.getUNL ().nodeLoad (context.app.config().VALIDATORS_FILE)) - { - return rpcError (rpcLOAD_FAILED); - } - - return RPC::makeObjectValue ("loading"); -} - -} // ripple diff --git a/src/ripple/rpc/handlers/UnlNetwork.cpp b/src/ripple/rpc/handlers/UnlNetwork.cpp deleted file mode 100644 index d1a8fbed2b..0000000000 --- a/src/ripple/rpc/handlers/UnlNetwork.cpp +++ /dev/null @@ -1,41 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012-2014 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include - -namespace ripple { - -namespace RPC { -struct Context; -} - -// Populate the UNL from ripple.com's validators.txt file. -Json::Value doUnlNetwork (RPC::Context& context) -{ - auto lock = beast::make_lock(context.app.getMasterMutex()); - context.app.getUNL ().nodeNetwork (); - - return RPC::makeObjectValue ("fetching"); -} - -} // ripple diff --git a/src/ripple/rpc/handlers/UnlReset.cpp b/src/ripple/rpc/handlers/UnlReset.cpp deleted file mode 100644 index 98554d7dfc..0000000000 --- a/src/ripple/rpc/handlers/UnlReset.cpp +++ /dev/null @@ -1,40 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012-2014 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include - -namespace ripple { - -namespace RPC { -struct Context; -} - -Json::Value doUnlReset (RPC::Context& context) -{ - auto lock = beast::make_lock(context.app.getMasterMutex()); - context.app.getUNL ().nodeReset (); - - return RPC::makeObjectValue ("removing nodes"); -} - -} // ripple diff --git a/src/ripple/rpc/handlers/UnlScore.cpp b/src/ripple/rpc/handlers/UnlScore.cpp deleted file mode 100644 index 8b2ecb6384..0000000000 --- a/src/ripple/rpc/handlers/UnlScore.cpp +++ /dev/null @@ -1,41 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012-2014 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ripple { - -// unl_score -Json::Value doUnlScore (RPC::Context& context) -{ - auto lock = beast::make_lock(context.app.getMasterMutex()); - context.app.getUNL ().nodeScore (); - - return RPC::makeObjectValue ("scoring requested"); -} - -} // ripple diff --git a/src/ripple/rpc/impl/Handler.cpp b/src/ripple/rpc/impl/Handler.cpp index b6e18fcea9..fdca94dd3b 100644 --- a/src/ripple/rpc/impl/Handler.cpp +++ b/src/ripple/rpc/impl/Handler.cpp @@ -149,10 +149,6 @@ Handler handlerArray[] { { "unl_add", byRef (&doUnlAdd), Role::ADMIN, NO_CONDITION }, { "unl_delete", byRef (&doUnlDelete), Role::ADMIN, NO_CONDITION }, { "unl_list", byRef (&doUnlList), Role::ADMIN, NO_CONDITION }, - { "unl_load", byRef (&doUnlLoad), Role::ADMIN, NO_CONDITION }, - { "unl_network", byRef (&doUnlNetwork), Role::ADMIN, NO_CONDITION }, - { "unl_reset", byRef (&doUnlReset), Role::ADMIN, NO_CONDITION }, - { "unl_score", byRef (&doUnlScore), Role::ADMIN, NO_CONDITION }, { "validation_create", byRef (&doValidationCreate), Role::ADMIN, NO_CONDITION }, { "validation_seed", byRef (&doValidationSeed), Role::ADMIN, NO_CONDITION }, { "wallet_propose", byRef (&doWalletPropose), Role::ADMIN, NO_CONDITION }, diff --git a/src/ripple/unity/app_misc.cpp b/src/ripple/unity/app_misc.cpp index dbcae13909..75745f9f61 100644 --- a/src/ripple/unity/app_misc.cpp +++ b/src/ripple/unity/app_misc.cpp @@ -25,9 +25,9 @@ #include #include #include -#include #include #include #include #include +#include diff --git a/src/ripple/unity/app_tests.cpp b/src/ripple/unity/app_tests.cpp index 0633beccb8..96626d9cac 100644 --- a/src/ripple/unity/app_tests.cpp +++ b/src/ripple/unity/app_tests.cpp @@ -34,3 +34,4 @@ #include #include #include +#include diff --git a/src/ripple/unity/rpcx.cpp b/src/ripple/unity/rpcx.cpp index 22b395b22f..775b8ba88c 100644 --- a/src/ripple/unity/rpcx.cpp +++ b/src/ripple/unity/rpcx.cpp @@ -83,10 +83,6 @@ #include #include #include -#include -#include -#include -#include #include #include #include