diff --git a/src/ripple/app/misc/IHashRouter.h b/src/ripple/app/misc/IHashRouter.h index b7f513400..074d3a20a 100644 --- a/src/ripple/app/misc/IHashRouter.h +++ b/src/ripple/app/misc/IHashRouter.h @@ -80,10 +80,6 @@ public: virtual int getFlags (uint256 const& index) = 0; virtual bool swapSet (uint256 const& index, std::set& peers, int flag) = 0; - - // VFALCO TODO This appears to be unused! - // -// virtual Entry getEntry (uint256 const&) = 0; }; } // ripple diff --git a/src/ripple/app/misc/UniqueNodeList.cpp b/src/ripple/app/misc/UniqueNodeList.cpp index e5d0eab76..dd73bba06 100644 --- a/src/ripple/app/misc/UniqueNodeList.cpp +++ b/src/ripple/app/misc/UniqueNodeList.cpp @@ -76,14 +76,15 @@ namespace ripple { #define REFERRAL_IPS_MAX 50 template +static std::string -strJoin (Iterator first, Iterator last, std::string strSeperator) +strJoin (Iterator first, Iterator last, std::string strSeparator) { std::ostringstream ossValues; - for (Iterator start = first; first != last; first++) + for (Iterator start = first; first != last; ++first) { - ossValues << str (boost::format ("%s%s") % (start == first ? "" : strSeperator) % *first); + ossValues << str (boost::format ("%s%s") % (start == first ? "" : strSeparator) % *first); } return ossValues.str (); @@ -196,1981 +197,13 @@ private: std::vector viReferrals; }; -public: - explicit UniqueNodeListImp (Stoppable& parent) - : UniqueNodeList (parent) - , m_scoreTimer (this) - , mFetchActive (0) - , m_fetchTimer (this) - { - node_file_name_ = std::string (systemName ()) + ".txt"; - node_file_path_ = "/" + node_file_name_; - } - - //-------------------------------------------------------------------------- - - void onStop () - { - m_fetchTimer.cancel (); - m_scoreTimer.cancel (); - - stopped (); - } - - //-------------------------------------------------------------------------- - - void doScore () - { - mtpScoreNext = boost::posix_time::ptime (boost::posix_time::not_a_date_time); // Timer not set. - mtpScoreStart = boost::posix_time::second_clock::universal_time (); // Scoring. - - WriteLog (lsTRACE, UniqueNodeList) << "Scoring: Start"; - - scoreCompute (); - - WriteLog (lsTRACE, UniqueNodeList) << "Scoring: End"; - - // Save update time. - mtpScoreUpdated = mtpScoreStart; - miscSave (); - - mtpScoreStart = boost::posix_time::ptime (boost::posix_time::not_a_date_time); // Not scoring. - - // Score again if needed. - scoreNext (false); - } - - void doFetch () - { - // Time to check for another fetch. - WriteLog (lsTRACE, UniqueNodeList) << "fetchTimerHandler"; - fetchNext (); - } - - void onDeadlineTimer (beast::DeadlineTimer& timer) - { - if (timer == m_scoreTimer) - { - getApp().getJobQueue ().addJob (jtUNL, "UNL.score", - std::bind (&UniqueNodeListImp::doScore, this)); - } - else if (timer == m_fetchTimer) - { - getApp().getJobQueue ().addJob (jtUNL, "UNL.fetch", - std::bind (&UniqueNodeListImp::doFetch, this)); - } - } - - //-------------------------------------------------------------------------- - - // This is called when the application is started. - // Get update times and start fetching and scoring as needed. - void start () - { - miscLoad (); - - WriteLog (lsDEBUG, UniqueNodeList) << "Validator fetch updated: " << mtpFetchUpdated; - WriteLog (lsDEBUG, UniqueNodeList) << "Validator score updated: " << mtpScoreUpdated; - - fetchNext (); // Start fetching. - scoreNext (false); // Start scoring. - } - - //-------------------------------------------------------------------------- - - // Add a trusted node. Called by RPC or other source. - void nodeAddPublic (RippleAddress const& naNodePublic, ValidatorSource vsWhy, std::string const& strComment) - { - seedNode snCurrent; - - bool bFound = getSeedNodes (naNodePublic, snCurrent); - bool bChanged = false; - - if (!bFound) - { - snCurrent.naPublicKey = naNodePublic; - snCurrent.tpNext = boost::posix_time::second_clock::universal_time (); - } - - // Promote source, if needed. - if (!bFound /*|| iSourceScore (vsWhy) >= iSourceScore (snCurrent.vsSource)*/) - { - snCurrent.vsSource = vsWhy; - snCurrent.strComment = strComment; - bChanged = true; - } - - if (vsManual == vsWhy) - { - // A manual add forces immediate scan. - snCurrent.tpNext = boost::posix_time::second_clock::universal_time (); - bChanged = true; - } - - if (bChanged) - setSeedNodes (snCurrent, true); - } - - //-------------------------------------------------------------------------- - - // Queue a domain for a single attempt fetch a ripple.txt. - // --> strComment: only used on vsManual - // YYY As a lot of these may happen at once, would be nice to wrap multiple calls in a transaction. - void nodeAddDomain (std::string strDomain, ValidatorSource vsWhy, std::string const& strComment) - { - boost::trim (strDomain); - boost::to_lower (strDomain); - - // YYY Would be best to verify strDomain is a valid domain. - // WriteLog (lsTRACE) << str(boost::format("nodeAddDomain: '%s' %c '%s'") - // % strDomain - // % vsWhy - // % strComment); - - seedDomain sdCurrent; - - bool bFound = getSeedDomains (strDomain, sdCurrent); - bool bChanged = false; - - if (!bFound) - { - sdCurrent.strDomain = strDomain; - sdCurrent.tpNext = boost::posix_time::second_clock::universal_time (); - } - - // Promote source, if needed. - if (!bFound || iSourceScore (vsWhy) >= iSourceScore (sdCurrent.vsSource)) - { - sdCurrent.vsSource = vsWhy; - sdCurrent.strComment = strComment; - bChanged = true; - } - - if (vsManual == vsWhy) - { - // A manual add forces immediate scan. - sdCurrent.tpNext = boost::posix_time::second_clock::universal_time (); - bChanged = true; - } - - if (bChanged) - setSeedDomains (sdCurrent, true); - } - - //-------------------------------------------------------------------------- - - void nodeRemovePublic (RippleAddress const& naNodePublic) - { - { - auto db = getApp().getWalletDB ().checkoutDb (); - - *db << str ( - boost::format ("DELETE FROM SeedNodes WHERE PublicKey=%s;") % - sqlEscape (naNodePublic.humanNodePublic ())); - *db << str ( - boost::format ("DELETE FROM TrustedNodes WHERE PublicKey=%s;") % - sqlEscape (naNodePublic.humanNodePublic ())); - } - - // YYY Only dirty on successful delete. - fetchDirty (); - - ScopedUNLLockType sl (mUNLLock); - mUNL.erase (naNodePublic.humanNodePublic ()); - } - - //-------------------------------------------------------------------------- - - void nodeRemoveDomain (std::string strDomain) - { - boost::trim (strDomain); - boost::to_lower (strDomain); - - { - auto db = getApp().getWalletDB ().checkoutDb (); - - *db << str (boost::format ("DELETE FROM SeedDomains WHERE Domain=%s;") % sqlEscape (strDomain)); - } - - // YYY Only dirty on successful delete. - fetchDirty (); - } - - //-------------------------------------------------------------------------- - - void nodeReset () - { - { - auto db = getApp().getWalletDB ().checkoutDb (); - - *db << "DELETE FROM SeedDomains;"; - *db << "DELETE FROM SeedNodes;"; - } - - fetchDirty (); - } - - //-------------------------------------------------------------------------- - - // For debugging, schedule forced scoring. - void nodeScore () - { - scoreNext (true); - } - - //-------------------------------------------------------------------------- - - bool nodeInUNL (RippleAddress const& naNodePublic) - { - ScopedUNLLockType sl (mUNLLock); - - return mUNL.end () != mUNL.find (naNodePublic.humanNodePublic ()); - } - - //-------------------------------------------------------------------------- - - bool nodeInCluster (RippleAddress const& naNodePublic) - { - ScopedUNLLockType sl (mUNLLock); - return m_clusterNodes.end () != m_clusterNodes.find (naNodePublic); - } - - //-------------------------------------------------------------------------- - - bool nodeInCluster (RippleAddress const& naNodePublic, std::string& name) - { - ScopedUNLLockType sl (mUNLLock); - std::map::iterator it = m_clusterNodes.find (naNodePublic); - - if (it == m_clusterNodes.end ()) - return false; - - name = it->second.getName(); - return true; - } - - //-------------------------------------------------------------------------- - - bool nodeUpdate (RippleAddress const& naNodePublic, ClusterNodeStatus const& cnsStatus) - { - ScopedUNLLockType sl (mUNLLock); - return m_clusterNodes[naNodePublic].update(cnsStatus); - } - - //-------------------------------------------------------------------------- - - std::map getClusterStatus () - { - std::map ret; - { - ScopedUNLLockType sl (mUNLLock); - ret = m_clusterNodes; - } - return ret; - } - - //-------------------------------------------------------------------------- - - std::uint32_t getClusterFee () - { - int thresh = getApp().getOPs().getNetworkTimeNC() - 90; - - std::vector fees; - { - ScopedUNLLockType sl (mUNLLock); - { - for (std::map::iterator it = m_clusterNodes.begin(), - end = m_clusterNodes.end(); it != end; ++it) - { - if (it->second.getReportTime() >= thresh) - fees.push_back(it->second.getLoadFee()); - } - } - } - - if (fees.empty()) - return 0; - std::sort (fees.begin(), fees.end()); - return fees[fees.size() / 2]; - } - - //-------------------------------------------------------------------------- - - void addClusterStatus (Json::Value& obj) - { - ScopedUNLLockType sl (mUNLLock); - if (m_clusterNodes.size() > 1) // nodes other than us - { - int now = getApp().getOPs().getNetworkTimeNC(); - std::uint32_t ref = getApp().getFeeTrack().getLoadBase(); - Json::Value& nodes = (obj[jss::cluster] = Json::objectValue); - - for (std::map::iterator it = m_clusterNodes.begin(), - end = m_clusterNodes.end(); it != end; ++it) - { - if (it->first != getApp().getLocalCredentials().getNodePublic()) - { - Json::Value& node = nodes[it->first.humanNodePublic()]; - - if (!it->second.getName().empty()) - node["tag"] = it->second.getName(); - - if ((it->second.getLoadFee() != ref) && (it->second.getLoadFee() != 0)) - node["fee"] = static_cast(it->second.getLoadFee()) / ref; - - if (it->second.getReportTime() != 0) - node["age"] = (it->second.getReportTime() >= now) ? 0 : (now - it->second.getReportTime()); - } - } - } - } - - //-------------------------------------------------------------------------- - - void nodeBootstrap () - { - int iDomains = 0; - int iNodes = 0; - -#if 0 - { - auto sl (getApp().getWalletDB ().lock ()); - auto db = getApp().getWalletDB ().getDB (); - - if (db->executeSQL (str (boost::format ("SELECT COUNT(*) AS Count FROM SeedDomains WHERE Source='%s' OR Source='%c';") % vsManual % vsValidator)) && db->startIterRows ()) - iDomains = db->getInt ("Count"); - - db->endIterRows (); - - if (db->executeSQL (str (boost::format ("SELECT COUNT(*) AS Count FROM SeedNodes WHERE Source='%s' OR Source='%c';") % vsManual % vsValidator)) && db->startIterRows ()) - iNodes = db->getInt ("Count"); - - db->endIterRows (); - } -#endif - - bool bLoaded = iDomains || iNodes; - - // Always merge in the file specified in the config. - if (!getConfig ().VALIDATORS_FILE.empty ()) - { - WriteLog (lsINFO, UniqueNodeList) << "Bootstrapping UNL: loading from unl_default."; - - bLoaded = nodeLoad (getConfig ().VALIDATORS_FILE); - } - - // If never loaded anything try the current directory. - if (!bLoaded && getConfig ().VALIDATORS_FILE.empty ()) - { - WriteLog (lsINFO, UniqueNodeList) << boost::str (boost::format ("Bootstrapping UNL: loading from '%s'.") - % getConfig ().VALIDATORS_BASE); - - bLoaded = nodeLoad (getConfig ().VALIDATORS_BASE); - } - - // Always load from rippled.cfg - if (!getConfig ().validators.empty ()) - { - RippleAddress naInvalid; // Don't want a referrer on added entries. - - WriteLog (lsINFO, UniqueNodeList) << boost::str (boost::format ("Bootstrapping UNL: loading from '%s'.") - % getConfig ().CONFIG_FILE); - - if (processValidators ("local", getConfig ().CONFIG_FILE.string (), naInvalid, vsConfig, &getConfig ().validators)) - bLoaded = true; - } - - if (!bLoaded) - { - WriteLog (lsINFO, UniqueNodeList) << boost::str (boost::format ("Bootstrapping UNL: loading from '%s'.") - % getConfig ().VALIDATORS_SITE); - - nodeNetwork (); - } - } - - //-------------------------------------------------------------------------- - - bool nodeLoad (boost::filesystem::path pConfig) - { - if (pConfig.empty ()) - { - WriteLog (lsINFO, UniqueNodeList) << Config::Helpers::getValidatorsFileName() << - " path not specified."; - - return false; - } - - if (!boost::filesystem::exists (pConfig)) - { - WriteLog (lsWARNING, UniqueNodeList) << Config::Helpers::getValidatorsFileName() << - " not found: " << pConfig; - - return false; - } - - if (!boost::filesystem::is_regular_file (pConfig)) - { - WriteLog (lsWARNING, UniqueNodeList) << Config::Helpers::getValidatorsFileName() << - " not regular file: " << pConfig; - - return false; - } - - std::ifstream ifsDefault (pConfig.native ().c_str (), std::ios::in); - - if (!ifsDefault) - { - WriteLog (lsFATAL, UniqueNodeList) << Config::Helpers::getValidatorsFileName() << - " failed to open: " << pConfig; - - return false; - } - - std::string strValidators; - - strValidators.assign ((std::istreambuf_iterator (ifsDefault)), - std::istreambuf_iterator ()); - - if (ifsDefault.bad ()) - { - WriteLog (lsFATAL, UniqueNodeList) << Config::Helpers::getValidatorsFileName() << - "Failed to read: " << pConfig; - - return false; - } - - nodeProcess ("local", strValidators, pConfig.string ()); - - WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("Processing: %s") % pConfig); - - return true; - } - - //-------------------------------------------------------------------------- - - void nodeNetwork () - { - if (!getConfig ().VALIDATORS_SITE.empty ()) - { - HTTPClient::get ( - true, - getApp().getIOService (), - getConfig ().VALIDATORS_SITE, - 443, - getConfig ().VALIDATORS_URI, - VALIDATORS_FILE_BYTES_MAX, - boost::posix_time::seconds (VALIDATORS_FETCH_SECONDS), - std::bind (&UniqueNodeListImp::validatorsResponse, this, - std::placeholders::_1, - std::placeholders::_2, - std::placeholders::_3)); - } - } - - //-------------------------------------------------------------------------- - - Json::Value getUnlJson () - { - - Json::Value ret (Json::arrayValue); - - auto db = getApp().getWalletDB ().checkoutDb (); - - - std::vector, 2>> columns; - selectBlobsIntoStrings(*db, - "SELECT PublicKey, Comment FROM TrustedNodes;", - columns); - for(auto const& strArray : columns) - { - Json::Value node (Json::objectValue); - - node["publicKey"] = strArray[0].value_or(""); - node["comment"] = strArray[1].value_or(""); - - ret.append (node); - } - - return ret; - } - - //-------------------------------------------------------------------------- - - // For each kind of source, have a starting number of points to be distributed. - int iSourceScore (ValidatorSource vsWhy) - { - int iScore = 0; - - switch (vsWhy) - { - case vsConfig: - iScore = 1500; - break; - - case vsInbound: - iScore = 0; - break; - - case vsManual: - iScore = 1500; - break; - - case vsReferral: - iScore = 0; - break; - - case vsTold: - iScore = 0; - break; - - case vsValidator: - iScore = 1000; - break; - - case vsWeb: - iScore = 200; - break; - - default: - throw std::runtime_error ("Internal error: bad ValidatorSource."); - } - - return iScore; - } - - //-------------------------------------------------------------------------- private: - // Load information about when we last updated. - bool miscLoad () - { - auto db = getApp().getWalletDB ().checkoutDb (); - - boost::optional suO, fuO; - - *db << "SELECT ScoreUpdated, FetchUpdated FROM Misc WHERE Magic=1;", - soci::into(suO), soci::into(fuO); - - if (!db->got_data() ) - return false; - - mtpFetchUpdated = ptFromSeconds (fuO.value_or(-1)); - mtpScoreUpdated = ptFromSeconds (suO.value_or(-1)); - - trustedLoad (); - - return true; - } - - //-------------------------------------------------------------------------- - - // Persist update information. - bool miscSave () - { - auto db = getApp().getWalletDB ().checkoutDb (); - - *db << str (boost::format ("REPLACE INTO Misc (Magic,FetchUpdated,ScoreUpdated) VALUES (1,%d,%d);") - % iToSeconds (mtpFetchUpdated) - % iToSeconds (mtpScoreUpdated)); - - return true; - } - - //-------------------------------------------------------------------------- - - void trustedLoad () - { - boost::regex rNode ("\\`\\s*(\\S+)[\\s]*(.*)\\'"); - for (auto const& c : getConfig ().CLUSTER_NODES) - { - boost::smatch match; - - if (boost::regex_match (c, match, rNode)) - { - RippleAddress a = RippleAddress::createNodePublic (match[1]); - - if (a.isValid ()) - m_clusterNodes.insert (std::make_pair (a, ClusterNodeStatus(match[2]))); - } - else - WriteLog (lsWARNING, UniqueNodeList) << "Entry in cluster list invalid: '" << c << "'"; - } - - auto db = getApp().getWalletDB ().checkoutDb (); - ScopedUNLLockType slUNL (mUNLLock); - - mUNL.clear (); - - std::vector, 1>> columns; - selectBlobsIntoStrings(*db, - "SELECT PublicKey FROM TrustedNodes WHERE Score != 0;", - columns); - for(auto const& strArray : columns) - { - mUNL.insert (strArray[0].value_or("")); - } - } - - //-------------------------------------------------------------------------- - - // For a round of scoring we destribute points from a node to nodes it refers to. - // Returns true, iff scores were distributed. - // - bool scoreRound (std::vector& vsnNodes) - { - bool bDist = false; - - // For each node, distribute roundSeed to roundScores. - for (auto& sn : vsnNodes) - { - int iEntries = sn.viReferrals.size (); - - if (sn.iRoundSeed && iEntries) - { - score iTotal = (iEntries + 1) * iEntries / 2; - score iBase = sn.iRoundSeed * iEntries / iTotal; - - // Distribute the current entires' seed score to validators - // prioritized by mention order. - for (int i = 0; i != iEntries; i++) - { - score iPoints = iBase * (iEntries - i) / iEntries; - - vsnNodes[sn.viReferrals[i]].iRoundScore += iPoints; - } - } - } - - if (ShouldLog (lsTRACE, UniqueNodeList)) - { - WriteLog (lsTRACE, UniqueNodeList) << "midway: "; - for (auto& sn : vsnNodes) - { - WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("%s| %d, %d, %d: [%s]") - % sn.strValidator - % sn.iScore - % sn.iRoundScore - % sn.iRoundSeed - % strJoin (sn.viReferrals.begin (), sn.viReferrals.end (), ",")); - } - } - - // Add roundScore to score. - // Make roundScore new roundSeed. - for (auto& sn : vsnNodes) - { - if (!bDist && sn.iRoundScore) - bDist = true; - - sn.iScore += sn.iRoundScore; - sn.iRoundSeed = sn.iRoundScore; - sn.iRoundScore = 0; - } - - if (ShouldLog (lsTRACE, UniqueNodeList)) - { - WriteLog (lsTRACE, UniqueNodeList) << "finish: "; - for (auto& sn : vsnNodes) - { - WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("%s| %d, %d, %d: [%s]") - % sn.strValidator - % sn.iScore - % sn.iRoundScore - % sn.iRoundSeed - % strJoin (sn.viReferrals.begin (), sn.viReferrals.end (), ",")); - } - } - - return bDist; - } - - //-------------------------------------------------------------------------- - - // From SeedDomains and ValidatorReferrals compute scores and update TrustedNodes. - // - // VFALCO TODO Shrink this function, break it up - // - void scoreCompute () - { - hash_map umPulicIdx; // Map of public key to index. - hash_map umDomainIdx; // Map of domain to index. - std::vector vsnNodes; // Index to scoring node. - - // For each entry in SeedDomains with a PublicKey: - // - Add an entry in umPulicIdx, umDomainIdx, and vsnNodes. - { - auto db = getApp().getWalletDB ().checkoutDb (); - - std::vector, 3>> columns; - selectBlobsIntoStrings(*db, - "SELECT Domain,PublicKey,Source FROM SeedDomains;", - columns); - for(auto const& strArray : columns) - { - if (!strArray[1]) - // We ignore entries we don't have public keys for. - continue; - - std::string const strDomain = strArray[0].value_or(""); - std::string const strPublicKey = *strArray[1]; - std::string const strSource = strArray[2].value_or(""); - - assert (!strSource.empty ()); - - int const iScore = iSourceScore (static_cast (strSource[0])); - auto siOld = umPulicIdx.find (strPublicKey); - - if (siOld == umPulicIdx.end ()) - { - // New node - int iNode = vsnNodes.size (); - - umPulicIdx[strPublicKey] = iNode; - umDomainIdx[strDomain] = iNode; - - scoreNode snCurrent; - - snCurrent.strValidator = strPublicKey; - snCurrent.iScore = iScore; - snCurrent.iRoundSeed = snCurrent.iScore; - snCurrent.iRoundScore = 0; - snCurrent.iSeen = -1; - - vsnNodes.push_back (snCurrent); - } - else - { - scoreNode& snOld = vsnNodes[siOld->second]; - - if (snOld.iScore < iScore) - { - // Update old node - - snOld.iScore = iScore; - snOld.iRoundSeed = snOld.iScore; - } - } - } - } - - // For each entry in SeedNodes: - // - Add an entry in umPulicIdx, umDomainIdx, and vsnNodes. - { - auto db = getApp().getWalletDB ().checkoutDb (); - - std::vector, 2>> columns; - selectBlobsIntoStrings(*db, - "SELECT PublicKey,Source FROM SeedNodes;", - columns); - for(auto const& strArray : columns) - { - std::string strPublicKey = strArray[0].value_or(""); - std::string strSource = strArray[1].value_or(""); - assert (!strSource.empty ()); - int iScore = iSourceScore (static_cast (strSource[0])); - auto siOld = umPulicIdx.find (strPublicKey); - - if (siOld == umPulicIdx.end ()) - { - // New node - int iNode = vsnNodes.size (); - - umPulicIdx[strPublicKey] = iNode; - - scoreNode snCurrent; - - snCurrent.strValidator = strPublicKey; - snCurrent.iScore = iScore; - snCurrent.iRoundSeed = snCurrent.iScore; - snCurrent.iRoundScore = 0; - snCurrent.iSeen = -1; - - vsnNodes.push_back (snCurrent); - } - else - { - scoreNode& snOld = vsnNodes[siOld->second]; - - if (snOld.iScore < iScore) - { - // Update old node - - snOld.iScore = iScore; - snOld.iRoundSeed = snOld.iScore; - } - } - } - } - - // For debugging, print out initial scores. - if (ShouldLog (lsTRACE, UniqueNodeList)) - { - for (auto& sn : vsnNodes) - { - WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("%s| %d, %d, %d") - % sn.strValidator - % sn.iScore - % sn.iRoundScore - % sn.iRoundSeed); - } - } - - // WriteLog (lsTRACE, UniqueNodeList) << str(boost::format("vsnNodes.size=%d") % vsnNodes.size()); - - // Step through growing list of nodes adding each validation list. - // - Each validator may have provided referals. Add those referals as validators. - for (int iNode = 0; iNode != vsnNodes.size (); ++iNode) - { - scoreNode& sn = vsnNodes[iNode]; - std::string& strValidator = sn.strValidator; - std::vector& viReferrals = sn.viReferrals; - - auto db = getApp().getWalletDB ().checkoutDb (); - - std::vector, 1>> columns; - selectBlobsIntoStrings(*db, - boost::str (boost::format ( - "SELECT Referral FROM ValidatorReferrals " - "WHERE Validator=%s ORDER BY Entry;") % - sqlEscape (strValidator)), - columns); - std::string strReferral; - for(auto const& strArray : columns) - { - strReferral = strArray[0].value_or(""); - - int iReferral; - - RippleAddress na; - - if (na.setNodePublic (strReferral)) - { - // Referring a public key. - auto itEntry = umPulicIdx.find (strReferral); - - if (itEntry == umPulicIdx.end ()) - { - // Not found add public key to list of nodes. - iReferral = vsnNodes.size (); - - umPulicIdx[strReferral] = iReferral; - - scoreNode snCurrent; - - snCurrent.strValidator = strReferral; - snCurrent.iScore = iSourceScore (vsReferral); - snCurrent.iRoundSeed = snCurrent.iScore; - snCurrent.iRoundScore = 0; - snCurrent.iSeen = -1; - - vsnNodes.push_back (snCurrent); - } - else - { - iReferral = itEntry->second; - } - } - else - { - // Referring a domain. - auto itEntry = umDomainIdx.find (strReferral); - iReferral = itEntry == umDomainIdx.end () - ? -1 // We ignore domains we can't find entires for. - : itEntry->second; - } - - if (iReferral >= 0 && iNode != iReferral) - viReferrals.push_back (iReferral); - } - } - - // - // Distribute the points from the seeds. - // - bool bDist = true; - - for (int i = SCORE_ROUNDS; bDist && i--;) - bDist = scoreRound (vsnNodes); - - if (ShouldLog (lsTRACE, UniqueNodeList)) - { - WriteLog (lsTRACE, UniqueNodeList) << "Scored:"; - for (auto& sn : vsnNodes) - { - WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("%s| %d, %d, %d: [%s]") - % sn.strValidator - % sn.iScore - % sn.iRoundScore - % sn.iRoundSeed - % strJoin (sn.viReferrals.begin (), sn.viReferrals.end (), ",")); - } - } - - // Persist validator scores. - auto db = getApp().getWalletDB ().checkoutDb (); - - soci::transaction tr(*db); - *db << "UPDATE TrustedNodes SET Score = 0 WHERE Score != 0;"; - - if (!vsnNodes.empty ()) - { - // Load existing Seens from DB. - std::vector vstrPublicKeys; - - vstrPublicKeys.resize (vsnNodes.size ()); - - for (int iNode = vsnNodes.size (); iNode--;) - { - vstrPublicKeys[iNode] = sqlEscape (vsnNodes[iNode].strValidator); - } - - // Iterate through the result rows with a fectch b/c putting a - // column of type DATETIME into a boost::tuple can throw when the - // datetime column is invalid (even if the value as int is valid). - std::vector, - boost::optional>> columns; - selectBlobsIntoStrings ( - *db, - str (boost::format ( - "SELECT PublicKey,Seen FROM TrustedNodes WHERE " - "PublicKey IN (%s);") % - strJoin ( - vstrPublicKeys.begin (), vstrPublicKeys.end (), ",")), - columns); - std::string pk; - for(auto const& col : columns) - { - pk = get<0>(col).value_or (""); - - vsnNodes[umPulicIdx[pk]].iSeen = get<1>(col).value_or (-1); - } - } - - hash_set usUNL; - - if (!vsnNodes.empty ()) - { - // Update the score old entries and add new entries as needed. - std::vector vstrValues; - - vstrValues.resize (vsnNodes.size ()); - - for (int iNode = vsnNodes.size (); iNode--;) - { - scoreNode& sn = vsnNodes[iNode]; - std::string strSeen = sn.iSeen >= 0 ? str (boost::format ("%d") % sn.iSeen) : "NULL"; - - vstrValues[iNode] = str (boost::format ("(%s,%s,%s)") - % sqlEscape (sn.strValidator) - % sn.iScore - % strSeen); - - usUNL.insert (sn.strValidator); - } - - *db << str (boost::format ("REPLACE INTO TrustedNodes (PublicKey,Score,Seen) VALUES %s;") - % strJoin (vstrValues.begin (), vstrValues.end (), ",")); - } - - { - ScopedUNLLockType sl (mUNLLock); - - // XXX Should limit to scores above a certain minimum and limit to a certain number. - mUNL.swap (usUNL); - } - - hash_map umValidators; - - if (!vsnNodes.empty ()) - { - // For every IpReferral add a score for the IP and PORT. - std::vector, - boost::optional>> columns; - selectBlobsIntoStrings ( - *db, - "SELECT Validator,COUNT(*) AS Count FROM " - "IpReferrals GROUP BY Validator;", - columns); - for(auto const& col : columns) - { - umValidators[get<0>(col).value_or("")] = get<1>(col).value_or(0); - - // WriteLog (lsTRACE, UniqueNodeList) << strValidator << ":" << db->getInt("Count"); - } - } - - // For each validator, get each referral and add its score to ip's score. - // map of pair :: score - hash_map, score> umScore; - - for (auto& vc : umValidators) - { - std::string strValidator = vc.first; - - auto itIndex = umPulicIdx.find (strValidator); - - if (itIndex != umPulicIdx.end ()) - { - int iSeed = vsnNodes[itIndex->second].iScore; - int iEntries = vc.second; - score iTotal = (iEntries + 1) * iEntries / 2; - score iBase = iSeed * iEntries / iTotal; - int iEntry = 0; - - std::vector, - boost::optional>> columns; - selectBlobsIntoStrings ( - *db, - str (boost::format ( - "SELECT IP,Port FROM IpReferrals WHERE " - "Validator=%s ORDER BY Entry;") % - sqlEscape (strValidator)), - columns); - for(auto const& col : columns) - { - score iPoints = iBase * (iEntries - iEntry) / iEntries; - int iPort; - - iPort = get<1>(col).value_or(0); - - std::pair< std::string, int> ep = std::make_pair (get<0>(col).value_or(""), iPort); - - auto itEp = umScore.find (ep); - - umScore[ep] = itEp == umScore.end () ? iPoints : itEp->second + iPoints; - iEntry++; - } - } - } - - tr.commit (); - } - - //-------------------------------------------------------------------------- - - // Start a timer to update scores. - // <-- bNow: true, to force scoring for debugging. - void scoreNext (bool bNow) - { - // WriteLog (lsTRACE, UniqueNodeList) << str(boost::format("scoreNext: mtpFetchUpdated=%s mtpScoreStart=%s mtpScoreUpdated=%s mtpScoreNext=%s") % mtpFetchUpdated % mtpScoreStart % mtpScoreUpdated % mtpScoreNext); - bool bCanScore = mtpScoreStart.is_not_a_date_time () // Not scoring. - && !mtpFetchUpdated.is_not_a_date_time (); // Something to score. - - bool bDirty = - (mtpScoreUpdated.is_not_a_date_time () || mtpScoreUpdated <= mtpFetchUpdated) // Not already scored. - && (mtpScoreNext.is_not_a_date_time () // Timer is not fine. - || mtpScoreNext < mtpFetchUpdated + boost::posix_time::seconds (SCORE_DELAY_SECONDS)); - - if (bCanScore && (bNow || bDirty)) - { - // Need to update or set timer. - double const secondsFromNow = bNow ? 0 : SCORE_DELAY_SECONDS; - mtpScoreNext = boost::posix_time::second_clock::universal_time () // Past now too. - + boost::posix_time::seconds (secondsFromNow); - - // WriteLog (lsTRACE, UniqueNodeList) << str(boost::format("scoreNext: @%s") % mtpScoreNext); - m_scoreTimer.setExpiration (secondsFromNow); - } - } - - //-------------------------------------------------------------------------- - - // Given a ripple.txt, process it. - // - // VFALCO TODO Can't we take a filename or stream instead of a string? - // - bool responseFetch (std::string const& strDomain, const boost::system::error_code& err, int iStatus, std::string const& strSiteFile) - { - bool bReject = !err && iStatus != 200; - - if (!bReject) - { - IniFileSections secSite = parseIniFile (strSiteFile, true); - bool bGood = !err; - - if (bGood) - { - WriteLog (lsTRACE, UniqueNodeList) << strDomain - << ": retrieved configuration"; - } - else - { - WriteLog (lsTRACE, UniqueNodeList) << strDomain - << ": unable to retrieve configuration: " - << err.message (); - } - - // - // Verify file domain - // - std::string strSite; - - if (bGood && !getSingleSection (secSite, SECTION_DOMAIN, strSite)) - { - bGood = false; - - WriteLog (lsTRACE, UniqueNodeList) << strDomain - << ": " << SECTION_DOMAIN - << "entry missing."; - } - - if (bGood && strSite != strDomain) - { - bGood = false; - - WriteLog (lsTRACE, UniqueNodeList) << strDomain - << ": " << SECTION_DOMAIN << " does not match " << strSite; - } - - // - // Process public key - // - std::string strNodePublicKey; - - if (bGood && !getSingleSection (secSite, SECTION_PUBLIC_KEY, strNodePublicKey)) - { - // Bad [validation_public_key] IniFileSections. - bGood = false; - - WriteLog (lsTRACE, UniqueNodeList) << strDomain - << ": " << SECTION_PUBLIC_KEY << " entry missing."; - } - - RippleAddress naNodePublic; - - if (bGood && !naNodePublic.setNodePublic (strNodePublicKey)) - { - // Bad public key. - bGood = false; - - WriteLog (lsTRACE, UniqueNodeList) << strDomain - << ": " << SECTION_PUBLIC_KEY << " is not a public key: " - << strNodePublicKey; - } - - if (bGood) - { - seedDomain sdCurrent; - - bool bFound = getSeedDomains (strDomain, sdCurrent); - assert (bFound); - (void) bFound; - - uint256 iSha256 = getSHA512Half (strSiteFile); - bool bChangedB = sdCurrent.iSha256 != iSha256; - - sdCurrent.strDomain = strDomain; - // XXX If the node public key is changing, delete old public key information? - // XXX Only if no other refs to keep it arround, other wise we have an attack vector. - sdCurrent.naPublicKey = naNodePublic; - - // WriteLog (lsTRACE, UniqueNodeList) << boost::format("sdCurrent.naPublicKey: '%s'") % sdCurrent.naPublicKey.humanNodePublic(); - - sdCurrent.tpFetch = boost::posix_time::second_clock::universal_time (); - sdCurrent.iSha256 = iSha256; - - setSeedDomains (sdCurrent, true); - - if (bChangedB) - { - WriteLog (lsTRACE, UniqueNodeList) << strDomain - << ": processing new " << node_file_name_ << "."; - processFile (strDomain, naNodePublic, secSite); - } - else - { - WriteLog (lsTRACE, UniqueNodeList) << strDomain - << ": no change in " << node_file_name_ << "."; - fetchFinish (); - } - } - else - { - // Failed: Update - - // XXX If we have public key, perhaps try look up in CAS? - fetchFinish (); - } - } - - return bReject; - } - - //-------------------------------------------------------------------------- - - // Try to process the next fetch of a ripple.txt. - void fetchNext () - { - bool bFull; - - { - ScopedFetchLockType sl (mFetchLock); - - bFull = (mFetchActive == NODE_FETCH_JOBS); - } - - if (!bFull) - { - // Determine next scan. - std::string strDomain; - boost::posix_time::ptime tpNext (boost::posix_time::min_date_time); - boost::posix_time::ptime tpNow (boost::posix_time::second_clock::universal_time ()); - - auto db = getApp().getWalletDB ().checkoutDb (); - - - { - soci::blob b(*db); - soci::indicator ind; - boost::optional nO; - *db << "SELECT Domain,Next FROM SeedDomains INDEXED BY SeedDomainNext ORDER BY Next LIMIT 1;", - soci::into(b, ind), - soci::into(nO); - if (nO) - { - int iNext (*nO); - - tpNext = ptFromSeconds (iNext); - - WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("fetchNext: iNext=%s tpNext=%s tpNow=%s") % iNext % tpNext % tpNow); - if (soci::i_ok == ind) - convert (b, strDomain); - else - strDomain.clear (); - } - } - - if (!strDomain.empty ()) - { - ScopedFetchLockType sl (mFetchLock); - - bFull = (mFetchActive == NODE_FETCH_JOBS); - - if (!bFull && tpNext <= tpNow) - { - mFetchActive++; - } - } - - if (strDomain.empty () || bFull) - { - WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("fetchNext: strDomain=%s bFull=%d") % strDomain % bFull); - } - else if (tpNext > tpNow) - { - WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("fetchNext: set timer : strDomain=%s") % strDomain); - // Fetch needs to happen in the future. Set a timer to wake us. - mtpFetchNext = tpNext; - - double seconds = (tpNext - tpNow).seconds (); - - // VFALCO check this. - if (seconds == 0) - seconds = 1; - - m_fetchTimer.setExpiration (seconds); - } - else - { - WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("fetchNext: fetch now: strDomain=%s tpNext=%s tpNow=%s") % strDomain % tpNext % tpNow); - // Fetch needs to happen now. - mtpFetchNext = boost::posix_time::ptime (boost::posix_time::not_a_date_time); - - seedDomain sdCurrent; - bool bFound = getSeedDomains (strDomain, sdCurrent); - assert (bFound); - (void) bFound; - - // Update time of next fetch and this scan attempt. - sdCurrent.tpScan = tpNow; - - // XXX Use a longer duration if we have lots of validators. - sdCurrent.tpNext = sdCurrent.tpScan + boost::posix_time::hours (7 * 24); - - setSeedDomains (sdCurrent, false); - - WriteLog (lsTRACE, UniqueNodeList) << strDomain - << " fetching " << node_file_name_ << "."; - - fetchProcess (strDomain); // Go get it. - - fetchNext (); // Look for more. - } - } - } - - //-------------------------------------------------------------------------- - - // Called when we need to update scores. - void fetchDirty () - { - // Note update. - mtpFetchUpdated = boost::posix_time::second_clock::universal_time (); - miscSave (); - - // Update scores. - scoreNext (false); - } - - - //-------------------------------------------------------------------------- - - void fetchFinish () - { - { - ScopedFetchLockType sl (mFetchLock); - mFetchActive--; - } - - fetchNext (); - } - - //-------------------------------------------------------------------------- - - // Get the ripple.txt and process it. - void fetchProcess (std::string strDomain) - { - WriteLog (lsTRACE, UniqueNodeList) << strDomain - << ": fetching " << node_file_name_ << "."; - - std::deque deqSites; - - // Order searching from most specifically for purpose to generic. - // This order allows the client to take the most burden rather than the servers. - deqSites.push_back (systemName () + strDomain); - deqSites.push_back ("www." + strDomain); - deqSites.push_back (strDomain); - - HTTPClient::get ( - true, - getApp().getIOService (), - deqSites, - 443, - node_file_path_, - NODE_FILE_BYTES_MAX, - boost::posix_time::seconds (NODE_FETCH_SECONDS), - std::bind (&UniqueNodeListImp::responseFetch, this, strDomain, - std::placeholders::_1, std::placeholders::_2, - std::placeholders::_3)); - } - - // Process IniFileSections [validators_url]. - void getValidatorsUrl (RippleAddress const& naNodePublic, - IniFileSections secSite) - { - std::string strValidatorsUrl; - std::string strScheme; - std::string strDomain; - int iPort; - std::string strPath; - - if (getSingleSection (secSite, SECTION_VALIDATORS_URL, strValidatorsUrl) - && !strValidatorsUrl.empty () - && parseUrl (strValidatorsUrl, strScheme, strDomain, iPort, strPath) - && -1 == iPort - && strScheme == "https") - { - HTTPClient::get ( - true, - getApp().getIOService (), - strDomain, - 443, - strPath, - NODE_FILE_BYTES_MAX, - boost::posix_time::seconds (NODE_FETCH_SECONDS), - std::bind (&UniqueNodeListImp::responseValidators, this, - strValidatorsUrl, naNodePublic, secSite, strDomain, - std::placeholders::_1, std::placeholders::_2, - std::placeholders::_3)); - } - else - { - getIpsUrl (naNodePublic, secSite); - } - } - - //-------------------------------------------------------------------------- - - // Process IniFileSections [ips_url]. - // If we have a IniFileSections with a single entry, fetch the url and process it. - void getIpsUrl (RippleAddress const& naNodePublic, IniFileSections secSite) - { - std::string strIpsUrl; - std::string strScheme; - std::string strDomain; - int iPort; - std::string strPath; - - if (getSingleSection (secSite, SECTION_IPS_URL, strIpsUrl) - && !strIpsUrl.empty () - && parseUrl (strIpsUrl, strScheme, strDomain, iPort, strPath) - && -1 == iPort - && strScheme == "https") - { - HTTPClient::get ( - true, - getApp().getIOService (), - strDomain, - 443, - strPath, - NODE_FILE_BYTES_MAX, - boost::posix_time::seconds (NODE_FETCH_SECONDS), - std::bind (&UniqueNodeListImp::responseIps, this, strDomain, - naNodePublic, std::placeholders::_1, - std::placeholders::_2, std::placeholders::_3)); - } - else - { - fetchFinish (); - } - } - - - //-------------------------------------------------------------------------- - - // Given a IniFileSections with IPs, parse and persist it for a validator. - bool responseIps (std::string const& strSite, RippleAddress const& naNodePublic, const boost::system::error_code& err, int iStatus, std::string const& strIpsFile) - { - bool bReject = !err && iStatus != 200; - - if (!bReject) - { - if (!err) - { - IniFileSections secFile = parseIniFile (strIpsFile, true); - - processIps (strSite, naNodePublic, getIniFileSection (secFile, SECTION_IPS)); - } - - fetchFinish (); - } - - return bReject; - } - - // After fetching a ripple.txt from a web site, given a IniFileSections with validators, parse and persist it. - bool responseValidators (std::string const& strValidatorsUrl, RippleAddress const& naNodePublic, IniFileSections secSite, std::string const& strSite, const boost::system::error_code& err, int iStatus, std::string const& strValidatorsFile) - { - bool bReject = !err && iStatus != 200; - - if (!bReject) - { - if (!err) - { - IniFileSections secFile = parseIniFile (strValidatorsFile, true); - - processValidators (strSite, strValidatorsUrl, naNodePublic, vsValidator, getIniFileSection (secFile, SECTION_VALIDATORS)); - } - - getIpsUrl (naNodePublic, secSite); - } - - return bReject; - } - - - //-------------------------------------------------------------------------- - - // Persist the IPs refered to by a Validator. - // --> strSite: source of the IPs (for debugging) - // --> naNodePublic: public key of the validating node. - void processIps (std::string const& strSite, RippleAddress const& naNodePublic, IniFileSections::mapped_type* pmtVecStrIps) - { - std::string strEscNodePublic = sqlEscape (naNodePublic.humanNodePublic ()); - - WriteLog (lsDEBUG, UniqueNodeList) - << str (boost::format ("Validator: '%s' processing %d ips.") - % strSite % ( pmtVecStrIps ? pmtVecStrIps->size () : 0)); - - // Remove all current Validator's entries in IpReferrals - { - auto db = getApp().getWalletDB ().checkoutDb (); - *db << str (boost::format ("DELETE FROM IpReferrals WHERE Validator=%s;") % strEscNodePublic); - } - - // Add new referral entries. - if (pmtVecStrIps && !pmtVecStrIps->empty ()) - { - std::vector vstrValues; - - vstrValues.resize (std::min ((int) pmtVecStrIps->size (), REFERRAL_IPS_MAX)); - - int iValues = 0; - for (auto const& strReferral : *pmtVecStrIps) - { - if (iValues == REFERRAL_VALIDATORS_MAX) - break; - - std::string strIP; - int iPort; - bool bValid = parseIpPort (strReferral, strIP, iPort); - - // XXX Filter out private network ips. - // XXX http://en.wikipedia.org/wiki/Private_network - - if (bValid) - { - vstrValues[iValues] = str (boost::format ("(%s,%d,%s,%d)") - % strEscNodePublic % iValues % sqlEscape (strIP) % iPort); - iValues++; - } - else - { - WriteLog (lsTRACE, UniqueNodeList) - << str (boost::format ("Validator: '%s' [" SECTION_IPS "]: rejecting '%s'") - % strSite % strReferral); - } - } - - if (iValues) - { - vstrValues.resize (iValues); - - auto db = getApp().getWalletDB ().checkoutDb (); - *db << str (boost::format ("INSERT INTO IpReferrals (Validator,Entry,IP,Port) VALUES %s;") - % strJoin (vstrValues.begin (), vstrValues.end (), ",")); - // XXX Check result. - } - } - - fetchDirty (); - } - - //-------------------------------------------------------------------------- - - // Persist ValidatorReferrals. - // --> strSite: source site for display - // --> strValidatorsSrc: source details for display - // --> naNodePublic: remote source public key - not valid for local - // --> vsWhy: reason for adding validator to SeedDomains or SeedNodes. - int processValidators (std::string const& strSite, std::string const& strValidatorsSrc, RippleAddress const& naNodePublic, ValidatorSource vsWhy, IniFileSections::mapped_type* pmtVecStrValidators) - { - std::string strNodePublic = naNodePublic.isValid () ? naNodePublic.humanNodePublic () : strValidatorsSrc; - int iValues = 0; - - WriteLog (lsTRACE, UniqueNodeList) - << str (boost::format ("Validator: '%s' : '%s' : processing %d validators.") - % strSite - % strValidatorsSrc - % ( pmtVecStrValidators ? pmtVecStrValidators->size () : 0)); - - // Remove all current Validator's entries in ValidatorReferrals - { - auto db = getApp().getWalletDB ().checkoutDb (); - - *db << str (boost::format ("DELETE FROM ValidatorReferrals WHERE Validator='%s';") % strNodePublic); - // XXX Check result. - } - - // Add new referral entries. - if (pmtVecStrValidators && pmtVecStrValidators->size ()) - { - std::vector vstrValues; - - vstrValues.reserve (std::min ((int) pmtVecStrValidators->size (), REFERRAL_VALIDATORS_MAX)); - - for (auto const& strReferral : *pmtVecStrValidators) - { - if (iValues == REFERRAL_VALIDATORS_MAX) - break; - - boost::smatch smMatch; - - // domain comment? - // public_key comment? - static boost::regex reReferral ("\\`\\s*(\\S+)(?:\\s+(.+))?\\s*\\'"); - - if (!boost::regex_match (strReferral, smMatch, reReferral)) - { - WriteLog (lsWARNING, UniqueNodeList) << str (boost::format ("Bad validator: syntax error: %s: %s") % strSite % strReferral); - } - else - { - std::string strRefered = smMatch[1]; - std::string strComment = smMatch[2]; - RippleAddress naValidator; - - if (naValidator.setSeedGeneric (strRefered)) - { - WriteLog (lsWARNING, UniqueNodeList) << str (boost::format ("Bad validator: domain or public key required: %s %s") % strRefered % strComment); - } - else if (naValidator.setNodePublic (strRefered)) - { - // A public key. - // XXX Schedule for CAS lookup. - nodeAddPublic (naValidator, vsWhy, strComment); - - WriteLog (lsINFO, UniqueNodeList) << str (boost::format ("Node Public: %s %s") % strRefered % strComment); - - if (naNodePublic.isValid ()) - vstrValues.push_back (str (boost::format ("('%s',%d,'%s')") % strNodePublic % iValues % naValidator.humanNodePublic ())); - - iValues++; - } - else - { - // A domain: need to look it up. - nodeAddDomain (strRefered, vsWhy, strComment); - - WriteLog (lsINFO, UniqueNodeList) << str (boost::format ("Node Domain: %s %s") % strRefered % strComment); - - if (naNodePublic.isValid ()) - vstrValues.push_back (str (boost::format ("('%s',%d,%s)") % strNodePublic % iValues % sqlEscape (strRefered))); - - iValues++; - } - } - } - - if (!vstrValues.empty ()) - { - std::string strSql = str (boost::format ("INSERT INTO ValidatorReferrals (Validator,Entry,Referral) VALUES %s;") - % strJoin (vstrValues.begin (), vstrValues.end (), ",")); - - auto db = getApp().getWalletDB ().checkoutDb (); - - *db << strSql; - // XXX Check result. - } - } - - fetchDirty (); - - return iValues; - } - - //-------------------------------------------------------------------------- - - // Process a ripple.txt. - void processFile (std::string const& strDomain, RippleAddress const& naNodePublic, IniFileSections secSite) - { - // - // Process Validators - // - processValidators (strDomain, node_file_name_, naNodePublic, - vsReferral, getIniFileSection (secSite, SECTION_VALIDATORS)); - - // - // Process ips - // - processIps (strDomain, naNodePublic, getIniFileSection (secSite, SECTION_IPS)); - - // - // Process currencies - // - IniFileSections::mapped_type* pvCurrencies; - - if ((pvCurrencies = getIniFileSection (secSite, SECTION_CURRENCIES)) && pvCurrencies->size ()) - { - // XXX Process currencies. - WriteLog (lsWARNING, UniqueNodeList) << "Ignoring currencies: not implemented."; - } - - getValidatorsUrl (naNodePublic, secSite); - } - - //-------------------------------------------------------------------------- - - // Retrieve a SeedDomain from DB. - bool getSeedDomains (std::string const& strDomain, seedDomain& dstSeedDomain) - { - bool bResult = false; - - std::string strSql = boost::str ( - boost::format ( - "SELECT Domain, PublicKey, Source, Next, Scan, Fetch, Sha256, " - "Comment FROM SeedDomains WHERE Domain=%s;") % - sqlEscape (strDomain)); - - auto db = getApp().getWalletDB ().checkoutDb (); - - // Iterate through the result rows with a fectch b/c putting a - // column of type DATETIME into a boost::tuple can throw when the - // datetime column is invalid (even if the value as int is valid). - soci::blob domainBlob(*db); - soci::indicator di; - boost::optional strPublicKey; - soci:: blob sourceBlob(*db); - soci::indicator si; - std::string strSource; - boost::optional iNext; - boost::optional iScan; - boost::optional iFetch; - boost::optional strSha256; - soci::blob commentBlob(*db); - soci::indicator ci; - boost::optional strComment; - - soci::statement st = (db->prepare << strSql, - soci::into (domainBlob, di), - soci::into (strPublicKey), - soci::into (sourceBlob, si), - soci::into (iNext), - soci::into (iScan), - soci::into (iFetch), - soci::into (strSha256), - soci::into (commentBlob, ci)); - - st.execute (); - while (st.fetch ()) - { - bResult = true; - - if (soci::i_ok == di) - convert (domainBlob, dstSeedDomain.strDomain); - - if (strPublicKey && !strPublicKey->empty ()) - dstSeedDomain.naPublicKey.setNodePublic (*strPublicKey); - else - dstSeedDomain.naPublicKey.clear (); - - if (soci::i_ok == si) - { - convert (sourceBlob, strSource); - dstSeedDomain.vsSource = static_cast (strSource[0]); - } - else - { - assert (0); - } - - dstSeedDomain.tpNext = ptFromSeconds (iNext.value_or (0)); - dstSeedDomain.tpScan = ptFromSeconds (iScan.value_or (0)); - dstSeedDomain.tpFetch = ptFromSeconds (iFetch.value_or (0)); - - if (strSha256 && !strSha256->empty ()) - dstSeedDomain.iSha256.SetHex (*strSha256); - else - dstSeedDomain.iSha256.zero (); - - if (soci::i_ok == ci) - convert (commentBlob, dstSeedDomain.strComment); - else - dstSeedDomain.strComment.clear (); - } - - return bResult; - } - - //-------------------------------------------------------------------------- - - // Persist a SeedDomain. - void setSeedDomains (const seedDomain& sdSource, bool bNext) - { - int iNext = iToSeconds (sdSource.tpNext); - int iScan = iToSeconds (sdSource.tpScan); - int iFetch = iToSeconds (sdSource.tpFetch); - - // WriteLog (lsTRACE) << str(boost::format("setSeedDomains: iNext=%s tpNext=%s") % iNext % sdSource.tpNext); - - std::string strSql = boost::str (boost::format ("REPLACE INTO SeedDomains (Domain,PublicKey,Source,Next,Scan,Fetch,Sha256,Comment) VALUES (%s, %s, %s, %d, %d, %d, '%s', %s);") - % sqlEscape (sdSource.strDomain) - % (sdSource.naPublicKey.isValid () ? sqlEscape (sdSource.naPublicKey.humanNodePublic ()) : "NULL") - % sqlEscape (std::string (1, static_cast (sdSource.vsSource))) - % iNext - % iScan - % iFetch - % to_string (sdSource.iSha256) - % sqlEscape (sdSource.strComment) - ); - - auto db = getApp().getWalletDB ().checkoutDb (); - - try - { - *db << strSql; - } - catch (soci::soci_error& e) - { - // XXX Check result. - WriteLog (lsWARNING, UniqueNodeList) << "setSeedDomains: failed. Error: " << e.what(); - } - - if (bNext && (mtpFetchNext.is_not_a_date_time () || mtpFetchNext > sdSource.tpNext)) - { - // Schedule earlier wake up. - fetchNext (); - } - } - - - //-------------------------------------------------------------------------- - - // Retrieve a SeedNode from DB. - bool getSeedNodes (RippleAddress const& naNodePublic, seedNode& dstSeedNode) - { - std::string strSql = - str (boost::format ( - "SELECT PublicKey, Source, Next, Scan, Fetch, Sha256, " - "Comment FROM SeedNodes WHERE PublicKey='%s';") % - naNodePublic.humanNodePublic ()); - - auto db = getApp().getWalletDB ().checkoutDb (); - - std::string strPublicKey; - std::string strSource; - soci::blob sourceBlob(*db); - soci::indicator si; - boost::optional iNext; - boost::optional iScan; - boost::optional iFetch; - boost::optional strSha256; - soci::blob commentBlob(*db); - soci::indicator ci; - - *db << strSql, - soci::into (strPublicKey), - soci::into (sourceBlob, si), - soci::into (iNext), - soci::into (iScan), - soci::into (iFetch), - soci::into (strSha256), - soci::into (commentBlob, ci); - - if (!db->got_data ()) - return false; - - if (!strPublicKey.empty ()) - dstSeedNode.naPublicKey.setNodePublic (strPublicKey); - else - dstSeedNode.naPublicKey.clear (); - - if (soci::i_ok == si) - { - convert (sourceBlob, strSource); - dstSeedNode.vsSource = static_cast (strSource[0]); - } - else - assert (0); - - dstSeedNode.tpNext = ptFromSeconds (iNext.value_or(0)); - dstSeedNode.tpScan = ptFromSeconds (iScan.value_or(0)); - dstSeedNode.tpFetch = ptFromSeconds (iFetch.value_or(0)); - - if (strSha256 && !strSha256->empty ()) - dstSeedNode.iSha256.SetHex (*strSha256); - else - dstSeedNode.iSha256.zero (); - - if (soci::i_ok == ci) - convert (commentBlob, dstSeedNode.strComment); - else - dstSeedNode.strComment.clear (); - - return true; - } - - //-------------------------------------------------------------------------- - - // Persist a SeedNode. - // <-- bNext: true, to do fetching if needed. - void setSeedNodes (const seedNode& snSource, bool bNext) - { - int iNext = iToSeconds (snSource.tpNext); - int iScan = iToSeconds (snSource.tpScan); - int iFetch = iToSeconds (snSource.tpFetch); - - // WriteLog (lsTRACE) << str(boost::format("setSeedNodes: iNext=%s tpNext=%s") % iNext % sdSource.tpNext); - - assert (snSource.naPublicKey.isValid ()); - - std::string strSql = str (boost::format ("REPLACE INTO SeedNodes (PublicKey,Source,Next,Scan,Fetch,Sha256,Comment) VALUES ('%s', '%c', %d, %d, %d, '%s', %s);") - % snSource.naPublicKey.humanNodePublic () - % static_cast (snSource.vsSource) - % iNext - % iScan - % iFetch - % to_string (snSource.iSha256) - % sqlEscape (snSource.strComment) - ); - - { - auto db = getApp().getWalletDB ().checkoutDb (); - - try - { - *db << strSql; - } - catch(soci::soci_error& e) - { - WriteLog (lsTRACE, UniqueNodeList) << "setSeedNodes: failed. Error: " << e.what (); - } - } - - #if 0 - - // YYY When we have a cas schedule lookups similar to this. - if (bNext && (mtpFetchNext.is_not_a_date_time () || mtpFetchNext > snSource.tpNext)) - { - // Schedule earlier wake up. - fetchNext (); - } - - #else - fetchDirty (); - #endif - } - - //-------------------------------------------------------------------------- - - bool validatorsResponse (const boost::system::error_code& err, int iStatus, std::string strResponse) - { - bool bReject = !err && iStatus != 200; - - if (!bReject) - { - WriteLog (lsTRACE, UniqueNodeList) << - "Fetch '" << - Config::Helpers::getValidatorsFileName () << - "' complete."; - - if (!err) - { - nodeProcess ("network", strResponse, getConfig ().VALIDATORS_SITE); - } - else - { - WriteLog (lsWARNING, UniqueNodeList) << "Error: " << err.message (); - } - } - - return bReject; - } - - //-------------------------------------------------------------------------- - - // Process a validators.txt. - // --> strSite: source of validators - // --> strValidators: contents of a validators.txt - // - // VFALCO TODO Can't we name this processValidatorList? - // - void nodeProcess (std::string const& strSite, std::string const& strValidators, std::string const& strSource) - { - IniFileSections secValidators = parseIniFile (strValidators, true); - - IniFileSections::mapped_type* pmtEntries = getIniFileSection (secValidators, SECTION_VALIDATORS); - - if (pmtEntries) - { - RippleAddress naInvalid; // Don't want a referrer on added entries. - - // YYY Unspecified might be bootstrap or rpc command - processValidators (strSite, strSource, naInvalid, vsValidator, pmtEntries); - } - else - { - WriteLog (lsWARNING, UniqueNodeList) << boost::str (boost::format ("'%s' missing [" SECTION_VALIDATORS "].") - % getConfig ().VALIDATORS_BASE); - } - } -private: - using FetchLockType = RippleMutex; - using ScopedFetchLockType = std::lock_guard ; + typedef RippleMutex FetchLockType; + typedef std::lock_guard ScopedFetchLockType; FetchLockType mFetchLock; - using UNLLockType = RippleRecursiveMutex; - using ScopedUNLLockType = std::lock_guard ; + typedef RippleRecursiveMutex UNLLockType; + typedef std::lock_guard ScopedUNLLockType; UNLLockType mUNLLock; // VFALCO TODO Replace ptime with beast::Time @@ -2195,6 +228,160 @@ private: std::string node_file_name_; std::string node_file_path_; + +public: + explicit UniqueNodeListImp (Stoppable& parent); + + void onStop(); + + void doScore(); + + void doFetch(); + + void onDeadlineTimer (beast::DeadlineTimer& timer); + + // This is called when the application is started. + // Get update times and start fetching and scoring as needed. + void start(); + + // Add a trusted node. Called by RPC or other source. + void nodeAddPublic (RippleAddress const& naNodePublic, ValidatorSource vsWhy, std::string const& strComment); + + // Queue a domain for a single attempt fetch a ripple.txt. + // --> strComment: only used on vsManual + // YYY As a lot of these may happen at once, would be nice to wrap multiple calls in a transaction. + void nodeAddDomain (std::string strDomain, ValidatorSource vsWhy, std::string const& strComment); + + void nodeRemovePublic (RippleAddress const& naNodePublic); + + void nodeRemoveDomain (std::string strDomain); + + void nodeReset(); + + // For debugging, schedule forced scoring. + void nodeScore(); + + bool nodeInUNL (RippleAddress const& naNodePublic); + + bool nodeInCluster (RippleAddress const& naNodePublic); + bool nodeInCluster (RippleAddress const& naNodePublic, std::string& name); + + bool nodeUpdate (RippleAddress const& naNodePublic, ClusterNodeStatus const& cnsStatus); + + std::map getClusterStatus(); + + std::uint32_t getClusterFee(); + + void addClusterStatus (Json::Value& obj); + + void nodeBootstrap(); + + bool nodeLoad (boost::filesystem::path pConfig); + + void nodeNetwork(); + + Json::Value getUnlJson(); + + // For each kind of source, have a starting number of points to be distributed. + int iSourceScore (ValidatorSource vsWhy); + + //-------------------------------------------------------------------------- +private: + // Load information about when we last updated. + bool miscLoad(); + + // Persist update information. + bool miscSave(); + + void trustedLoad(); + + // For a round of scoring we destribute points from a node to nodes it refers to. + // Returns true, iff scores were distributed. + // + bool scoreRound (std::vector& vsnNodes); + + // From SeedDomains and ValidatorReferrals compute scores and update TrustedNodes. + // + // VFALCO TODO Shrink this function, break it up + // + void scoreCompute(); + + // Start a timer to update scores. + // <-- bNow: true, to force scoring for debugging. + void scoreNext (bool bNow); + + // Given a ripple.txt, process it. + // + // VFALCO TODO Can't we take a filename or stream instead of a string? + // + bool responseFetch (std::string const& strDomain, const boost::system::error_code& err, int iStatus, std::string const& strSiteFile); + + // Try to process the next fetch of a ripple.txt. + void fetchNext(); + + // Called when we need to update scores. + void fetchDirty(); + + + void fetchFinish(); + + // Get the ripple.txt and process it. + void fetchProcess (std::string strDomain); + + // Process IniFileSections [validators_url]. + void getValidatorsUrl (RippleAddress const& naNodePublic, + IniFileSections secSite); + + // Process IniFileSections [ips_url]. + // If we have a IniFileSections with a single entry, fetch the url and process it. + void getIpsUrl (RippleAddress const& naNodePublic, IniFileSections secSite); + + + // Given a IniFileSections with IPs, parse and persist it for a validator. + bool responseIps (std::string const& strSite, RippleAddress const& naNodePublic, const boost::system::error_code& err, int iStatus, std::string const& strIpsFile);; + + // After fetching a ripple.txt from a web site, given a IniFileSections with validators, parse and persist it. + bool responseValidators (std::string const& strValidatorsUrl, RippleAddress const& naNodePublic, IniFileSections secSite, std::string const& strSite, const boost::system::error_code& err, int iStatus, std::string const& strValidatorsFile); + + + // Persist the IPs refered to by a Validator. + // --> strSite: source of the IPs (for debugging) + // --> naNodePublic: public key of the validating node. + void processIps (std::string const& strSite, RippleAddress const& naNodePublic, IniFileSections::mapped_type* pmtVecStrIps); + + // Persist ValidatorReferrals. + // --> strSite: source site for display + // --> strValidatorsSrc: source details for display + // --> naNodePublic: remote source public key - not valid for local + // --> vsWhy: reason for adding validator to SeedDomains or SeedNodes. + int processValidators (std::string const& strSite, std::string const& strValidatorsSrc, RippleAddress const& naNodePublic, ValidatorSource vsWhy, IniFileSections::mapped_type* pmtVecStrValidators); + + // Process a ripple.txt. + void processFile (std::string const& strDomain, RippleAddress const& naNodePublic, IniFileSections secSite); + + // Retrieve a SeedDomain from DB. + bool getSeedDomains (std::string const& strDomain, seedDomain& dstSeedDomain); + + // Persist a SeedDomain. + void setSeedDomains (const seedDomain& sdSource, bool bNext); + + + // Retrieve a SeedNode from DB. + bool getSeedNodes (RippleAddress const& naNodePublic, seedNode& dstSeedNode); + + // Persist a SeedNode. + // <-- bNext: true, to do fetching if needed. + void setSeedNodes (const seedNode& snSource, bool bNext); + + bool validatorsResponse (const boost::system::error_code& err, int iStatus, std::string strResponse); + + // Process a validators.txt. + // --> strSite: source of validators + // --> strValidators: contents of a validators.txt + // + // VFALCO TODO Can't we name this processValidatorList? + // + void nodeProcess (std::string const& strSite, std::string const& strValidators, std::string const& strSource); }; //------------------------------------------------------------------------------ @@ -2206,6 +393,1966 @@ UniqueNodeList::UniqueNodeList (Stoppable& parent) //------------------------------------------------------------------------------ +UniqueNodeListImp::UniqueNodeListImp (Stoppable& parent) + : UniqueNodeList (parent) + , m_scoreTimer (this) + , mFetchActive (0) + , m_fetchTimer (this) +{ + node_file_name_ = std::string (systemName ()) + ".txt"; + node_file_path_ = "/" + node_file_name_; +} + +void UniqueNodeListImp::onStop() +{ + m_fetchTimer.cancel (); + m_scoreTimer.cancel (); + + stopped (); +} + +void UniqueNodeListImp::doScore() +{ + mtpScoreNext = boost::posix_time::ptime (boost::posix_time::not_a_date_time); // Timer not set. + mtpScoreStart = boost::posix_time::second_clock::universal_time (); // Scoring. + + WriteLog (lsTRACE, UniqueNodeList) << "Scoring: Start"; + + scoreCompute (); + + WriteLog (lsTRACE, UniqueNodeList) << "Scoring: End"; + + // Save update time. + mtpScoreUpdated = mtpScoreStart; + miscSave (); + + mtpScoreStart = boost::posix_time::ptime (boost::posix_time::not_a_date_time); // Not scoring. + + // Score again if needed. + scoreNext (false); +} + +void UniqueNodeListImp::doFetch() +{ + // Time to check for another fetch. + WriteLog (lsTRACE, UniqueNodeList) << "fetchTimerHandler"; + fetchNext (); +} + +void UniqueNodeListImp::onDeadlineTimer (beast::DeadlineTimer& timer) +{ + if (timer == m_scoreTimer) + { + getApp().getJobQueue ().addJob (jtUNL, "UNL.score", + std::bind (&UniqueNodeListImp::doScore, this)); + } + else if (timer == m_fetchTimer) + { + getApp().getJobQueue ().addJob (jtUNL, "UNL.fetch", + std::bind (&UniqueNodeListImp::doFetch, this)); + } +} + +// This is called when the application is started. +// Get update times and start fetching and scoring as needed. +void UniqueNodeListImp::start() +{ + miscLoad (); + + WriteLog (lsDEBUG, UniqueNodeList) << "Validator fetch updated: " << mtpFetchUpdated; + WriteLog (lsDEBUG, UniqueNodeList) << "Validator score updated: " << mtpScoreUpdated; + + fetchNext (); // Start fetching. + scoreNext (false); // Start scoring. +} + +//-------------------------------------------------------------------------- + +// Add a trusted node. Called by RPC or other source. +void UniqueNodeListImp::nodeAddPublic (RippleAddress const& naNodePublic, ValidatorSource vsWhy, std::string const& strComment) +{ + seedNode snCurrent; + + bool bFound = getSeedNodes (naNodePublic, snCurrent); + bool bChanged = false; + + if (!bFound) + { + snCurrent.naPublicKey = naNodePublic; + snCurrent.tpNext = boost::posix_time::second_clock::universal_time (); + } + + // Promote source, if needed. + if (!bFound /*|| iSourceScore (vsWhy) >= iSourceScore (snCurrent.vsSource)*/) + { + snCurrent.vsSource = vsWhy; + snCurrent.strComment = strComment; + bChanged = true; + } + + if (vsManual == vsWhy) + { + // A manual add forces immediate scan. + snCurrent.tpNext = boost::posix_time::second_clock::universal_time (); + bChanged = true; + } + + if (bChanged) + setSeedNodes (snCurrent, true); +} + +//-------------------------------------------------------------------------- + +// Queue a domain for a single attempt fetch a ripple.txt. +// --> strComment: only used on vsManual +// YYY As a lot of these may happen at once, would be nice to wrap multiple calls in a transaction. +void UniqueNodeListImp::nodeAddDomain (std::string strDomain, ValidatorSource vsWhy, std::string const& strComment) +{ + boost::trim (strDomain); + boost::to_lower (strDomain); + + // YYY Would be best to verify strDomain is a valid domain. + // WriteLog (lsTRACE) << str(boost::format("nodeAddDomain: '%s' %c '%s'") + // % strDomain + // % vsWhy + // % strComment); + + seedDomain sdCurrent; + + bool bFound = getSeedDomains (strDomain, sdCurrent); + bool bChanged = false; + + if (!bFound) + { + sdCurrent.strDomain = strDomain; + sdCurrent.tpNext = boost::posix_time::second_clock::universal_time (); + } + + // Promote source, if needed. + if (!bFound || iSourceScore (vsWhy) >= iSourceScore (sdCurrent.vsSource)) + { + sdCurrent.vsSource = vsWhy; + sdCurrent.strComment = strComment; + bChanged = true; + } + + if (vsManual == vsWhy) + { + // A manual add forces immediate scan. + sdCurrent.tpNext = boost::posix_time::second_clock::universal_time (); + bChanged = true; + } + + if (bChanged) + setSeedDomains (sdCurrent, true); +} + +//-------------------------------------------------------------------------- + +void UniqueNodeListImp::nodeRemovePublic (RippleAddress const& naNodePublic) +{ + { + auto db = getApp().getWalletDB ().checkoutDb (); + + *db << str ( + boost::format ("DELETE FROM SeedNodes WHERE PublicKey=%s;") % + sqlEscape (naNodePublic.humanNodePublic ())); + *db << str ( + boost::format ("DELETE FROM TrustedNodes WHERE PublicKey=%s;") % + sqlEscape (naNodePublic.humanNodePublic ())); + } + + // YYY Only dirty on successful delete. + fetchDirty (); + + ScopedUNLLockType sl (mUNLLock); + mUNL.erase (naNodePublic.humanNodePublic ()); +} + +//-------------------------------------------------------------------------- + +void UniqueNodeListImp::nodeRemoveDomain (std::string strDomain) +{ + boost::trim (strDomain); + boost::to_lower (strDomain); + + { + auto db = getApp().getWalletDB ().checkoutDb (); + + *db << str (boost::format ("DELETE FROM SeedDomains WHERE Domain=%s;") % sqlEscape (strDomain)); + } + + // YYY Only dirty on successful delete. + fetchDirty (); +} + +//-------------------------------------------------------------------------- + +void UniqueNodeListImp::nodeReset() +{ + { + auto db = getApp().getWalletDB ().checkoutDb (); + + *db << "DELETE FROM SeedDomains;"; + *db << "DELETE FROM SeedNodes;"; + } + + fetchDirty (); +} + +//-------------------------------------------------------------------------- + +// For debugging, schedule forced scoring. +void UniqueNodeListImp::nodeScore() +{ + scoreNext (true); +} + +//-------------------------------------------------------------------------- + +bool UniqueNodeListImp::nodeInUNL (RippleAddress const& naNodePublic) +{ + ScopedUNLLockType sl (mUNLLock); + + return mUNL.end () != mUNL.find (naNodePublic.humanNodePublic ()); +} + +//-------------------------------------------------------------------------- + +bool UniqueNodeListImp::nodeInCluster (RippleAddress const& naNodePublic) +{ + ScopedUNLLockType sl (mUNLLock); + return m_clusterNodes.end () != m_clusterNodes.find (naNodePublic); +} + +//-------------------------------------------------------------------------- + +bool UniqueNodeListImp::nodeInCluster (RippleAddress const& naNodePublic, std::string& name) +{ + ScopedUNLLockType sl (mUNLLock); + std::map::iterator it = m_clusterNodes.find (naNodePublic); + + if (it == m_clusterNodes.end ()) + return false; + + name = it->second.getName(); + return true; +} + +//-------------------------------------------------------------------------- + +bool UniqueNodeListImp::nodeUpdate (RippleAddress const& naNodePublic, ClusterNodeStatus const& cnsStatus) +{ + ScopedUNLLockType sl (mUNLLock); + return m_clusterNodes[naNodePublic].update(cnsStatus); +} + +//-------------------------------------------------------------------------- + +std::map +UniqueNodeListImp::getClusterStatus() +{ + std::map ret; + { + ScopedUNLLockType sl (mUNLLock); + ret = m_clusterNodes; + } + return ret; +} + +//-------------------------------------------------------------------------- + +std::uint32_t UniqueNodeListImp::getClusterFee() +{ + int thresh = getApp().getOPs().getNetworkTimeNC() - 90; + + std::vector fees; + { + ScopedUNLLockType sl (mUNLLock); + { + for (std::map::iterator it = m_clusterNodes.begin(), + end = m_clusterNodes.end(); it != end; ++it) + { + if (it->second.getReportTime() >= thresh) + fees.push_back(it->second.getLoadFee()); + } + } + } + + if (fees.empty()) + return 0; + std::sort (fees.begin(), fees.end()); + return fees[fees.size() / 2]; +} + +//-------------------------------------------------------------------------- + +void UniqueNodeListImp::addClusterStatus (Json::Value& obj) +{ + ScopedUNLLockType sl (mUNLLock); + if (m_clusterNodes.size() > 1) // nodes other than us + { + int now = getApp().getOPs().getNetworkTimeNC(); + std::uint32_t ref = getApp().getFeeTrack().getLoadBase(); + Json::Value& nodes = (obj[jss::cluster] = Json::objectValue); + + for (std::map::iterator it = m_clusterNodes.begin(), + end = m_clusterNodes.end(); it != end; ++it) + { + if (it->first != getApp().getLocalCredentials().getNodePublic()) + { + Json::Value& node = nodes[it->first.humanNodePublic()]; + + if (!it->second.getName().empty()) + node["tag"] = it->second.getName(); + + if ((it->second.getLoadFee() != ref) && (it->second.getLoadFee() != 0)) + node["fee"] = static_cast(it->second.getLoadFee()) / ref; + + if (it->second.getReportTime() != 0) + node["age"] = (it->second.getReportTime() >= now) ? 0 : (now - it->second.getReportTime()); + } + } + } +} + +//-------------------------------------------------------------------------- + +void UniqueNodeListImp::nodeBootstrap() +{ + int iDomains = 0; + int iNodes = 0; + +#if 0 + { + auto sl (getApp().getWalletDB ().lock ()); + auto db = getApp().getWalletDB ().getDB (); + + if (db->executeSQL (str (boost::format ("SELECT COUNT(*) AS Count FROM SeedDomains WHERE Source='%s' OR Source='%c';") % vsManual % vsValidator)) && db->startIterRows ()) + iDomains = db->getInt ("Count"); + + db->endIterRows (); + + if (db->executeSQL (str (boost::format ("SELECT COUNT(*) AS Count FROM SeedNodes WHERE Source='%s' OR Source='%c';") % vsManual % vsValidator)) && db->startIterRows ()) + iNodes = db->getInt ("Count"); + + db->endIterRows (); + } +#endif + + bool bLoaded = iDomains || iNodes; + + // Always merge in the file specified in the config. + if (!getConfig ().VALIDATORS_FILE.empty ()) + { + WriteLog (lsINFO, UniqueNodeList) << "Bootstrapping UNL: loading from unl_default."; + + bLoaded = nodeLoad (getConfig ().VALIDATORS_FILE); + } + + // If never loaded anything try the current directory. + if (!bLoaded && getConfig ().VALIDATORS_FILE.empty ()) + { + WriteLog (lsINFO, UniqueNodeList) << boost::str (boost::format ("Bootstrapping UNL: loading from '%s'.") + % getConfig ().VALIDATORS_BASE); + + bLoaded = nodeLoad (getConfig ().VALIDATORS_BASE); + } + + // Always load from rippled.cfg + if (!getConfig ().validators.empty ()) + { + RippleAddress naInvalid; // Don't want a referrer on added entries. + + WriteLog (lsINFO, UniqueNodeList) << boost::str (boost::format ("Bootstrapping UNL: loading from '%s'.") + % getConfig ().CONFIG_FILE); + + if (processValidators ("local", getConfig ().CONFIG_FILE.string (), naInvalid, vsConfig, &getConfig ().validators)) + bLoaded = true; + } + + if (!bLoaded) + { + WriteLog (lsINFO, UniqueNodeList) << boost::str (boost::format ("Bootstrapping UNL: loading from '%s'.") + % getConfig ().VALIDATORS_SITE); + + nodeNetwork (); + } +} + +//-------------------------------------------------------------------------- + +bool UniqueNodeListImp::nodeLoad (boost::filesystem::path pConfig) +{ + if (pConfig.empty ()) + { + WriteLog (lsINFO, UniqueNodeList) << Config::Helpers::getValidatorsFileName() << + " path not specified."; + + return false; + } + + if (!boost::filesystem::exists (pConfig)) + { + WriteLog (lsWARNING, UniqueNodeList) << Config::Helpers::getValidatorsFileName() << + " not found: " << pConfig; + + return false; + } + + if (!boost::filesystem::is_regular_file (pConfig)) + { + WriteLog (lsWARNING, UniqueNodeList) << Config::Helpers::getValidatorsFileName() << + " not regular file: " << pConfig; + + return false; + } + + std::ifstream ifsDefault (pConfig.native ().c_str (), std::ios::in); + + if (!ifsDefault) + { + WriteLog (lsFATAL, UniqueNodeList) << Config::Helpers::getValidatorsFileName() << + " failed to open: " << pConfig; + + return false; + } + + std::string strValidators; + + strValidators.assign ((std::istreambuf_iterator (ifsDefault)), + std::istreambuf_iterator ()); + + if (ifsDefault.bad ()) + { + WriteLog (lsFATAL, UniqueNodeList) << Config::Helpers::getValidatorsFileName() << + "Failed to read: " << pConfig; + + return false; + } + + nodeProcess ("local", strValidators, pConfig.string ()); + + WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("Processing: %s") % pConfig); + + return true; +} + +//-------------------------------------------------------------------------- + +void UniqueNodeListImp::nodeNetwork() +{ + if (!getConfig ().VALIDATORS_SITE.empty ()) + { + HTTPClient::get ( + true, + getApp().getIOService (), + getConfig ().VALIDATORS_SITE, + 443, + getConfig ().VALIDATORS_URI, + VALIDATORS_FILE_BYTES_MAX, + boost::posix_time::seconds (VALIDATORS_FETCH_SECONDS), + std::bind (&UniqueNodeListImp::validatorsResponse, this, + std::placeholders::_1, + std::placeholders::_2, + std::placeholders::_3)); + } +} + +//-------------------------------------------------------------------------- + +Json::Value UniqueNodeListImp::getUnlJson() +{ + + Json::Value ret (Json::arrayValue); + + auto db = getApp().getWalletDB ().checkoutDb (); + + + std::vector, 2>> columns; + selectBlobsIntoStrings(*db, + "SELECT PublicKey, Comment FROM TrustedNodes;", + columns); + for(auto const& strArray : columns) + { + Json::Value node (Json::objectValue); + + node["publicKey"] = strArray[0].value_or(""); + node["comment"] = strArray[1].value_or(""); + + ret.append (node); + } + + return ret; +} + +//-------------------------------------------------------------------------- + +// For each kind of source, have a starting number of points to be distributed. +int UniqueNodeListImp::iSourceScore (ValidatorSource vsWhy) +{ + int iScore = 0; + + switch (vsWhy) + { + case vsConfig: + iScore = 1500; + break; + + case vsInbound: + iScore = 0; + break; + + case vsManual: + iScore = 1500; + break; + + case vsReferral: + iScore = 0; + break; + + case vsTold: + iScore = 0; + break; + + case vsValidator: + iScore = 1000; + break; + + case vsWeb: + iScore = 200; + break; + + default: + throw std::runtime_error ("Internal error: bad ValidatorSource."); + } + + return iScore; +} + +// Load information about when we last updated. +bool UniqueNodeListImp::miscLoad() +{ + auto db = getApp().getWalletDB ().checkoutDb (); + + boost::optional suO, fuO; + + *db << "SELECT ScoreUpdated, FetchUpdated FROM Misc WHERE Magic=1;", + soci::into(suO), soci::into(fuO); + + if (!db->got_data() ) + return false; + + mtpFetchUpdated = ptFromSeconds (fuO.value_or(-1)); + mtpScoreUpdated = ptFromSeconds (suO.value_or(-1)); + + trustedLoad (); + + return true; +} + +// Persist update information. +bool UniqueNodeListImp::miscSave() +{ + auto db = getApp().getWalletDB ().checkoutDb (); + + *db << str (boost::format ("REPLACE INTO Misc (Magic,FetchUpdated,ScoreUpdated) VALUES (1,%d,%d);") + % iToSeconds (mtpFetchUpdated) + % iToSeconds (mtpScoreUpdated)); + + return true; +} + +//-------------------------------------------------------------------------- + +void UniqueNodeListImp::trustedLoad() +{ + boost::regex rNode ("\\`\\s*(\\S+)[\\s]*(.*)\\'"); + for (auto const& c : getConfig ().CLUSTER_NODES) + { + boost::smatch match; + + if (boost::regex_match (c, match, rNode)) + { + RippleAddress a = RippleAddress::createNodePublic (match[1]); + + if (a.isValid ()) + m_clusterNodes.insert (std::make_pair (a, ClusterNodeStatus(match[2]))); + } + else + WriteLog (lsWARNING, UniqueNodeList) << "Entry in cluster list invalid: '" << c << "'"; + } + + auto db = getApp().getWalletDB ().checkoutDb (); + ScopedUNLLockType slUNL (mUNLLock); + + mUNL.clear (); + + std::vector, 1>> columns; + selectBlobsIntoStrings(*db, + "SELECT PublicKey FROM TrustedNodes WHERE Score != 0;", + columns); + for(auto const& strArray : columns) + { + mUNL.insert (strArray[0].value_or("")); + } +} + +//-------------------------------------------------------------------------- + +// For a round of scoring we destribute points from a node to nodes it refers to. +// Returns true, iff scores were distributed. +// +bool UniqueNodeListImp::scoreRound (std::vector& vsnNodes) +{ + bool bDist = false; + + // For each node, distribute roundSeed to roundScores. + for (auto& sn : vsnNodes) + { + int iEntries = sn.viReferrals.size (); + + if (sn.iRoundSeed && iEntries) + { + score iTotal = (iEntries + 1) * iEntries / 2; + score iBase = sn.iRoundSeed * iEntries / iTotal; + + // Distribute the current entires' seed score to validators + // prioritized by mention order. + for (int i = 0; i != iEntries; i++) + { + score iPoints = iBase * (iEntries - i) / iEntries; + + vsnNodes[sn.viReferrals[i]].iRoundScore += iPoints; + } + } + } + + if (ShouldLog (lsTRACE, UniqueNodeList)) + { + WriteLog (lsTRACE, UniqueNodeList) << "midway: "; + for (auto& sn : vsnNodes) + { + WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("%s| %d, %d, %d: [%s]") + % sn.strValidator + % sn.iScore + % sn.iRoundScore + % sn.iRoundSeed + % strJoin (sn.viReferrals.begin (), sn.viReferrals.end (), ",")); + } + } + + // Add roundScore to score. + // Make roundScore new roundSeed. + for (auto& sn : vsnNodes) + { + if (!bDist && sn.iRoundScore) + bDist = true; + + sn.iScore += sn.iRoundScore; + sn.iRoundSeed = sn.iRoundScore; + sn.iRoundScore = 0; + } + + if (ShouldLog (lsTRACE, UniqueNodeList)) + { + WriteLog (lsTRACE, UniqueNodeList) << "finish: "; + for (auto& sn : vsnNodes) + { + WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("%s| %d, %d, %d: [%s]") + % sn.strValidator + % sn.iScore + % sn.iRoundScore + % sn.iRoundSeed + % strJoin (sn.viReferrals.begin (), sn.viReferrals.end (), ",")); + } + } + + return bDist; +} + +//-------------------------------------------------------------------------- + +// From SeedDomains and ValidatorReferrals compute scores and update TrustedNodes. +// +// VFALCO TODO Shrink this function, break it up +// +void UniqueNodeListImp::scoreCompute() +{ + hash_map umPulicIdx; // Map of public key to index. + hash_map umDomainIdx; // Map of domain to index. + std::vector vsnNodes; // Index to scoring node. + + // For each entry in SeedDomains with a PublicKey: + // - Add an entry in umPulicIdx, umDomainIdx, and vsnNodes. + { + auto db = getApp().getWalletDB ().checkoutDb (); + + std::vector, 3>> columns; + selectBlobsIntoStrings(*db, + "SELECT Domain,PublicKey,Source FROM SeedDomains;", + columns); + for(auto const& strArray : columns) + { + if (!strArray[1]) + // We ignore entries we don't have public keys for. + continue; + + std::string const strDomain = strArray[0].value_or(""); + std::string const strPublicKey = *strArray[1]; + std::string const strSource = strArray[2].value_or(""); + + assert (!strSource.empty ()); + + int const iScore = iSourceScore (static_cast (strSource[0])); + auto siOld = umPulicIdx.find (strPublicKey); + + if (siOld == umPulicIdx.end ()) + { + // New node + int iNode = vsnNodes.size (); + + umPulicIdx[strPublicKey] = iNode; + umDomainIdx[strDomain] = iNode; + + scoreNode snCurrent; + + snCurrent.strValidator = strPublicKey; + snCurrent.iScore = iScore; + snCurrent.iRoundSeed = snCurrent.iScore; + snCurrent.iRoundScore = 0; + snCurrent.iSeen = -1; + + vsnNodes.push_back (snCurrent); + } + else + { + scoreNode& snOld = vsnNodes[siOld->second]; + + if (snOld.iScore < iScore) + { + // Update old node + + snOld.iScore = iScore; + snOld.iRoundSeed = snOld.iScore; + } + } + } + } + + // For each entry in SeedNodes: + // - Add an entry in umPulicIdx, umDomainIdx, and vsnNodes. + { + auto db = getApp().getWalletDB ().checkoutDb (); + + std::vector, 2>> columns; + selectBlobsIntoStrings(*db, + "SELECT PublicKey,Source FROM SeedNodes;", + columns); + for(auto const& strArray : columns) + { + std::string strPublicKey = strArray[0].value_or(""); + std::string strSource = strArray[1].value_or(""); + assert (!strSource.empty ()); + int iScore = iSourceScore (static_cast (strSource[0])); + auto siOld = umPulicIdx.find (strPublicKey); + + if (siOld == umPulicIdx.end ()) + { + // New node + int iNode = vsnNodes.size (); + + umPulicIdx[strPublicKey] = iNode; + + scoreNode snCurrent; + + snCurrent.strValidator = strPublicKey; + snCurrent.iScore = iScore; + snCurrent.iRoundSeed = snCurrent.iScore; + snCurrent.iRoundScore = 0; + snCurrent.iSeen = -1; + + vsnNodes.push_back (snCurrent); + } + else + { + scoreNode& snOld = vsnNodes[siOld->second]; + + if (snOld.iScore < iScore) + { + // Update old node + + snOld.iScore = iScore; + snOld.iRoundSeed = snOld.iScore; + } + } + } + } + + // For debugging, print out initial scores. + if (ShouldLog (lsTRACE, UniqueNodeList)) + { + for (auto& sn : vsnNodes) + { + WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("%s| %d, %d, %d") + % sn.strValidator + % sn.iScore + % sn.iRoundScore + % sn.iRoundSeed); + } + } + + // WriteLog (lsTRACE, UniqueNodeList) << str(boost::format("vsnNodes.size=%d") % vsnNodes.size()); + + // Step through growing list of nodes adding each validation list. + // - Each validator may have provided referals. Add those referals as validators. + for (int iNode = 0; iNode != vsnNodes.size (); ++iNode) + { + scoreNode& sn = vsnNodes[iNode]; + std::string& strValidator = sn.strValidator; + std::vector& viReferrals = sn.viReferrals; + + auto db = getApp().getWalletDB ().checkoutDb (); + + std::vector, 1>> columns; + selectBlobsIntoStrings(*db, + boost::str (boost::format ( + "SELECT Referral FROM ValidatorReferrals " + "WHERE Validator=%s ORDER BY Entry;") % + sqlEscape (strValidator)), + columns); + std::string strReferral; + for(auto const& strArray : columns) + { + strReferral = strArray[0].value_or(""); + + int iReferral; + + RippleAddress na; + + if (na.setNodePublic (strReferral)) + { + // Referring a public key. + auto itEntry = umPulicIdx.find (strReferral); + + if (itEntry == umPulicIdx.end ()) + { + // Not found add public key to list of nodes. + iReferral = vsnNodes.size (); + + umPulicIdx[strReferral] = iReferral; + + scoreNode snCurrent; + + snCurrent.strValidator = strReferral; + snCurrent.iScore = iSourceScore (vsReferral); + snCurrent.iRoundSeed = snCurrent.iScore; + snCurrent.iRoundScore = 0; + snCurrent.iSeen = -1; + + vsnNodes.push_back (snCurrent); + } + else + { + iReferral = itEntry->second; + } + } + else + { + // Referring a domain. + auto itEntry = umDomainIdx.find (strReferral); + iReferral = itEntry == umDomainIdx.end () + ? -1 // We ignore domains we can't find entires for. + : itEntry->second; + } + + if (iReferral >= 0 && iNode != iReferral) + viReferrals.push_back (iReferral); + } + } + + // + // Distribute the points from the seeds. + // + bool bDist = true; + + for (int i = SCORE_ROUNDS; bDist && i--;) + bDist = scoreRound (vsnNodes); + + if (ShouldLog (lsTRACE, UniqueNodeList)) + { + WriteLog (lsTRACE, UniqueNodeList) << "Scored:"; + for (auto& sn : vsnNodes) + { + WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("%s| %d, %d, %d: [%s]") + % sn.strValidator + % sn.iScore + % sn.iRoundScore + % sn.iRoundSeed + % strJoin (sn.viReferrals.begin (), sn.viReferrals.end (), ",")); + } + } + + // Persist validator scores. + auto db = getApp().getWalletDB ().checkoutDb (); + + soci::transaction tr(*db); + *db << "UPDATE TrustedNodes SET Score = 0 WHERE Score != 0;"; + + if (!vsnNodes.empty ()) + { + // Load existing Seens from DB. + std::vector vstrPublicKeys; + + vstrPublicKeys.resize (vsnNodes.size ()); + + for (int iNode = vsnNodes.size (); iNode--;) + { + vstrPublicKeys[iNode] = sqlEscape (vsnNodes[iNode].strValidator); + } + + // Iterate through the result rows with a fectch b/c putting a + // column of type DATETIME into a boost::tuple can throw when the + // datetime column is invalid (even if the value as int is valid). + std::vector, + boost::optional>> columns; + selectBlobsIntoStrings ( + *db, + str (boost::format ( + "SELECT PublicKey,Seen FROM TrustedNodes WHERE " + "PublicKey IN (%s);") % + strJoin ( + vstrPublicKeys.begin (), vstrPublicKeys.end (), ",")), + columns); + std::string pk; + for(auto const& col : columns) + { + pk = get<0>(col).value_or (""); + + vsnNodes[umPulicIdx[pk]].iSeen = get<1>(col).value_or (-1); + } + } + + hash_set usUNL; + + if (!vsnNodes.empty ()) + { + // Update the score old entries and add new entries as needed. + std::vector vstrValues; + + vstrValues.resize (vsnNodes.size ()); + + for (int iNode = vsnNodes.size (); iNode--;) + { + scoreNode& sn = vsnNodes[iNode]; + std::string strSeen = sn.iSeen >= 0 ? str (boost::format ("%d") % sn.iSeen) : "NULL"; + + vstrValues[iNode] = str (boost::format ("(%s,%s,%s)") + % sqlEscape (sn.strValidator) + % sn.iScore + % strSeen); + + usUNL.insert (sn.strValidator); + } + + *db << str (boost::format ("REPLACE INTO TrustedNodes (PublicKey,Score,Seen) VALUES %s;") + % strJoin (vstrValues.begin (), vstrValues.end (), ",")); + } + + { + ScopedUNLLockType sl (mUNLLock); + + // XXX Should limit to scores above a certain minimum and limit to a certain number. + mUNL.swap (usUNL); + } + + hash_map umValidators; + + if (!vsnNodes.empty ()) + { + // For every IpReferral add a score for the IP and PORT. + std::vector, + boost::optional>> columns; + selectBlobsIntoStrings ( + *db, + "SELECT Validator,COUNT(*) AS Count FROM " + "IpReferrals GROUP BY Validator;", + columns); + for(auto const& col : columns) + { + umValidators[get<0>(col).value_or("")] = get<1>(col).value_or(0); + + // WriteLog (lsTRACE, UniqueNodeList) << strValidator << ":" << db->getInt("Count"); + } + } + + // For each validator, get each referral and add its score to ip's score. + // map of pair :: score + hash_map, score> umScore; + + for (auto& vc : umValidators) + { + std::string strValidator = vc.first; + + auto itIndex = umPulicIdx.find (strValidator); + + if (itIndex != umPulicIdx.end ()) + { + int iSeed = vsnNodes[itIndex->second].iScore; + int iEntries = vc.second; + score iTotal = (iEntries + 1) * iEntries / 2; + score iBase = iSeed * iEntries / iTotal; + int iEntry = 0; + + std::vector, + boost::optional>> columns; + selectBlobsIntoStrings ( + *db, + str (boost::format ( + "SELECT IP,Port FROM IpReferrals WHERE " + "Validator=%s ORDER BY Entry;") % + sqlEscape (strValidator)), + columns); + for(auto const& col : columns) + { + score iPoints = iBase * (iEntries - iEntry) / iEntries; + int iPort; + + iPort = get<1>(col).value_or(0); + + std::pair< std::string, int> ep = std::make_pair (get<0>(col).value_or(""), iPort); + + auto itEp = umScore.find (ep); + + umScore[ep] = itEp == umScore.end () ? iPoints : itEp->second + iPoints; + iEntry++; + } + } + } + + tr.commit (); +} + +//-------------------------------------------------------------------------- + +// Start a timer to update scores. +// <-- bNow: true, to force scoring for debugging. +void UniqueNodeListImp::scoreNext (bool bNow) +{ + // WriteLog (lsTRACE, UniqueNodeList) << str(boost::format("scoreNext: mtpFetchUpdated=%s mtpScoreStart=%s mtpScoreUpdated=%s mtpScoreNext=%s") % mtpFetchUpdated % mtpScoreStart % mtpScoreUpdated % mtpScoreNext); + bool bCanScore = mtpScoreStart.is_not_a_date_time () // Not scoring. + && !mtpFetchUpdated.is_not_a_date_time (); // Something to score. + + bool bDirty = + (mtpScoreUpdated.is_not_a_date_time () || mtpScoreUpdated <= mtpFetchUpdated) // Not already scored. + && (mtpScoreNext.is_not_a_date_time () // Timer is not fine. + || mtpScoreNext < mtpFetchUpdated + boost::posix_time::seconds (SCORE_DELAY_SECONDS)); + + if (bCanScore && (bNow || bDirty)) + { + // Need to update or set timer. + double const secondsFromNow = bNow ? 0 : SCORE_DELAY_SECONDS; + mtpScoreNext = boost::posix_time::second_clock::universal_time () // Past now too. + + boost::posix_time::seconds (secondsFromNow); + + // WriteLog (lsTRACE, UniqueNodeList) << str(boost::format("scoreNext: @%s") % mtpScoreNext); + m_scoreTimer.setExpiration (secondsFromNow); + } +} + +//-------------------------------------------------------------------------- + +// Given a ripple.txt, process it. +// +// VFALCO TODO Can't we take a filename or stream instead of a string? +// +bool UniqueNodeListImp::responseFetch (std::string const& strDomain, const boost::system::error_code& err, int iStatus, std::string const& strSiteFile) +{ + bool bReject = !err && iStatus != 200; + + if (!bReject) + { + IniFileSections secSite = parseIniFile (strSiteFile, true); + bool bGood = !err; + + if (bGood) + { + WriteLog (lsTRACE, UniqueNodeList) << strDomain + << ": retrieved configuration"; + } + else + { + WriteLog (lsTRACE, UniqueNodeList) << strDomain + << ": unable to retrieve configuration: " + << err.message (); + } + + // + // Verify file domain + // + std::string strSite; + + if (bGood && !getSingleSection (secSite, SECTION_DOMAIN, strSite)) + { + bGood = false; + + WriteLog (lsTRACE, UniqueNodeList) << strDomain + << ": " << SECTION_DOMAIN + << "entry missing."; + } + + if (bGood && strSite != strDomain) + { + bGood = false; + + WriteLog (lsTRACE, UniqueNodeList) << strDomain + << ": " << SECTION_DOMAIN << " does not match " << strSite; + } + + // + // Process public key + // + std::string strNodePublicKey; + + if (bGood && !getSingleSection (secSite, SECTION_PUBLIC_KEY, strNodePublicKey)) + { + // Bad [validation_public_key] IniFileSections. + bGood = false; + + WriteLog (lsTRACE, UniqueNodeList) << strDomain + << ": " << SECTION_PUBLIC_KEY << " entry missing."; + } + + RippleAddress naNodePublic; + + if (bGood && !naNodePublic.setNodePublic (strNodePublicKey)) + { + // Bad public key. + bGood = false; + + WriteLog (lsTRACE, UniqueNodeList) << strDomain + << ": " << SECTION_PUBLIC_KEY << " is not a public key: " + << strNodePublicKey; + } + + if (bGood) + { + seedDomain sdCurrent; + + bool bFound = getSeedDomains (strDomain, sdCurrent); + assert (bFound); + (void) bFound; + + uint256 iSha256 = getSHA512Half (strSiteFile); + bool bChangedB = sdCurrent.iSha256 != iSha256; + + sdCurrent.strDomain = strDomain; + // XXX If the node public key is changing, delete old public key information? + // XXX Only if no other refs to keep it arround, other wise we have an attack vector. + sdCurrent.naPublicKey = naNodePublic; + + // WriteLog (lsTRACE, UniqueNodeList) << boost::format("sdCurrent.naPublicKey: '%s'") % sdCurrent.naPublicKey.humanNodePublic(); + + sdCurrent.tpFetch = boost::posix_time::second_clock::universal_time (); + sdCurrent.iSha256 = iSha256; + + setSeedDomains (sdCurrent, true); + + if (bChangedB) + { + WriteLog (lsTRACE, UniqueNodeList) << strDomain + << ": processing new " << node_file_name_ << "."; + processFile (strDomain, naNodePublic, secSite); + } + else + { + WriteLog (lsTRACE, UniqueNodeList) << strDomain + << ": no change in " << node_file_name_ << "."; + fetchFinish (); + } + } + else + { + // Failed: Update + + // XXX If we have public key, perhaps try look up in CAS? + fetchFinish (); + } + } + + return bReject; +} + +//-------------------------------------------------------------------------- + +// Try to process the next fetch of a ripple.txt. +void UniqueNodeListImp::fetchNext() +{ + bool bFull; + + { + ScopedFetchLockType sl (mFetchLock); + + bFull = (mFetchActive == NODE_FETCH_JOBS); + } + + if (!bFull) + { + // Determine next scan. + std::string strDomain; + boost::posix_time::ptime tpNext (boost::posix_time::min_date_time); + boost::posix_time::ptime tpNow (boost::posix_time::second_clock::universal_time ()); + + auto db = getApp().getWalletDB ().checkoutDb (); + + + { + soci::blob b(*db); + soci::indicator ind; + boost::optional nO; + *db << "SELECT Domain,Next FROM SeedDomains INDEXED BY SeedDomainNext ORDER BY Next LIMIT 1;", + soci::into(b, ind), + soci::into(nO); + if (nO) + { + int iNext (*nO); + + tpNext = ptFromSeconds (iNext); + + WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("fetchNext: iNext=%s tpNext=%s tpNow=%s") % iNext % tpNext % tpNow); + if (soci::i_ok == ind) + convert (b, strDomain); + else + strDomain.clear (); + } + } + + if (!strDomain.empty ()) + { + ScopedFetchLockType sl (mFetchLock); + + bFull = (mFetchActive == NODE_FETCH_JOBS); + + if (!bFull && tpNext <= tpNow) + { + mFetchActive++; + } + } + + if (strDomain.empty () || bFull) + { + WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("fetchNext: strDomain=%s bFull=%d") % strDomain % bFull); + } + else if (tpNext > tpNow) + { + WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("fetchNext: set timer : strDomain=%s") % strDomain); + // Fetch needs to happen in the future. Set a timer to wake us. + mtpFetchNext = tpNext; + + double seconds = (tpNext - tpNow).seconds (); + + // VFALCO check this. + if (seconds == 0) + seconds = 1; + + m_fetchTimer.setExpiration (seconds); + } + else + { + WriteLog (lsTRACE, UniqueNodeList) << str (boost::format ("fetchNext: fetch now: strDomain=%s tpNext=%s tpNow=%s") % strDomain % tpNext % tpNow); + // Fetch needs to happen now. + mtpFetchNext = boost::posix_time::ptime (boost::posix_time::not_a_date_time); + + seedDomain sdCurrent; + bool bFound = getSeedDomains (strDomain, sdCurrent); + assert (bFound); + (void) bFound; + + // Update time of next fetch and this scan attempt. + sdCurrent.tpScan = tpNow; + + // XXX Use a longer duration if we have lots of validators. + sdCurrent.tpNext = sdCurrent.tpScan + boost::posix_time::hours (7 * 24); + + setSeedDomains (sdCurrent, false); + + WriteLog (lsTRACE, UniqueNodeList) << strDomain + << " fetching " << node_file_name_ << "."; + + fetchProcess (strDomain); // Go get it. + + fetchNext (); // Look for more. + } + } +} + +//-------------------------------------------------------------------------- + +// Called when we need to update scores. +void UniqueNodeListImp::fetchDirty() +{ + // Note update. + mtpFetchUpdated = boost::posix_time::second_clock::universal_time (); + miscSave (); + + // Update scores. + scoreNext (false); +} + + +//-------------------------------------------------------------------------- + +void UniqueNodeListImp::fetchFinish() +{ + { + ScopedFetchLockType sl (mFetchLock); + mFetchActive--; + } + + fetchNext (); +} + +//-------------------------------------------------------------------------- + +// Get the ripple.txt and process it. +void UniqueNodeListImp::fetchProcess (std::string strDomain) +{ + WriteLog (lsTRACE, UniqueNodeList) << strDomain + << ": fetching " << node_file_name_ << "."; + + std::deque deqSites; + + // Order searching from most specifically for purpose to generic. + // This order allows the client to take the most burden rather than the servers. + deqSites.push_back (systemName () + strDomain); + deqSites.push_back ("www." + strDomain); + deqSites.push_back (strDomain); + + HTTPClient::get ( + true, + getApp().getIOService (), + deqSites, + 443, + node_file_path_, + NODE_FILE_BYTES_MAX, + boost::posix_time::seconds (NODE_FETCH_SECONDS), + std::bind (&UniqueNodeListImp::responseFetch, this, strDomain, + std::placeholders::_1, std::placeholders::_2, + std::placeholders::_3)); +} + +// Process IniFileSections [validators_url]. +void UniqueNodeListImp::getValidatorsUrl (RippleAddress const& naNodePublic, + IniFileSections secSite) +{ + std::string strValidatorsUrl; + std::string strScheme; + std::string strDomain; + int iPort; + std::string strPath; + + if (getSingleSection (secSite, SECTION_VALIDATORS_URL, strValidatorsUrl) + && !strValidatorsUrl.empty () + && parseUrl (strValidatorsUrl, strScheme, strDomain, iPort, strPath) + && -1 == iPort + && strScheme == "https") + { + HTTPClient::get ( + true, + getApp().getIOService (), + strDomain, + 443, + strPath, + NODE_FILE_BYTES_MAX, + boost::posix_time::seconds (NODE_FETCH_SECONDS), + std::bind (&UniqueNodeListImp::responseValidators, this, + strValidatorsUrl, naNodePublic, secSite, strDomain, + std::placeholders::_1, std::placeholders::_2, + std::placeholders::_3)); + } + else + { + getIpsUrl (naNodePublic, secSite); + } +} + +//-------------------------------------------------------------------------- + +// Process IniFileSections [ips_url]. +// If we have a IniFileSections with a single entry, fetch the url and process it. +void UniqueNodeListImp::getIpsUrl (RippleAddress const& naNodePublic, IniFileSections secSite) +{ + std::string strIpsUrl; + std::string strScheme; + std::string strDomain; + int iPort; + std::string strPath; + + if (getSingleSection (secSite, SECTION_IPS_URL, strIpsUrl) + && !strIpsUrl.empty () + && parseUrl (strIpsUrl, strScheme, strDomain, iPort, strPath) + && -1 == iPort + && strScheme == "https") + { + HTTPClient::get ( + true, + getApp().getIOService (), + strDomain, + 443, + strPath, + NODE_FILE_BYTES_MAX, + boost::posix_time::seconds (NODE_FETCH_SECONDS), + std::bind (&UniqueNodeListImp::responseIps, this, strDomain, + naNodePublic, std::placeholders::_1, + std::placeholders::_2, std::placeholders::_3)); + } + else + { + fetchFinish (); + } +} + + +//-------------------------------------------------------------------------- + +// Given a IniFileSections with IPs, parse and persist it for a validator. +bool UniqueNodeListImp::responseIps (std::string const& strSite, RippleAddress const& naNodePublic, const boost::system::error_code& err, int iStatus, std::string const& strIpsFile) +{ + bool bReject = !err && iStatus != 200; + + if (!bReject) + { + if (!err) + { + IniFileSections secFile = parseIniFile (strIpsFile, true); + + processIps (strSite, naNodePublic, getIniFileSection (secFile, SECTION_IPS)); + } + + fetchFinish (); + } + + return bReject; +} + +// After fetching a ripple.txt from a web site, given a IniFileSections with validators, parse and persist it. +bool UniqueNodeListImp::responseValidators (std::string const& strValidatorsUrl, RippleAddress const& naNodePublic, IniFileSections secSite, std::string const& strSite, const boost::system::error_code& err, int iStatus, std::string const& strValidatorsFile) +{ + bool bReject = !err && iStatus != 200; + + if (!bReject) + { + if (!err) + { + IniFileSections secFile = parseIniFile (strValidatorsFile, true); + + processValidators (strSite, strValidatorsUrl, naNodePublic, vsValidator, getIniFileSection (secFile, SECTION_VALIDATORS)); + } + + getIpsUrl (naNodePublic, secSite); + } + + return bReject; +} + + +//-------------------------------------------------------------------------- + +// Persist the IPs refered to by a Validator. +// --> strSite: source of the IPs (for debugging) +// --> naNodePublic: public key of the validating node. +void UniqueNodeListImp::processIps (std::string const& strSite, RippleAddress const& naNodePublic, IniFileSections::mapped_type* pmtVecStrIps) +{ + std::string strEscNodePublic = sqlEscape (naNodePublic.humanNodePublic ()); + + WriteLog (lsDEBUG, UniqueNodeList) + << str (boost::format ("Validator: '%s' processing %d ips.") + % strSite % ( pmtVecStrIps ? pmtVecStrIps->size () : 0)); + + // Remove all current Validator's entries in IpReferrals + { + auto db = getApp().getWalletDB ().checkoutDb (); + *db << str (boost::format ("DELETE FROM IpReferrals WHERE Validator=%s;") % strEscNodePublic); + } + + // Add new referral entries. + if (pmtVecStrIps && !pmtVecStrIps->empty ()) + { + std::vector vstrValues; + + vstrValues.resize (std::min ((int) pmtVecStrIps->size (), REFERRAL_IPS_MAX)); + + int iValues = 0; + for (auto const& strReferral : *pmtVecStrIps) + { + if (iValues == REFERRAL_VALIDATORS_MAX) + break; + + std::string strIP; + int iPort; + bool bValid = parseIpPort (strReferral, strIP, iPort); + + // XXX Filter out private network ips. + // XXX http://en.wikipedia.org/wiki/Private_network + + if (bValid) + { + vstrValues[iValues] = str (boost::format ("(%s,%d,%s,%d)") + % strEscNodePublic % iValues % sqlEscape (strIP) % iPort); + iValues++; + } + else + { + WriteLog (lsTRACE, UniqueNodeList) + << str (boost::format ("Validator: '%s' [" SECTION_IPS "]: rejecting '%s'") + % strSite % strReferral); + } + } + + if (iValues) + { + vstrValues.resize (iValues); + + auto db = getApp().getWalletDB ().checkoutDb (); + *db << str (boost::format ("INSERT INTO IpReferrals (Validator,Entry,IP,Port) VALUES %s;") + % strJoin (vstrValues.begin (), vstrValues.end (), ",")); + // XXX Check result. + } + } + + fetchDirty (); +} + +//-------------------------------------------------------------------------- + +// Persist ValidatorReferrals. +// --> strSite: source site for display +// --> strValidatorsSrc: source details for display +// --> naNodePublic: remote source public key - not valid for local +// --> vsWhy: reason for adding validator to SeedDomains or SeedNodes. +int UniqueNodeListImp::processValidators (std::string const& strSite, std::string const& strValidatorsSrc, RippleAddress const& naNodePublic, ValidatorSource vsWhy, IniFileSections::mapped_type* pmtVecStrValidators) +{ + std::string strNodePublic = naNodePublic.isValid () ? naNodePublic.humanNodePublic () : strValidatorsSrc; + int iValues = 0; + + WriteLog (lsTRACE, UniqueNodeList) + << str (boost::format ("Validator: '%s' : '%s' : processing %d validators.") + % strSite + % strValidatorsSrc + % ( pmtVecStrValidators ? pmtVecStrValidators->size () : 0)); + + // Remove all current Validator's entries in ValidatorReferrals + { + auto db = getApp().getWalletDB ().checkoutDb (); + + *db << str (boost::format ("DELETE FROM ValidatorReferrals WHERE Validator='%s';") % strNodePublic); + // XXX Check result. + } + + // Add new referral entries. + if (pmtVecStrValidators && pmtVecStrValidators->size ()) + { + std::vector vstrValues; + + vstrValues.reserve (std::min ((int) pmtVecStrValidators->size (), REFERRAL_VALIDATORS_MAX)); + + for (auto const& strReferral : *pmtVecStrValidators) + { + if (iValues == REFERRAL_VALIDATORS_MAX) + break; + + boost::smatch smMatch; + + // domain comment? + // public_key comment? + static boost::regex reReferral ("\\`\\s*(\\S+)(?:\\s+(.+))?\\s*\\'"); + + if (!boost::regex_match (strReferral, smMatch, reReferral)) + { + WriteLog (lsWARNING, UniqueNodeList) << str (boost::format ("Bad validator: syntax error: %s: %s") % strSite % strReferral); + } + else + { + std::string strRefered = smMatch[1]; + std::string strComment = smMatch[2]; + RippleAddress naValidator; + + if (naValidator.setSeedGeneric (strRefered)) + { + WriteLog (lsWARNING, UniqueNodeList) << str (boost::format ("Bad validator: domain or public key required: %s %s") % strRefered % strComment); + } + else if (naValidator.setNodePublic (strRefered)) + { + // A public key. + // XXX Schedule for CAS lookup. + nodeAddPublic (naValidator, vsWhy, strComment); + + WriteLog (lsINFO, UniqueNodeList) << str (boost::format ("Node Public: %s %s") % strRefered % strComment); + + if (naNodePublic.isValid ()) + vstrValues.push_back (str (boost::format ("('%s',%d,'%s')") % strNodePublic % iValues % naValidator.humanNodePublic ())); + + iValues++; + } + else + { + // A domain: need to look it up. + nodeAddDomain (strRefered, vsWhy, strComment); + + WriteLog (lsINFO, UniqueNodeList) << str (boost::format ("Node Domain: %s %s") % strRefered % strComment); + + if (naNodePublic.isValid ()) + vstrValues.push_back (str (boost::format ("('%s',%d,%s)") % strNodePublic % iValues % sqlEscape (strRefered))); + + iValues++; + } + } + } + + if (!vstrValues.empty ()) + { + std::string strSql = str (boost::format ("INSERT INTO ValidatorReferrals (Validator,Entry,Referral) VALUES %s;") + % strJoin (vstrValues.begin (), vstrValues.end (), ",")); + + auto db = getApp().getWalletDB ().checkoutDb (); + + *db << strSql; + // XXX Check result. + } + } + + fetchDirty (); + + return iValues; +} + +//-------------------------------------------------------------------------- + +// Process a ripple.txt. +void UniqueNodeListImp::processFile (std::string const& strDomain, RippleAddress const& naNodePublic, IniFileSections secSite) +{ + // + // Process Validators + // + processValidators (strDomain, node_file_name_, naNodePublic, + vsReferral, getIniFileSection (secSite, SECTION_VALIDATORS)); + + // + // Process ips + // + processIps (strDomain, naNodePublic, getIniFileSection (secSite, SECTION_IPS)); + + // + // Process currencies + // + IniFileSections::mapped_type* pvCurrencies; + + if ((pvCurrencies = getIniFileSection (secSite, SECTION_CURRENCIES)) && pvCurrencies->size ()) + { + // XXX Process currencies. + WriteLog (lsWARNING, UniqueNodeList) << "Ignoring currencies: not implemented."; + } + + getValidatorsUrl (naNodePublic, secSite); +} + +//-------------------------------------------------------------------------- + +// Retrieve a SeedDomain from DB. +bool UniqueNodeListImp::getSeedDomains (std::string const& strDomain, seedDomain& dstSeedDomain) +{ + bool bResult = false; + + std::string strSql = boost::str ( + boost::format ( + "SELECT Domain, PublicKey, Source, Next, Scan, Fetch, Sha256, " + "Comment FROM SeedDomains WHERE Domain=%s;") % + sqlEscape (strDomain)); + + auto db = getApp().getWalletDB ().checkoutDb (); + + // Iterate through the result rows with a fectch b/c putting a + // column of type DATETIME into a boost::tuple can throw when the + // datetime column is invalid (even if the value as int is valid). + soci::blob domainBlob(*db); + soci::indicator di; + boost::optional strPublicKey; + soci:: blob sourceBlob(*db); + soci::indicator si; + std::string strSource; + boost::optional iNext; + boost::optional iScan; + boost::optional iFetch; + boost::optional strSha256; + soci::blob commentBlob(*db); + soci::indicator ci; + boost::optional strComment; + + soci::statement st = (db->prepare << strSql, + soci::into (domainBlob, di), + soci::into (strPublicKey), + soci::into (sourceBlob, si), + soci::into (iNext), + soci::into (iScan), + soci::into (iFetch), + soci::into (strSha256), + soci::into (commentBlob, ci)); + + st.execute (); + while (st.fetch ()) + { + bResult = true; + + if (soci::i_ok == di) + convert (domainBlob, dstSeedDomain.strDomain); + + if (strPublicKey && !strPublicKey->empty ()) + dstSeedDomain.naPublicKey.setNodePublic (*strPublicKey); + else + dstSeedDomain.naPublicKey.clear (); + + if (soci::i_ok == si) + { + convert (sourceBlob, strSource); + dstSeedDomain.vsSource = static_cast (strSource[0]); + } + else + { + assert (0); + } + + dstSeedDomain.tpNext = ptFromSeconds (iNext.value_or (0)); + dstSeedDomain.tpScan = ptFromSeconds (iScan.value_or (0)); + dstSeedDomain.tpFetch = ptFromSeconds (iFetch.value_or (0)); + + if (strSha256 && !strSha256->empty ()) + dstSeedDomain.iSha256.SetHex (*strSha256); + else + dstSeedDomain.iSha256.zero (); + + if (soci::i_ok == ci) + convert (commentBlob, dstSeedDomain.strComment); + else + dstSeedDomain.strComment.clear (); + } + + return bResult; +} + +//-------------------------------------------------------------------------- + +// Persist a SeedDomain. +void UniqueNodeListImp::setSeedDomains (const seedDomain& sdSource, bool bNext) +{ + int iNext = iToSeconds (sdSource.tpNext); + int iScan = iToSeconds (sdSource.tpScan); + int iFetch = iToSeconds (sdSource.tpFetch); + + // WriteLog (lsTRACE) << str(boost::format("setSeedDomains: iNext=%s tpNext=%s") % iNext % sdSource.tpNext); + + std::string strSql = boost::str (boost::format ("REPLACE INTO SeedDomains (Domain,PublicKey,Source,Next,Scan,Fetch,Sha256,Comment) VALUES (%s, %s, %s, %d, %d, %d, '%s', %s);") + % sqlEscape (sdSource.strDomain) + % (sdSource.naPublicKey.isValid () ? sqlEscape (sdSource.naPublicKey.humanNodePublic ()) : "NULL") + % sqlEscape (std::string (1, static_cast (sdSource.vsSource))) + % iNext + % iScan + % iFetch + % to_string (sdSource.iSha256) + % sqlEscape (sdSource.strComment) + ); + + auto db = getApp().getWalletDB ().checkoutDb (); + + try + { + *db << strSql; + } + catch (soci::soci_error& e) + { + // XXX Check result. + WriteLog (lsWARNING, UniqueNodeList) << "setSeedDomains: failed. Error: " << e.what(); + } + + if (bNext && (mtpFetchNext.is_not_a_date_time () || mtpFetchNext > sdSource.tpNext)) + { + // Schedule earlier wake up. + fetchNext (); + } +} + + +//-------------------------------------------------------------------------- + +// Retrieve a SeedNode from DB. +bool UniqueNodeListImp::getSeedNodes (RippleAddress const& naNodePublic, seedNode& dstSeedNode) +{ + std::string strSql = + str (boost::format ( + "SELECT PublicKey, Source, Next, Scan, Fetch, Sha256, " + "Comment FROM SeedNodes WHERE PublicKey='%s';") % + naNodePublic.humanNodePublic ()); + + auto db = getApp().getWalletDB ().checkoutDb (); + + std::string strPublicKey; + std::string strSource; + soci::blob sourceBlob(*db); + soci::indicator si; + boost::optional iNext; + boost::optional iScan; + boost::optional iFetch; + boost::optional strSha256; + soci::blob commentBlob(*db); + soci::indicator ci; + + *db << strSql, + soci::into (strPublicKey), + soci::into (sourceBlob, si), + soci::into (iNext), + soci::into (iScan), + soci::into (iFetch), + soci::into (strSha256), + soci::into (commentBlob, ci); + + if (!db->got_data ()) + return false; + + if (!strPublicKey.empty ()) + dstSeedNode.naPublicKey.setNodePublic (strPublicKey); + else + dstSeedNode.naPublicKey.clear (); + + if (soci::i_ok == si) + { + convert (sourceBlob, strSource); + dstSeedNode.vsSource = static_cast (strSource[0]); + } + else + assert (0); + + dstSeedNode.tpNext = ptFromSeconds (iNext.value_or(0)); + dstSeedNode.tpScan = ptFromSeconds (iScan.value_or(0)); + dstSeedNode.tpFetch = ptFromSeconds (iFetch.value_or(0)); + + if (strSha256 && !strSha256->empty ()) + dstSeedNode.iSha256.SetHex (*strSha256); + else + dstSeedNode.iSha256.zero (); + + if (soci::i_ok == ci) + convert (commentBlob, dstSeedNode.strComment); + else + dstSeedNode.strComment.clear (); + + return true; +} + +//-------------------------------------------------------------------------- + +// Persist a SeedNode. +// <-- bNext: true, to do fetching if needed. +void UniqueNodeListImp::setSeedNodes (const seedNode& snSource, bool bNext) +{ + int iNext = iToSeconds (snSource.tpNext); + int iScan = iToSeconds (snSource.tpScan); + int iFetch = iToSeconds (snSource.tpFetch); + + // WriteLog (lsTRACE) << str(boost::format("setSeedNodes: iNext=%s tpNext=%s") % iNext % sdSource.tpNext); + + assert (snSource.naPublicKey.isValid ()); + + std::string strSql = str (boost::format ("REPLACE INTO SeedNodes (PublicKey,Source,Next,Scan,Fetch,Sha256,Comment) VALUES ('%s', '%c', %d, %d, %d, '%s', %s);") + % snSource.naPublicKey.humanNodePublic () + % static_cast (snSource.vsSource) + % iNext + % iScan + % iFetch + % to_string (snSource.iSha256) + % sqlEscape (snSource.strComment) + ); + + { + auto db = getApp().getWalletDB ().checkoutDb (); + + try + { + *db << strSql; + } + catch(soci::soci_error& e) + { + WriteLog (lsTRACE, UniqueNodeList) << "setSeedNodes: failed. Error: " << e.what (); + } + } + +#if 0 + + // YYY When we have a cas schedule lookups similar to this. + if (bNext && (mtpFetchNext.is_not_a_date_time () || mtpFetchNext > snSource.tpNext)) + { + // Schedule earlier wake up. + fetchNext (); + } + +#else + fetchDirty (); +#endif +} + +//-------------------------------------------------------------------------- + +bool UniqueNodeListImp::validatorsResponse (const boost::system::error_code& err, int iStatus, std::string strResponse) +{ + bool bReject = !err && iStatus != 200; + + if (!bReject) + { + WriteLog (lsTRACE, UniqueNodeList) << + "Fetch '" << + Config::Helpers::getValidatorsFileName () << + "' complete."; + + if (!err) + { + nodeProcess ("network", strResponse, getConfig ().VALIDATORS_SITE); + } + else + { + WriteLog (lsWARNING, UniqueNodeList) << "Error: " << err.message (); + } + } + return bReject; +} + +//-------------------------------------------------------------------------- + +// Process a validators.txt. +// --> strSite: source of validators +// --> strValidators: contents of a validators.txt +// +// VFALCO TODO Can't we name this processValidatorList? +// +void UniqueNodeListImp::nodeProcess (std::string const& strSite, std::string const& strValidators, std::string const& strSource) +{ + IniFileSections secValidators = parseIniFile (strValidators, true); + + IniFileSections::mapped_type* pmtEntries = getIniFileSection (secValidators, SECTION_VALIDATORS); + + if (pmtEntries) + { + RippleAddress naInvalid; // Don't want a referrer on added entries. + + // YYY Unspecified might be bootstrap or rpc command + processValidators (strSite, strSource, naInvalid, vsValidator, pmtEntries); + } + else + { + WriteLog (lsWARNING, UniqueNodeList) << boost::str (boost::format ("'%s' missing [" SECTION_VALIDATORS "].") + % getConfig ().VALIDATORS_BASE); + } +} + +//------------------------------------------------------------------------------ + std::unique_ptr make_UniqueNodeList (beast::Stoppable& parent) { diff --git a/src/ripple/basics/strHex.h b/src/ripple/basics/strHex.h index 0212056de..ea58f8014 100644 --- a/src/ripple/basics/strHex.h +++ b/src/ripple/basics/strHex.h @@ -25,6 +25,7 @@ #ifndef RIPPLE_BASICS_STRHEX_H_INCLUDED #define RIPPLE_BASICS_STRHEX_H_INCLUDED +#include #include namespace ripple { diff --git a/src/ripple/crypto/impl/Base58.cpp b/src/ripple/crypto/impl/Base58.cpp index adcbb2908..77c328b21 100644 --- a/src/ripple/crypto/impl/Base58.cpp +++ b/src/ripple/crypto/impl/Base58.cpp @@ -203,7 +203,7 @@ bool Base58::decode (const char* psz, Blob& vchRet, Alphabet const& alphabet) bn += bnChar; } - // Get bignum as little endian data + // Get bignum as big endian data Blob vchTmp = bn.getvch (); // Trim off sign byte if present @@ -218,7 +218,7 @@ bool Base58::decode (const char* psz, Blob& vchRet, Alphabet const& alphabet) vchRet.assign (nLeadingZeros + vchTmp.size (), 0); - // Convert little endian data to big endian + // Convert big endian data to little endian std::reverse_copy (vchTmp.begin (), vchTmp.end (), vchRet.end () - vchTmp.size ()); return true; } diff --git a/src/ripple/overlay/impl/OverlayImpl.h b/src/ripple/overlay/impl/OverlayImpl.h index 6378b6988..4b4f1d7e6 100644 --- a/src/ripple/overlay/impl/OverlayImpl.h +++ b/src/ripple/overlay/impl/OverlayImpl.h @@ -97,30 +97,21 @@ private: boost::asio::io_service& io_service_; boost::optional work_; boost::asio::io_service::strand strand_; - std::recursive_mutex mutex_; // VFALCO use std::mutex std::condition_variable_any cond_; std::weak_ptr timer_; boost::container::flat_map< Child*, std::weak_ptr> list_; - Setup setup_; beast::Journal journal_; ServerHandler& serverHandler_; - Resource::Manager& m_resourceManager; - std::unique_ptr m_peerFinder; - hash_map > m_peers; - hash_map> m_publicKeyMap; - hash_map> m_shortIdMap; - Resolver& m_resolver; - std::atomic next_id_; int timer_count_; @@ -212,7 +203,7 @@ public: void activate (std::shared_ptr const& peer); - /** Called when an active peer is destroyed. */ + // Called when an active peer is destroyed. void onPeerDeactivate (Peer::id_t id, RippleAddress const& publicKey); @@ -259,6 +250,11 @@ private: void connect (beast::IP::Endpoint const& remote_endpoint) override; + /* The number of active peers on the network + Active peers are only those peers that have completed the handshake + and are running the Ripple protocol. + */ + // VFALCO Why private? std::size_t size() override; diff --git a/src/ripple/overlay/impl/ProtocolMessage.h b/src/ripple/overlay/impl/ProtocolMessage.h index dde1a6c2b..12a69af3d 100644 --- a/src/ripple/overlay/impl/ProtocolMessage.h +++ b/src/ripple/overlay/impl/ProtocolMessage.h @@ -144,15 +144,15 @@ write (Streambuf& streambuf, { auto const size = m.ByteSize(); std::array v; - v[0] = static_cast ((size >> 24) & 0xFF); - v[1] = static_cast ((size >> 16) & 0xFF); - v[2] = static_cast ((size >> 8) & 0xFF); - v[3] = static_cast ( size & 0xFF); - v[4] = static_cast ((type >> 8) & 0xFF); - v[5] = static_cast ( type & 0xFF); - + v[0] = static_cast((size >> 24) & 0xFF); + v[1] = static_cast((size >> 16) & 0xFF); + v[2] = static_cast((size >> 8) & 0xFF); + v[3] = static_cast( size & 0xFF); + v[4] = static_cast((type >> 8) & 0xFF); + v[5] = static_cast( type & 0xFF); streambuf.commit(boost::asio::buffer_copy( - streambuf.prepare(Message::kHeaderBytes), boost::asio::buffer(v))); + streambuf.prepare(Message::kHeaderBytes), + boost::asio::buffer(v))); ZeroCopyOutputStream stream ( streambuf, blockBytes); m.SerializeToZeroCopyStream(&stream); diff --git a/src/ripple/protocol/RippleAddress.h b/src/ripple/protocol/RippleAddress.h index 8f9ff3180..3b30cb0b9 100644 --- a/src/ripple/protocol/RippleAddress.h +++ b/src/ripple/protocol/RippleAddress.h @@ -32,6 +32,18 @@ namespace ripple { +enum VersionEncoding +{ + VER_NONE = 1, + VER_NODE_PUBLIC = 28, + VER_NODE_PRIVATE = 32, + VER_ACCOUNT_ID = 0, + VER_ACCOUNT_PUBLIC = 35, + VER_ACCOUNT_PRIVATE = 34, + VER_FAMILY_GENERATOR = 41, + VER_FAMILY_SEED = 33, +}; + // // Used to hold addresses and parse and produce human formats. // @@ -40,18 +52,6 @@ namespace ripple { class RippleAddress : private CBase58Data { private: - enum VersionEncoding - { - VER_NONE = 1, - VER_NODE_PUBLIC = 28, - VER_NODE_PRIVATE = 32, - VER_ACCOUNT_ID = 0, - VER_ACCOUNT_PUBLIC = 35, - VER_ACCOUNT_PRIVATE = 34, - VER_FAMILY_GENERATOR = 41, - VER_FAMILY_SEED = 33, - }; - bool mIsValid; public: diff --git a/src/ripple/protocol/impl/HashPrefix.cpp b/src/ripple/protocol/impl/HashPrefix.cpp index b107420b7..5443918fa 100644 --- a/src/ripple/protocol/impl/HashPrefix.cpp +++ b/src/ripple/protocol/impl/HashPrefix.cpp @@ -22,8 +22,8 @@ namespace ripple { -// The prefix codes are part of the Ripple protocol and existing codes cannot be -// arbitrarily changed. +// The prefix codes are part of the Ripple protocol +// and existing codes cannot be arbitrarily changed. HashPrefix const HashPrefix::transactionID ('T', 'X', 'N'); HashPrefix const HashPrefix::txNode ('S', 'N', 'D');