From 347b903393f517f021d34f791ed48cb9f0890b50 Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Fri, 15 Jun 2012 16:09:17 -0700 Subject: [PATCH 01/19] Cosmetic changes. --- database/SqliteDatabase.h | 3 ++- src/ConnectionPool.cpp | 11 ++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/database/SqliteDatabase.h b/database/SqliteDatabase.h index 7cd94f45e..8ba9d796b 100644 --- a/database/SqliteDatabase.h +++ b/database/SqliteDatabase.h @@ -40,5 +40,6 @@ public: uint64 getBigInt(int colIndex); void escape(const unsigned char* start,int size,std::string& retStr); - }; + +// vim:ts=4 diff --git a/src/ConnectionPool.cpp b/src/ConnectionPool.cpp index 931fe93e2..728795c41 100644 --- a/src/ConnectionPool.cpp +++ b/src/ConnectionPool.cpp @@ -81,26 +81,27 @@ bool ConnectionPool::savePeer(const std::string& strIp, int iPort,char code) return false; } +// <-- true, if a peer is available to connect to bool ConnectionPool::peerAvailable(std::string& strIp, int& iPort) { Database* db = theApp->getWalletDB()->getDB(); std::vector vstrIpPort; + // Convert mIpMap (list of open connections) to a vector of " ". { boost::mutex::scoped_lock sl(mPeerLock); - pipPeer ipPeer; - vstrIpPort.reserve(mIpMap.size()); - BOOST_FOREACH(ipPeer, mIpMap) + BOOST_FOREACH(pipPeer ipPeer, mIpMap) { - std::string& strIp = ipPeer.first.first; - int iPort = ipPeer.first.second; + const std::string& strIp = ipPeer.first.first; + int iPort = ipPeer.first.second; vstrIpPort.push_back(db->escape(str(boost::format("%s %d") % strIp % iPort))); } } + // Get the first IpPort entry which is not in vector and which is not scheduled for scanning. std::string strIpPort; ScopedLock sl(theApp->getWalletDB()->getDBLock()); From 3ea3feb1c171b06904afa332b1f14b6b75a10c8e Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Fri, 15 Jun 2012 16:09:33 -0700 Subject: [PATCH 02/19] Add sqlEscape to utils.h. --- src/utils.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/utils.h b/src/utils.h index 2fa78e09d..cb4a1706f 100644 --- a/src/utils.h +++ b/src/utils.h @@ -82,6 +82,11 @@ inline std::string strHex(const uint64 uiHost) return strHex((unsigned char*) &uBig, sizeof(uBig)); } +inline static std::string sqlEscape(const std::string& strSrc) +{ + return str(boost::format("X'%s'") % strHex(strSrc)); +} + template bool isZero(Iterator first, int iSize) { From eb48ea77d7f32397004d736f763a85d6bfa1f2a8 Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Fri, 15 Jun 2012 20:11:43 -0700 Subject: [PATCH 03/19] Add getStrBinary to database functions. --- database/database.cpp | 81 ++++++++++++++++++++++++++----------------- database/database.h | 8 +++-- 2 files changed, 56 insertions(+), 33 deletions(-) diff --git a/database/database.cpp b/database/database.cpp index f0148e594..725af6a2f 100644 --- a/database/database.cpp +++ b/database/database.cpp @@ -5,9 +5,9 @@ Database::Database(const char* host,const char* user,const char* pass) : mNumCol(0) { - mDBPass=pass; - mHost=host; - mUser=user; + mDBPass = pass; + mHost = host; + mUser = user; } Database::~Database() @@ -17,68 +17,80 @@ Database::~Database() bool Database::getNull(const char* colName) { int index; - if(getColNumber(colName,&index)) + + if (getColNumber(colName,&index)) { - return(getNull(index)); + return getNull(index); } + return true; } char* Database::getStr(const char* colName,std::string& retStr) { int index; - if(getColNumber(colName,&index)) + + if (getColNumber(colName,&index)) { - return(getStr(index,retStr)); + return getStr(index,retStr); } - return(NULL); + + return NULL; } int32 Database::getInt(const char* colName) { int index; - if(getColNumber(colName,&index)) + + if (getColNumber(colName,&index)) { - return(getInt(index)); + return getInt(index); } - return(0); + + return 0; } float Database::getFloat(const char* colName) { int index; - if(getColNumber(colName,&index)) + + if (getColNumber(colName,&index)) { - return(getFloat(index)); + return getFloat(index); } - return(0); + + return 0; } bool Database::getBool(const char* colName) { int index; - if(getColNumber(colName,&index)) + + if (getColNumber(colName,&index)) { - return(getBool(index)); + return getBool(index); } - return(0); + + return 0; } int Database::getBinary(const char* colName,unsigned char* buf,int maxSize) { int index; - if(getColNumber(colName,&index)) + + if (getColNumber(colName,&index)) { return(getBinary(index,buf,maxSize)); } + return(0); } -std::vector Database::getBinary(const char* colName) +std::vector Database::getBinary(const std::string& strColName) { int index; - if (getColNumber(colName,&index)) + if (getColNumber(strColName.c_str(), &index)) { return getBinary(index); } @@ -86,37 +98,44 @@ std::vector Database::getBinary(const char* colName) return std::vector(); } +std::string Database::getStrBinary(const std::string& strColName) +{ + // YYY Could eliminate a copy if getStrBinary was a template. + return strCopy(getBinary(strColName.c_str())); +} + uint64 Database::getBigInt(const char* colName) { int index; - if(getColNumber(colName,&index)) + + if (getColNumber(colName,&index)) { - return(getBigInt(index)); + return getBigInt(index); } - return(0); + + return 0; } - - // returns false if can't find col bool Database::getColNumber(const char* colName,int* retIndex) { - for(unsigned int n=0; n #include #include "../src/types.h" +#include "../src/utils.h" #define SQL_FOREACH(_db, _strQuery) \ if ((_db)->executeSQL(_strQuery)) \ @@ -58,12 +59,15 @@ public: // get Data from the current row bool getNull(const char* colName); char* getStr(const char* colName,std::string& retStr); + std::string getStrBinary(const std::string& strColName); int32 getInt(const char* colName); float getFloat(const char* colName); bool getBool(const char* colName); + // returns amount stored in buf - int getBinary(const char* colName,unsigned char* buf,int maxSize); - std::vector getBinary(const char* colName); + int getBinary(const char* colName, unsigned char* buf, int maxSize); + std::vector getBinary(const std::string& strColName); + uint64 getBigInt(const char* colName); virtual bool getNull(int colIndex)=0; From d1cc5c0b8d6ec0f065746b50514e7f30c1358e77 Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Fri, 15 Jun 2012 20:12:11 -0700 Subject: [PATCH 04/19] Fix documentation for unl_delete. --- src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.cpp b/src/main.cpp index 7c1ee5374..e60720feb 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -59,7 +59,7 @@ void printHelp(const po::options_description& desc) cout << " transit_set " << endl; cout << " tx " << endl; cout << " unl_add | []" << endl; - cout << " unl_delete " << endl; + cout << " unl_delete |" << endl; cout << " unl_list" << endl; cout << " unl_reset" << endl; cout << " validation_create [||]" << endl; From e54c5603f675e5d17a785ff688e9d7daea3116a3 Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Fri, 15 Jun 2012 20:12:39 -0700 Subject: [PATCH 05/19] Add Score for SeedNodes table. --- src/DBInit.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/DBInit.cpp b/src/DBInit.cpp index fc4a1d6d4..7406ce2ce 100644 --- a/src/DBInit.cpp +++ b/src/DBInit.cpp @@ -117,6 +117,11 @@ const char *WalletDBInit[] = { // Table of PublicKeys user has asked to trust. // Fetches are made to the CAS. This gets the newcoin.txt so even validators without a web server can publish a newcoin.txt. + // Source: + // 'M' = Manually added. : 1500 + // 'V' = validators.txt : 1000 + // 'W' = Web browsing. : 200 + // 'R' = Referral : 0 // Next: // Time of next fetch attempt. // Scan: @@ -129,6 +134,7 @@ const char *WalletDBInit[] = { // User supplied comment. "CREATE TABLE SeedNodes ( \ PublicKey CHARACTER(53) PRIMARY KEY NOT NULL, \ + Source CHARACTER(1) NOT NULL, \ Next DATETIME, \ Scan DATETIME, \ Fetch DATETIME, \ @@ -139,7 +145,7 @@ const char *WalletDBInit[] = { // Allow us to easily find the next SeedNode to fetch. "CREATE INDEX SeedNodeNext ON SeedNodes (Next);", - // Nodes we trust not grossly consipire. Derived from SeedDomains, SeedNodes, and ValidatorReferrals. + // Nodes we trust to not grossly collude against us. Derived from SeedDomains, SeedNodes, and ValidatorReferrals. // // Score: // Computed trust score. Higher is better. From 86d175bfbda0d1374b893110890406a90268209f Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Fri, 15 Jun 2012 20:13:19 -0700 Subject: [PATCH 06/19] Implement initial support specify validators as public keys. --- src/RPCServer.cpp | 22 +-- src/UniqueNodeList.cpp | 404 +++++++++++++++++++++++++++-------------- src/UniqueNodeList.h | 29 ++- 3 files changed, 299 insertions(+), 156 deletions(-) diff --git a/src/RPCServer.cpp b/src/RPCServer.cpp index 2e8fbdc18..c1f59e5a1 100644 --- a/src/RPCServer.cpp +++ b/src/RPCServer.cpp @@ -1460,11 +1460,11 @@ Json::Value RPCServer::doUnlAdd(Json::Value& params) std::string strNode = params[0u].asString(); std::string strComment = (params.size() == 2) ? params[1u].asString() : ""; - NewcoinAddress nodePublic; + NewcoinAddress naNodePublic; - if (nodePublic.setNodePublic(strNode)) + if (naNodePublic.setNodePublic(strNode)) { - theApp->getUNL().nodeAddPublic(nodePublic, strComment); + theApp->getUNL().nodeAddPublic(naNodePublic, UniqueNodeList::vsManual, strComment); return "adding node by public key"; } @@ -1474,8 +1474,6 @@ Json::Value RPCServer::doUnlAdd(Json::Value& params) return "adding node by domain"; } - - return "invalid params"; } // validation_create [||] @@ -1973,22 +1971,24 @@ Json::Value RPCServer::doUnlDefault(Json::Value& params) { return RPCError(rpcINVALID_PARAMS); } -// unl_delete +// unl_delete | Json::Value RPCServer::doUnlDelete(Json::Value& params) { - std::string strNodePublic = params[0u].asString(); + std::string strNode = params[0u].asString(); NewcoinAddress naNodePublic; - if (naNodePublic.setNodePublic(strNodePublic)) + if (naNodePublic.setNodePublic(strNode)) { - theApp->getUNL().nodeRemove(naNodePublic); + theApp->getUNL().nodeRemovePublic(naNodePublic); - return "removing node"; + return "removing node by public key"; } else { - return "invalid public key"; + theApp->getUNL().nodeRemoveDomain(strNode); + + return "removing node by domain"; } } diff --git a/src/UniqueNodeList.cpp b/src/UniqueNodeList.cpp index 50a73df50..1427e3db8 100644 --- a/src/UniqueNodeList.cpp +++ b/src/UniqueNodeList.cpp @@ -9,6 +9,7 @@ #include "UniqueNodeList.h" #include "utils.h" +#include #include #include #include @@ -26,6 +27,7 @@ #define SECTION_VALIDATORS_URL "validators_url" // Limit pollution of database. +// YYY Move to config file. #define REFERRAL_VALIDATORS_MAX 50 #define REFERRAL_IPS_MAX 50 @@ -90,13 +92,10 @@ void UniqueNodeList::trustedLoad() mUNL.clear(); + // XXX Needs to limit by quanity and quality. SQL_FOREACH(db, "SELECT PublicKey FROM TrustedNodes WHERE Score != 0;") { - std::string strPublicKey; - - db->getStr("PublicKey", strPublicKey); - - mUNL.insert(strPublicKey); + mUNL.insert(db->getStrBinary("PublicKey")); } } @@ -169,12 +168,14 @@ void UniqueNodeList::scoreCompute() { strIndex umPulicIdx; // Map of public key to index. strIndex umDomainIdx; // Map of domain to index. - std::vector vsnNodes; + std::vector vsnNodes; // Index to scoring node. Database* db=theApp->getWalletDB()->getDB(); std::string strSql; + // For each entry in SeedDomains with a PublicKey: + // - Add an entry in umPulicIdx, umDomainIdx, and vsnNodes. { ScopedLock sl(theApp->getWalletDB()->getDBLock()); @@ -186,13 +187,9 @@ void UniqueNodeList::scoreCompute() } else { - std::string strDomain; - std::string strPublicKey; - std::string strSource; - - db->getStr("Domain", strDomain); - db->getStr("PublicKey", strPublicKey); - db->getStr("Source", strSource); + std::string strDomain = db->getStrBinary("Domain"); + std::string strPublicKey = db->getStrBinary("PublicKey"); + std::string strSource = db->getStrBinary("Source"); int iNode = vsnNodes.size(); @@ -212,6 +209,7 @@ void UniqueNodeList::scoreCompute() } } + // For debugging, print out initial scores. BOOST_FOREACH(scoreNode& sn, vsnNodes) { std::cerr << str(boost::format("%s| %d, %d, %d") @@ -227,6 +225,7 @@ void UniqueNodeList::scoreCompute() // std::cerr << str(boost::format("vsnNodes.size=%d") % vsnNodes.size()) << std::endl; // Step through growing list of nodes adding each validation list. + // - Each validator may have provided referals. Add those referals as validators. for (int iNode=0; iNode != vsnNodes.size(); iNode++) { scoreNode& sn = vsnNodes[iNode]; @@ -238,11 +237,9 @@ void UniqueNodeList::scoreCompute() SQL_FOREACH(db, str(boost::format("SELECT Referral FROM ValidatorReferrals WHERE Validator=%s ORDER BY Entry;") % db->escape(strValidator))) { - std::string strReferral; + std::string strReferral = db->getStrBinary("Referral"); int iReferral; - db->getStr("Referral", strReferral); - strIndex::iterator itEntry; NewcoinAddress na; @@ -335,11 +332,7 @@ void UniqueNodeList::scoreCompute() SQL_FOREACH(db, str(boost::format("SELECT PublicKey,Seen FROM TrustedNodes WHERE PublicKey IN (%s);") % strJoin(vstrPublicKeys.begin(), vstrPublicKeys.end(), ","))) { - std::string strPublicKey; - - db->getStr("PublicKey", strPublicKey); - - vsnNodes[umPulicIdx[strPublicKey]].iSeen = db->getNull("Seen") ? -1 : db->getInt("Seen"); + vsnNodes[umPulicIdx[db->getStrBinary("PublicKey")]].iSeen = db->getNull("Seen") ? -1 : db->getInt("Seen"); } } @@ -358,7 +351,7 @@ void UniqueNodeList::scoreCompute() std::string strSeen = sn.iSeen >= 0 ? str(boost::format("%d") % sn.iSeen) : "NULL"; vstrValues[iNode] = str(boost::format("(%s,%s,%s)") - % db->escape(sn.strValidator) + % sqlEscape(sn.strValidator) % sn.iScore % strSeen); @@ -372,6 +365,7 @@ void UniqueNodeList::scoreCompute() { ScopedLock sl(mUNLLock); + // XXX Should limit to scores above a certain minimum and limit to a certain number. mUNL.swap(usUNL); } @@ -387,11 +381,7 @@ void UniqueNodeList::scoreCompute() // For every IpReferral add a score for the IP and PORT. SQL_FOREACH(db, "SELECT Validator,COUNT(*) AS Count FROM IpReferrals GROUP BY Validator;") { - std::string strValidator; - - db->getStr("Validator", strValidator); - - umValidators[strValidator] = db->getInt("Count"); + umValidators[db->getStrBinary("Validator")] = db->getInt("Count"); // std::cerr << strValidator << ":" << db->getInt("Count") << std::endl; } @@ -418,13 +408,11 @@ void UniqueNodeList::scoreCompute() % db->escape(strValidator))) { score iPoints = iBase * (iEntries - iEntry) / iEntries; - std::string strIP; int iPort; - db->getStr("IP", strIP); iPort = db->getNull("Port") ? -1 : db->getInt("Port"); - std::pair< std::string, int> ep = std::make_pair(strIP, iPort); + std::pair< std::string, int> ep = std::make_pair(db->getStrBinary("IP"), iPort); epScore::iterator itEp = umScore.find(ep); @@ -491,6 +479,7 @@ void UniqueNodeList::scoreTimerHandler(const boost::system::error_code& err) } // Start a timer to update scores. +// <-- bNow: true, to force scoring for debugging. void UniqueNodeList::scoreNext(bool bNow) { // std::cerr << str(boost::format("scoreNext: mtpFetchUpdated=%s mtpScoreStart=%s mtpScoreUpdated=%s mtpScoreNext=%s") % mtpFetchUpdated % mtpScoreStart % mtpScoreUpdated % mtpScoreNext) << std::endl; @@ -534,6 +523,7 @@ void UniqueNodeList::fetchFinish() fetchNext(); } +// Called when we need to update scores. void UniqueNodeList::fetchDirty() { // Note update. @@ -633,9 +623,8 @@ void UniqueNodeList::processIps(const std::string& strSite, const NewcoinAddress // Persist ValidatorReferrals. void UniqueNodeList::processValidators(const std::string& strSite, const std::string& strValidatorsSrc, const NewcoinAddress& naNodePublic, section::mapped_type* pmtVecStrValidators) { - Database* db=theApp->getWalletDB()->getDB(); - - std::string strEscNodePublic = db->escape(naNodePublic.humanNodePublic()); + Database* db = theApp->getWalletDB()->getDB(); + std::string strNodePublic = naNodePublic.humanNodePublic(); std::cerr << str(boost::format("Validator: '%s' : '%s' : processing %d validators.") @@ -647,7 +636,8 @@ void UniqueNodeList::processValidators(const std::string& strSite, const std::st // Remove all current Validator's entries in ValidatorReferrals { ScopedLock sl(theApp->getWalletDB()->getDBLock()); - db->executeSQL(str(boost::format("DELETE FROM ValidatorReferrals WHERE Validator=%s;") % strEscNodePublic)); + + db->executeSQL(str(boost::format("DELETE FROM ValidatorReferrals WHERE Validator='%s';") % strNodePublic)); // XXX Check result. } @@ -655,28 +645,53 @@ void UniqueNodeList::processValidators(const std::string& strSite, const std::st if (pmtVecStrValidators && pmtVecStrValidators->size()) { std::vector vstrValues; - vstrValues.resize(MIN(pmtVecStrValidators->size(), REFERRAL_VALIDATORS_MAX)); + vstrValues.reserve(MIN(pmtVecStrValidators->size(), REFERRAL_VALIDATORS_MAX)); - int i = 0; + int iValues = 0; BOOST_FOREACH(std::string strReferral, *pmtVecStrValidators) { - if (i == REFERRAL_VALIDATORS_MAX) + if (iValues == REFERRAL_VALIDATORS_MAX) break; - vstrValues[i] = str(boost::format("(%s,%d,%s)") % strEscNodePublic % i % db->escape(strReferral)); - i++; + boost::smatch smMatch; + std::string strIP; - NewcoinAddress naValidator; + // domain comment? + // public_key comment? + static boost::regex reReferral("\\`\\s*(\\S+)(?:\\s+(\\d+))?\\s*\\'"); - if (naValidator.setNodePublic(strReferral)) + if (!boost::regex_match(strReferral, smMatch, reReferral)) { - // A public key. - // XXX Schedule for CAS lookup. + std::cerr + << str(boost::format("Validator: '%s' ["SECTION_VALIDATORS"]: rejecting '%s'") + % strSite % strReferral) + << std::endl; } else { - // A domain: need to look it up. - nodeAddDomain(strReferral, vsReferral); + std::string strRefered = smMatch[1]; + std::string strComment = smMatch[2]; + NewcoinAddress naValidator; + + if (naValidator.setNodePublic(strRefered)) + { + // A public key. + // XXX Schedule for CAS lookup. + nodeAddPublic(naValidator, vsReferral, strComment); + + vstrValues.push_back(str(boost::format("('%s',%d,'%s')") % strNodePublic % iValues % naValidator.humanNodePublic())); + } + else + { + // A domain: need to look it up. + boost::trim(strRefered); + boost::to_lower(strRefered); + nodeAddDomain(strRefered, vsReferral, strComment); + + vstrValues.push_back(str(boost::format("('%s',%d,%s)") % strNodePublic % iValues % sqlEscape(strRefered))); + } + + iValues++; } } @@ -707,7 +722,7 @@ void UniqueNodeList::responseIps(const std::string& strSite, const NewcoinAddres // Process section [ips_url]. // If we have a section with a single entry, fetch the url and process it. -void UniqueNodeList::getIpsUrl(NewcoinAddress naNodePublic, section secSite) +void UniqueNodeList::getIpsUrl(const NewcoinAddress& naNodePublic, section secSite) { std::string strIpsUrl; std::string strDomain; @@ -733,7 +748,7 @@ void UniqueNodeList::getIpsUrl(NewcoinAddress naNodePublic, section secSite) } // Given a section with validators, parse and persist it. -void UniqueNodeList::responseValidators(const std::string& strValidatorsUrl, NewcoinAddress naNodePublic, section secSite, const std::string& strSite, const boost::system::error_code& err, const std::string strValidatorsFile) +void UniqueNodeList::responseValidators(const std::string& strValidatorsUrl, const NewcoinAddress& naNodePublic, section secSite, const std::string& strSite, const boost::system::error_code& err, const std::string strValidatorsFile) { if (!err) { @@ -746,7 +761,7 @@ void UniqueNodeList::responseValidators(const std::string& strValidatorsUrl, New } // Process section [validators_url]. -void UniqueNodeList::getValidatorsUrl(NewcoinAddress naNodePublic, section secSite) +void UniqueNodeList::getValidatorsUrl(const NewcoinAddress& naNodePublic, section secSite) { std::string strValidatorsUrl; std::string strDomain; @@ -772,7 +787,7 @@ void UniqueNodeList::getValidatorsUrl(NewcoinAddress naNodePublic, section secSi } // Process a newcoin.txt. -void UniqueNodeList::processFile(const std::string strDomain, NewcoinAddress naNodePublic, section secSite) +void UniqueNodeList::processFile(const std::string strDomain, const NewcoinAddress& naNodePublic, section secSite) { // // Process Validators @@ -981,7 +996,7 @@ void UniqueNodeList::fetchNext() tpNow = boost::posix_time::second_clock::universal_time(); std::cerr << str(boost::format("fetchNext: iNext=%s tpNext=%s tpNow=%s") % iNext % tpNext % tpNow) << std::endl; - db->getStr("Domain", strDomain); + strDomain = db->getStrBinary("Domain"); db->endIterRows(); } @@ -1057,48 +1072,7 @@ int UniqueNodeList::iSourceScore(validatorSource vsWhy) return iScore; } -// Queue a domain for a single attempt fetch a newcoin.txt. -// --> strComment: only used on vsManual -// YYY As a lot of these may happen at once, would be nice to wrap multiple calls in a transaction. -void UniqueNodeList::nodeAddDomain(const std::string& strDomain, validatorSource vsWhy, std::string strComment) -{ - // YYY Would be best to verify strDomain is a valid domain. - // std::cerr << str(boost::format("nodeAddDomain: '%s' %c '%s'") - // % strDomain - // % vsWhy - // % strComment) << std::endl; - - seedDomain sdCurrent; - - bool bFound = getSeedDomains(strDomain, sdCurrent); - bool bChanged = false; - - if (!bFound) - { - sdCurrent.strDomain = strDomain; - sdCurrent.tpNext = boost::posix_time::second_clock::universal_time(); - } - - // Promote source, if needed. - if (!bFound || iSourceScore(vsWhy) >= iSourceScore(sdCurrent.vsSource)) - { - sdCurrent.vsSource = vsWhy; - bChanged = true; - } - - if (vsManual == vsWhy) - { - // A manual add forces immediate scan. - sdCurrent.tpNext = boost::posix_time::second_clock::universal_time(); - sdCurrent.strComment = strComment; - bChanged = true; - } - - if (bChanged) - setSeedDomains(sdCurrent, true); -} - -// Retrieve a SeedDomain for DB. +// Retrieve a SeedDomain from DB. bool UniqueNodeList::getSeedDomains(const std::string& strDomain, seedDomain& dstSeedDomain) { bool bResult; @@ -1113,13 +1087,12 @@ bool UniqueNodeList::getSeedDomains(const std::string& strDomain, seedDomain& ds if (bResult) { std::string strPublicKey; - std::string strSource; int iNext; int iScan; int iFetch; std::string strSha256; - db->getStr("Domain", dstSeedDomain.strDomain); + dstSeedDomain.strDomain = db->getStrBinary("Domain"); if (!db->getNull("PublicKey") && db->getStr("PublicKey", strPublicKey)) { @@ -1130,14 +1103,16 @@ bool UniqueNodeList::getSeedDomains(const std::string& strDomain, seedDomain& ds dstSeedDomain.naPublicKey.clear(); } - db->getStr("Source", strSource); + std::string strSource = db->getStrBinary("Source"); dstSeedDomain.vsSource = static_cast(strSource[0]); + iNext = db->getInt("Next"); dstSeedDomain.tpNext = ptFromSeconds(iNext); iScan = db->getInt("Scan"); dstSeedDomain.tpScan = ptFromSeconds(iScan); iFetch = db->getInt("Fetch"); dstSeedDomain.tpFetch = ptFromSeconds(iFetch); + if (!db->getNull("Sha256") && db->getStr("Sha256", strSha256)) { dstSeedDomain.iSha256.SetHex(strSha256); @@ -1146,7 +1121,7 @@ bool UniqueNodeList::getSeedDomains(const std::string& strDomain, seedDomain& ds { dstSeedDomain.iSha256.zero(); } - db->getStr("Comment", dstSeedDomain.strComment); + dstSeedDomain.strComment = db->getStrBinary("Comment"); db->endIterRows(); } @@ -1191,68 +1166,227 @@ void UniqueNodeList::setSeedDomains(const seedDomain& sdSource, bool bNext) } } -// Add a trusted node. Called by RPC or other source. -// XXX Broken should operate on seeds. -void UniqueNodeList::nodeAddPublic(const NewcoinAddress& naNodePublic, const std::string& strComment) +// Queue a domain for a single attempt fetch a newcoin.txt. +// --> strComment: only used on vsManual +// YYY As a lot of these may happen at once, would be nice to wrap multiple calls in a transaction. +void UniqueNodeList::nodeAddDomain(std::string strDomain, validatorSource vsWhy, const std::string& strComment) { - std::string strPublicKey = naNodePublic.humanNodePublic(); + boost::trim(strDomain); + boost::to_lower(strDomain); + // YYY Would be best to verify strDomain is a valid domain. + // std::cerr << str(boost::format("nodeAddDomain: '%s' %c '%s'") + // % strDomain + // % vsWhy + // % strComment) << std::endl; + + seedDomain sdCurrent; + + bool bFound = getSeedDomains(strDomain, sdCurrent); + bool bChanged = false; + + if (!bFound) { - Database* db=theApp->getWalletDB()->getDB(); - ScopedLock sl(theApp->getWalletDB()->getDBLock()); + sdCurrent.strDomain = strDomain; + sdCurrent.tpNext = boost::posix_time::second_clock::universal_time(); + } - if( db->executeSQL(str(boost::format("SELECT count(*) from TrustedNodes where PublicKey=%s;") % db->escape(strPublicKey))) && - db->startIterRows() && db->getInt(0)==1 ) - { // exists. update the comment - db->executeSQL(str(boost::format("UPDATE TrustedNodes set Comment=%s where PublicKey=%s;") % db->escape(strComment) % db->escape(strPublicKey) )); - }else - { // new node - db->executeSQL(str(boost::format("INSERT INTO TrustedNodes (PublicKey,Comment) values (%s,%s);") - % db->escape(strPublicKey) % db->escape(strComment))); + // Promote source, if needed. + if (!bFound || iSourceScore(vsWhy) >= iSourceScore(sdCurrent.vsSource)) + { + sdCurrent.vsSource = vsWhy; + bChanged = true; + } + + if (vsManual == vsWhy) + { + // A manual add forces immediate scan. + sdCurrent.tpNext = boost::posix_time::second_clock::universal_time(); + sdCurrent.strComment = strComment; + bChanged = true; + } + + if (bChanged) + setSeedDomains(sdCurrent, true); +} + +// Retrieve a SeedNode from DB. +bool UniqueNodeList::getSeedNodes(const NewcoinAddress& naNodePublic, seedNode& dstSeedNode) +{ + bool bResult; + Database* db=theApp->getWalletDB()->getDB(); + + std::string strSql = str(boost::format("SELECT * FROM SeedNodes WHERE PublicKey='%s';") + % naNodePublic.humanNodePublic()); + + ScopedLock sl(theApp->getWalletDB()->getDBLock()); + + bResult = db->executeSQL(strSql) && db->startIterRows(); + if (bResult) + { + std::string strPublicKey; + std::string strSource; + int iNext; + int iScan; + int iFetch; + std::string strSha256; + + if (!db->getNull("PublicKey") && db->getStr("PublicKey", strPublicKey)) + { + dstSeedNode.naPublicKey.setNodePublic(strPublicKey); + } + else + { + dstSeedNode.naPublicKey.clear(); } - + strSource = db->getStrBinary("Source"); + dstSeedNode.vsSource = static_cast(strSource[0]); + + iNext = db->getInt("Next"); + dstSeedNode.tpNext = ptFromSeconds(iNext); + iScan = db->getInt("Scan"); + dstSeedNode.tpScan = ptFromSeconds(iScan); + iFetch = db->getInt("Fetch"); + dstSeedNode.tpFetch = ptFromSeconds(iFetch); + + if (!db->getNull("Sha256") && db->getStr("Sha256", strSha256)) + { + dstSeedNode.iSha256.SetHex(strSha256); + } + else + { + dstSeedNode.iSha256.zero(); + } + dstSeedNode.strComment = db->getStrBinary("Comment"); + + db->endIterRows(); } - { - ScopedLock slUNL(mUNLLock); - - mUNL.insert(strPublicKey); - } + return bResult; } -// XXX Broken should operate on seeds. -void UniqueNodeList::nodeRemove(NewcoinAddress naNodePublic) +// Persist a SeedNode. +// <-- bNext: true, to do fetching if needed. +void UniqueNodeList::setSeedNodes(const seedNode& snSource, bool bNext) { - std::string strPublicKey = naNodePublic.humanNodePublic(); + Database* db=theApp->getWalletDB()->getDB(); + + int iNext = iToSeconds(snSource.tpNext); + int iScan = iToSeconds(snSource.tpScan); + int iFetch = iToSeconds(snSource.tpFetch); + + // std::cerr << str(boost::format("setSeedNodes: iNext=%s tpNext=%s") % iNext % sdSource.tpNext) << std::endl; + + assert(snSource.naPublicKey.isValid()); + + std::string strSql = str(boost::format("REPLACE INTO SeedNodes (PublicKey,Source,Next,Scan,Fetch,Sha256,Comment) VALUES ('%s', '%c', %d, %d, %d, '%s', %s);") + % snSource.naPublicKey.humanNodePublic() + % static_cast(snSource.vsSource) + % iNext + % iScan + % iFetch + % snSource.iSha256.GetHex() + % sqlEscape(snSource.strComment) + ); + + { + ScopedLock sl(theApp->getWalletDB()->getDBLock()); + + if (!db->executeSQL(strSql)) + { + // XXX Check result. + std::cerr << "setSeedNodes: failed." << std::endl; + } + } + +#if 0 + // YYY When we have a cas schedule lookups similar to this. + if (bNext && (mtpFetchNext.is_not_a_date_time() || mtpFetchNext > snSource.tpNext)) + { + // Schedule earlier wake up. + fetchNext(); + } +#else + fetchDirty(); +#endif +} + +// Add a trusted node. Called by RPC or other source. +void UniqueNodeList::nodeAddPublic(const NewcoinAddress& naNodePublic, validatorSource vsWhy, const std::string& strComment) +{ + seedNode snCurrent; + + bool bFound = getSeedNodes(naNodePublic, snCurrent); + bool bChanged = false; + + if (!bFound) + { + snCurrent.naPublicKey = naNodePublic; + snCurrent.tpNext = boost::posix_time::second_clock::universal_time(); + } + + // Promote source, if needed. + if (!bFound || iSourceScore(vsWhy) >= iSourceScore(snCurrent.vsSource)) + { + snCurrent.vsSource = vsWhy; + bChanged = true; + } + + if (vsManual == vsWhy) + { + // A manual add forces immediate scan. + snCurrent.tpNext = boost::posix_time::second_clock::universal_time(); + snCurrent.strComment = strComment; + bChanged = true; + } + + if (bChanged) + setSeedNodes(snCurrent, true); +} + +void UniqueNodeList::nodeRemovePublic(const NewcoinAddress& naNodePublic) +{ + { + Database* db=theApp->getWalletDB()->getDB(); + ScopedLock sl(theApp->getWalletDB()->getDBLock()); + + db->executeSQL(str(boost::format("DELETE FROM SeedNodes WHERE PublicKey=%s") % naNodePublic.humanNodePublic())); + } + + // YYY Only dirty on successful delete. + fetchDirty(); +} + +void UniqueNodeList::nodeRemoveDomain(std::string strDomain) +{ + boost::trim(strDomain); + boost::to_lower(strDomain); { Database* db=theApp->getWalletDB()->getDB(); ScopedLock sl(theApp->getWalletDB()->getDBLock()); - db->executeSQL(str(boost::format("DELETE FROM TrustedNodes where PublicKey=%s") % db->escape(strPublicKey))); + db->executeSQL(str(boost::format("DELETE FROM SeedDomains WHERE Domain=%s") % sqlEscape(strDomain))); } - { - ScopedLock slUNL(mUNLLock); - mUNL.erase(strPublicKey); - } + // YYY Only dirty on successful delete. + fetchDirty(); } -// XXX Broken should operate on seeds. void UniqueNodeList::nodeReset() { { Database* db=theApp->getWalletDB()->getDB(); ScopedLock sl(theApp->getWalletDB()->getDBLock()); - db->executeSQL("DELETE FROM TrustedNodes"); - } - { - ScopedLock slUNL(mUNLLock); - mUNL.clear(); + // XXX Check results. + db->executeSQL("DELETE FROM SeedDomains"); + db->executeSQL("DELETE FROM SeedNodes"); } + + fetchDirty(); } Json::Value UniqueNodeList::getUnlJson() @@ -1264,16 +1398,10 @@ Json::Value UniqueNodeList::getUnlJson() ScopedLock sl(theApp->getWalletDB()->getDBLock()); SQL_FOREACH(db, "SELECT * FROM TrustedNodes;") { - std::string strPublicKey; - std::string strComment; - - db->getStr("PublicKey", strPublicKey); - db->getStr("Comment", strComment); - Json::Value node(Json::objectValue); - node["publicKey"] = strPublicKey; - node["comment"] = strComment; + node["publicKey"] = db->getStrBinary("PublicKey"); + node["comment"] = db->getStrBinary("Comment"); ret.append(node); } diff --git a/src/UniqueNodeList.h b/src/UniqueNodeList.h index 40c6d9984..622fd589c 100644 --- a/src/UniqueNodeList.h +++ b/src/UniqueNodeList.h @@ -47,6 +47,7 @@ private: boost::recursive_mutex mUNLLock; // XXX Make this faster, make this the contents vector unsigned char or raw public key. + // XXX Contents needs to based on score. boost::unordered_set mUNL; bool miscLoad(); @@ -63,6 +64,16 @@ private: std::string strComment; } seedDomain; + typedef struct { + NewcoinAddress naPublicKey; + validatorSource vsSource; + boost::posix_time::ptime tpNext; + boost::posix_time::ptime tpScan; + boost::posix_time::ptime tpFetch; + uint256 iSha256; + std::string strComment; + } seedNode; + // Used to distribute scores. typedef struct { int iScore; @@ -104,28 +115,32 @@ private: void fetchProcess(std::string strDomain); void fetchTimerHandler(const boost::system::error_code& err); - void getValidatorsUrl(NewcoinAddress naNodePublic, section secSite); - void getIpsUrl(NewcoinAddress naNodePublic, section secSite); + void getValidatorsUrl(const NewcoinAddress& naNodePublic, section secSite); + void getIpsUrl(const NewcoinAddress& naNodePublic, section secSite); void responseIps(const std::string& strSite, const NewcoinAddress& naNodePublic, const boost::system::error_code& err, const std::string strIpsFile); - void responseValidators(const std::string& strValidatorsUrl, NewcoinAddress naNodePublic, section secSite, const std::string& strSite, const boost::system::error_code& err, const std::string strValidatorsFile); + void responseValidators(const std::string& strValidatorsUrl, const NewcoinAddress& naNodePublic, section secSite, const std::string& strSite, const boost::system::error_code& err, const std::string strValidatorsFile); void processIps(const std::string& strSite, const NewcoinAddress& naNodePublic, section::mapped_type* pmtVecStrIps); void processValidators(const std::string& strSite, const std::string& strValidatorsSrc, const NewcoinAddress& naNodePublic, section::mapped_type* pmtVecStrValidators); - void processFile(const std::string strDomain, NewcoinAddress naNodePublic, section secSite); + void processFile(const std::string strDomain, const NewcoinAddress& naNodePublic, section secSite); bool getSeedDomains(const std::string& strDomain, seedDomain& dstSeedDomain); void setSeedDomains(const seedDomain& dstSeedDomain, bool bNext); + bool getSeedNodes(const NewcoinAddress& naNodePublic, seedNode& dstSeedNode); + void setSeedNodes(const seedNode& snSource, bool bNext); + public: UniqueNodeList(boost::asio::io_service& io_service); // Begin processing. void start(); - void nodeAddPublic(const NewcoinAddress& naNodePublic, const std::string& strComment); - void nodeAddDomain(const std::string& strDomain, validatorSource vsWhy, std::string strComment=""); - void nodeRemove(NewcoinAddress naNodePublic); + void nodeAddPublic(const NewcoinAddress& naNodePublic, validatorSource vsWhy, const std::string& strComment); + void nodeAddDomain(std::string strDomain, validatorSource vsWhy, const std::string& strComment=""); + void nodeRemovePublic(const NewcoinAddress& naNodePublic); + void nodeRemoveDomain(std::string strDomain); void nodeDefault(const std::string& strValidators); void nodeReset(); From d3c1681c688f162cee8395f557b40bd64ede0c68 Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Sat, 16 Jun 2012 02:43:36 -0700 Subject: [PATCH 07/19] Add configuration support for UNL_DEFAULT. --- newcoind.cfg | 71 ++++++++++++++++++++++++++++++++++---------------- src/Config.cpp | 22 +++++++++------- src/Config.h | 1 + 3 files changed, 63 insertions(+), 31 deletions(-) diff --git a/newcoind.cfg b/newcoind.cfg index e34c6b249..e08702b46 100644 --- a/newcoind.cfg +++ b/newcoind.cfg @@ -1,46 +1,63 @@ # # Sample newcoind.cfg # -# This file is UTF-8 with Dos, UNIX, or Mac style end of lines. -# Blank lines and lines beginning with '#' are ignored. -# Undefined sections are reserved. -# No escapes are currently defined. +# This file should be named newcoind.cfg. This file is UTF-8 with Dos, UNIX, +# or Mac style end of lines. Blank lines and lines beginning with '#' are +# ignored. Undefined sections are reserved. No escapes are currently defined. # # When you launch newcoind, it will attempt to find this file. # -# You may specify the location of this file with --conf=. The base -# directory for other files will be the directory containing this file. +# --conf=: +# You may specify the location of this file with --conf=. The config +# directory is the directory containing this file. The data directory is a +# the subdirectory named "dbs". # -# Windows: -# This file is named newcoind.cfg. -# The base directory for this configuration file and other information is the -# same directory as the newcoind program. +# Windows and no --conf: +# The config directory is the same directory as the newcoind program. The +# data directory is a the subdirectory named "dbs". # -# Other OSes: -# This file may be named newcoind.cfg. The file will be looked for in the -# following order: +# Other OSes and no --conf: +# This file will be looked for in these places in the following order: # ./newcoind.cfg # $XDG_CONFIG_HOME/newcoin/newcoind.cfg # -# If newcoind.cfg, is found in the current working directory, the directory -# will be used as the base directory for other information. Otherwise, the -# base directory for data is: +# If newcoind.cfg, is found in the current working directory, the directory +# will be used as the config directory. The data directory is a the +# subdirectory named "dbs". +# +# Otherwise, the data directory data is: # $XDG_DATA_HOME/newcoin/ # # Note: $XDG_CONFIG_HOME defaults to $HOME/.config # $XDG_DATA_HOME defaults to $HOME/.local/share # -# To perform validation, one of these sections must be provided: -# [validation_key], [validation_password], or [validation_seed]. +# [unl_default]: +# Specifies how to bootstrap the UNL list. The UNL list is based on a +# validators.txt file and is maintained in the databases. When newcoind +# starts up, if the databases are missing or are obsolete due to an upgrade +# of newcoind, newcoind will reconstruct the UNL list as specified here. +# +# If this field is not present or empty, newcoind will look for a validators.txt in the +# config directory. If not found there, it will attempt to retrieve the file +# from the newcoin foundation's web site. +# +# Specify the file by specifying its full path. +# +# Examples: C:/home/johndoe/newcoin/newcoind.cfg +# /home/johndoe/newcoin/newcoind.cfg # # [peer_ip]: -# IP address or domain to bind to if allowing external connections from peers. +# IP address or domain to bind to allow external connections from peers. +# Defaults to not allow external connections from peers. +# +# Examples: 0.0.0.0 - Bind on all interfaces. # # [peer_port]: -# Port to bind to if allowing external connections from peers. +# Port to bind to allow external connections from peers. # # [rpc_ip]: -# IP address or domain to bind to if allowing insecure RPC connections. +# IP address or domain to bind to allow insecure RPC connections. +# Defaults to not allow RPC connections. # # [rpc_port]: # Port to bind to if allowing insecure RPC connections. @@ -48,14 +65,24 @@ # [rpc_allow_remote]: # 0 or 1. 0 only allows RPC connections from 127.0.0.1. [default 0] # +# [websocket_ip]: +# IP address or domain to bind to allow client connections. +# +# Examples: 0.0.0.0 - Bind on all interfaces. +# 127.0.0.1 - Bind on localhost interface. Only local programs may connect. +# +# [websocket_port]: +# Port to bind to allow client connections. +# # [validation_seed]: # To perform validation, this section should contain either a validation seed or key. # The validation seed is used to generate the validation public/private key pair. # To obtain a validation seed, use the validation_create command. +# # Examples: RASH BUSH MILK LOOK BAD BRIM AVID GAFF BAIT ROT POD LOVE # shfArahZT9Q9ckTf3s1psJ7C7qzVN - # + [peer_ip] 0.0.0.0 diff --git a/src/Config.cpp b/src/Config.cpp index b02c468e9..18a26737c 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -29,6 +29,7 @@ #define SECTION_FEE_NICKNAME_CREATE "fee_nickname_create" #define SECTION_FEE_DEFAULT "fee_default" #define SECTION_ACCOUNT_PROBE_MAX "account_probe_max" +#define SECTION_UNL_DEFAULT "unl_default" Config theConfig; @@ -169,7 +170,7 @@ void Config::load() (void) sectionSingleB(secConfig, SECTION_PEER_IP, PEER_IP); if (sectionSingleB(secConfig, SECTION_PEER_PORT, strTemp)) - PEER_PORT = boost::lexical_cast(strTemp); + PEER_PORT = boost::lexical_cast(strTemp); (void) sectionSingleB(secConfig, SECTION_RPC_IP, RPC_IP); @@ -177,7 +178,7 @@ void Config::load() RPC_PORT = boost::lexical_cast(strTemp); if (sectionSingleB(secConfig, SECTION_RPC_ALLOW_REMOTE, strTemp)) - RPC_ALLOW_REMOTE = boost::lexical_cast(strTemp); + RPC_ALLOW_REMOTE = boost::lexical_cast(strTemp); if (sectionSingleB(secConfig, SECTION_VALIDATION_SEED, strTemp)) VALIDATION_SEED.setSeedGeneric(strTemp); @@ -187,28 +188,31 @@ void Config::load() PEER_SCAN_INTERVAL_MIN = MAX(60, boost::lexical_cast(strTemp)); if (sectionSingleB(secConfig, SECTION_PEER_START_MAX, strTemp)) - PEER_START_MAX = MAX(1, boost::lexical_cast(strTemp)); + PEER_START_MAX = MAX(1, boost::lexical_cast(strTemp)); if (sectionSingleB(secConfig, SECTION_PEER_CONNECT_LOW_WATER, strTemp)) PEER_CONNECT_LOW_WATER = MAX(1, boost::lexical_cast(strTemp)); if (sectionSingleB(secConfig, SECTION_NETWORK_QUORUM, strTemp)) - NETWORK_QUORUM = MAX(0, boost::lexical_cast(strTemp)); + NETWORK_QUORUM = MAX(0, boost::lexical_cast(strTemp)); if (sectionSingleB(secConfig, SECTION_VALIDATION_QUORUM, strTemp)) - VALIDATION_QUORUM = MAX(0, boost::lexical_cast(strTemp)); + VALIDATION_QUORUM = MAX(0, boost::lexical_cast(strTemp)); if (sectionSingleB(secConfig, SECTION_FEE_ACCOUNT_CREATE, strTemp)) - FEE_ACCOUNT_CREATE = boost::lexical_cast(strTemp); + FEE_ACCOUNT_CREATE = boost::lexical_cast(strTemp); if (sectionSingleB(secConfig, SECTION_FEE_NICKNAME_CREATE, strTemp)) - FEE_NICKNAME_CREATE = boost::lexical_cast(strTemp); + FEE_NICKNAME_CREATE = boost::lexical_cast(strTemp); if (sectionSingleB(secConfig, SECTION_FEE_DEFAULT, strTemp)) - FEE_DEFAULT = boost::lexical_cast(strTemp); + FEE_DEFAULT = boost::lexical_cast(strTemp); if (sectionSingleB(secConfig, SECTION_ACCOUNT_PROBE_MAX, strTemp)) - ACCOUNT_PROBE_MAX = boost::lexical_cast(strTemp); + ACCOUNT_PROBE_MAX = boost::lexical_cast(strTemp); + + if (sectionSingleB(secConfig, SECTION_UNL_DEFAULT, strTemp)) + UNL_DEFAULT = strTemp; } } } diff --git a/src/Config.h b/src/Config.h index 4ef20c81b..6a77fde1f 100644 --- a/src/Config.h +++ b/src/Config.h @@ -44,6 +44,7 @@ public: boost::filesystem::path CONFIG_FILE; boost::filesystem::path CONFIG_DIR; boost::filesystem::path DATA_DIR; + boost::filesystem::path UNL_DEFAULT; // Network parameters int NETWORK_START_TIME; // The Unix time we start ledger 0 From 9964689f8fb88dd2f1c441a2a10c241d6c62ddf0 Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Sat, 16 Jun 2012 02:52:50 -0700 Subject: [PATCH 08/19] Add configuration support for WEBSOCKET_IP and WEBSOCKET_PORT. --- src/Config.cpp | 46 +++++++++++++++++++++++++++------------------- src/Config.h | 33 ++++++++++++++++----------------- 2 files changed, 43 insertions(+), 36 deletions(-) diff --git a/src/Config.cpp b/src/Config.cpp index 18a26737c..176bb2b52 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -3,33 +3,36 @@ #include "ParseSection.h" #include "utils.h" -#include -#include #include - -// Fees are in XNS raw. -#define DEFAULT_FEE_ACCOUNT_CREATE 1000 -#define DEFAULT_FEE_NICKNAME_CREATE 1000 -#define DEFAULT_FEE_DEFAULT 100 +#include +#include #define CONFIG_FILE_NAME SYSTEM_NAME "d.cfg" // newcoind.cfg + +#define SECTION_ACCOUNT_PROBE_MAX "account_probe_max" +#define SECTION_FEE_ACCOUNT_CREATE "fee_account_create" +#define SECTION_FEE_DEFAULT "fee_default" +#define SECTION_FEE_NICKNAME_CREATE "fee_nickname_create" +#define SECTION_NETWORK_QUORUM "network_quorum" +#define SECTION_PEER_CONNECT_LOW_WATER "peer_connect_low_water" #define SECTION_PEER_IP "peer_ip" #define SECTION_PEER_PORT "peer_port" +#define SECTION_PEER_SCAN_INTERVAL_MIN "peer_scan_interval_min" +#define SECTION_PEER_SSL_CIPHER_LIST "peer_ssl_cipher_list" +#define SECTION_PEER_START_MAX "peer_start_max" +#define SECTION_RPC_ALLOW_REMOTE "rpc_allow_remote" #define SECTION_RPC_IP "rpc_ip" #define SECTION_RPC_PORT "rpc_port" -#define SECTION_RPC_ALLOW_REMOTE "rpc_allow_remote" -#define SECTION_VALIDATION_SEED "validation_seed" -#define SECTION_PEER_SSL_CIPHER_LIST "peer_ssl_cipher_list" -#define SECTION_PEER_SCAN_INTERVAL_MIN "peer_scan_interval_min" -#define SECTION_PEER_START_MAX "peer_start_max" -#define SECTION_PEER_CONNECT_LOW_WATER "peer_connect_low_water" -#define SECTION_NETWORK_QUORUM "network_quorum" -#define SECTION_VALIDATION_QUORUM "validation_quorum" -#define SECTION_FEE_ACCOUNT_CREATE "fee_account_create" -#define SECTION_FEE_NICKNAME_CREATE "fee_nickname_create" -#define SECTION_FEE_DEFAULT "fee_default" -#define SECTION_ACCOUNT_PROBE_MAX "account_probe_max" #define SECTION_UNL_DEFAULT "unl_default" +#define SECTION_VALIDATION_QUORUM "validation_quorum" +#define SECTION_VALIDATION_SEED "validation_seed" +#define SECTION_WEBSOCKET_IP "websocket_ip" +#define SECTION_WEBSOCKET_PORT "websocket_port" + +// Fees are in XNB. +#define DEFAULT_FEE_ACCOUNT_CREATE 1000 +#define DEFAULT_FEE_NICKNAME_CREATE 1000 +#define DEFAULT_FEE_DEFAULT 100 Config theConfig; @@ -180,6 +183,11 @@ void Config::load() if (sectionSingleB(secConfig, SECTION_RPC_ALLOW_REMOTE, strTemp)) RPC_ALLOW_REMOTE = boost::lexical_cast(strTemp); + (void) sectionSingleB(secConfig, SECTION_WEBSOCKET_IP, WEBSOCKET_IP); + + if (sectionSingleB(secConfig, SECTION_WEBSOCKET_PORT, strTemp)) + WEBSOCKET_PORT = boost::lexical_cast(strTemp); + if (sectionSingleB(secConfig, SECTION_VALIDATION_SEED, strTemp)) VALIDATION_SEED.setSeedGeneric(strTemp); diff --git a/src/Config.h b/src/Config.h index 6a77fde1f..10d1a1ce1 100644 --- a/src/Config.h +++ b/src/Config.h @@ -47,30 +47,28 @@ public: boost::filesystem::path UNL_DEFAULT; // Network parameters - int NETWORK_START_TIME; // The Unix time we start ledger 0 - int TRANSACTION_FEE_BASE; - int LEDGER_SECONDS; - int LEDGER_PROPOSAL_DELAY_SECONDS; - int LEDGER_AVALANCHE_SECONDS; + int NETWORK_START_TIME; // The Unix time we start ledger 0 + int TRANSACTION_FEE_BASE; + int LEDGER_SECONDS; + int LEDGER_PROPOSAL_DELAY_SECONDS; + int LEDGER_AVALANCHE_SECONDS; // Note: The following parameters do not relate to the UNL or trust at all - int NETWORK_QUORUM; // Minimum number of nodes to consider the network present - int VALIDATION_QUORUM; // Minimum validations to consider ledger authoritative + int NETWORK_QUORUM; // Minimum number of nodes to consider the network present + int VALIDATION_QUORUM; // Minimum validations to consider ledger authoritative // Peer networking parameters std::string PEER_IP; - int PEER_PORT; - int NUMBER_CONNECTIONS; + int PEER_PORT; + int NUMBER_CONNECTIONS; std::string PEER_SSL_CIPHER_LIST; - int PEER_SCAN_INTERVAL_MIN; - int PEER_START_MAX; - int PEER_CONNECT_LOW_WATER; + int PEER_SCAN_INTERVAL_MIN; + int PEER_START_MAX; + int PEER_CONNECT_LOW_WATER; -// bool NODE_INBOUND; // We accept inbound connections -// bool NODE_DATABASE; // We offer historical data services -// bool NODE_PUBLIC; // We do not attempt to hide our identity -// bool NODE_DUMB; // We are a 'dumb' client -// bool NODE_SMART; // We offer services to 'dumb' clients + // Client networking parameters + std::string WEBSOCKET_IP; + int WEBSOCKET_PORT; // RPC parameters std::string RPC_IP; @@ -96,4 +94,5 @@ public: extern Config theConfig; #endif + // vim:ts=4 From f774baefebb91d35d6ebc08edb3f6fdb1abd0520 Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Sat, 16 Jun 2012 11:46:17 -0700 Subject: [PATCH 09/19] Change RPC unl_default to unl_load & unl_network and add UNL bootstrapping. --- src/Application.cpp | 17 +++--- src/RPCServer.cpp | 112 ++++++++++----------------------------- src/RPCServer.h | 10 ++-- src/UniqueNodeList.cpp | 117 +++++++++++++++++++++++++++++++++++++++++ src/UniqueNodeList.h | 6 +++ src/main.cpp | 2 + 6 files changed, 169 insertions(+), 95 deletions(-) diff --git a/src/Application.cpp b/src/Application.cpp index cf6809392..d451e9164 100644 --- a/src/Application.cpp +++ b/src/Application.cpp @@ -58,11 +58,11 @@ void Application::run() // // Construct databases. // - mTxnDB = new DatabaseCon("transaction.db", TxnDBInit, TxnDBCount); - mLedgerDB = new DatabaseCon("ledger.db", LedgerDBInit, LedgerDBCount); - mWalletDB = new DatabaseCon("wallet.db", WalletDBInit, WalletDBCount); - mHashNodeDB = new DatabaseCon("hashnode.db", HashNodeDBInit, HashNodeDBCount); - mNetNodeDB = new DatabaseCon("netnode.db", NetNodeDBInit, NetNodeDBCount); + mTxnDB = new DatabaseCon("transaction.db", TxnDBInit, TxnDBCount); + mLedgerDB = new DatabaseCon("ledger.db", LedgerDBInit, LedgerDBCount); + mWalletDB = new DatabaseCon("wallet.db", WalletDBInit, WalletDBCount); + mHashNodeDB = new DatabaseCon("hashnode.db", HashNodeDBInit, HashNodeDBCount); + mNetNodeDB = new DatabaseCon("netnode.db", NetNodeDBInit, NetNodeDBCount); // // Begin validation and ip maintenance. @@ -73,7 +73,7 @@ void Application::run() // // Allow peer connections. // - if(!theConfig.PEER_IP.empty() && theConfig.PEER_PORT) + if (!theConfig.PEER_IP.empty() && theConfig.PEER_PORT) { mPeerDoor = new PeerDoor(mIOService); } @@ -85,7 +85,7 @@ void Application::run() // // Allow RPC connections. // - if(!theConfig.RPC_IP.empty() && theConfig.RPC_PORT) + if (!theConfig.RPC_IP.empty() && theConfig.RPC_PORT) { mRPCDoor = new RPCDoor(mIOService); } @@ -122,7 +122,8 @@ void Application::run() mNetOps.setStateTimer(0); - // temporary + getUNL().nodeBootstrap(); + mIOService.run(); // This blocks std::cout << "Done." << std::endl; diff --git a/src/RPCServer.cpp b/src/RPCServer.cpp index c1f59e5a1..c2ded709b 100644 --- a/src/RPCServer.cpp +++ b/src/RPCServer.cpp @@ -1,5 +1,4 @@ -#include #include #include @@ -25,14 +24,6 @@ #include "NicknameState.h" #include "utils.h" -#define VALIDATORS_FETCH_SECONDS 30 -#define VALIDATORS_FILE_PATH "/" VALIDATORS_FILE_NAME -#define VALIDATORS_FILE_BYTES_MAX (50 << 10) - -/* -Just read from wire until the entire request is in. -*/ - RPCServer::RPCServer(boost::asio::io_service& io_service , NetworkOPs* nopNetwork) : mNetOps(nopNetwork), mSocket(io_service) { @@ -1902,77 +1893,8 @@ Json::Value RPCServer::doWalletSeed(Json::Value& params) } } -void RPCServer::validatorsResponse(const boost::system::error_code& err, std::string strResponse) -{ - std::cerr << "Fetch '" VALIDATORS_FILE_NAME "' complete." << std::endl; - - if (!err) - { - theApp->getUNL().nodeDefault(strResponse); - } - else - { - std::cerr << "Error: " << err.message() << std::endl; - } -} - -// Populate the UNL from a validators.txt file. -Json::Value RPCServer::doUnlDefault(Json::Value& params) { - if (!params.size() || (1==params.size() && !params[0u].compare("network"))) - { - bool bNetwork = 1 == params.size(); - std::string strValidators; - - if (!bNetwork) - { - std::ifstream ifsDefault(VALIDATORS_FILE_NAME, std::ios::in); - - if (!ifsDefault) - { - std::cerr << "Failed to open '" VALIDATORS_FILE_NAME "'." << std::endl; - - bNetwork = true; - } - else - { - strValidators.assign((std::istreambuf_iterator(ifsDefault)), - std::istreambuf_iterator()); - - if (ifsDefault.bad()) - { - std::cerr << "Failed to read '" VALIDATORS_FILE_NAME "'." << std::endl; - - bNetwork = true; - } - } - } - - if (bNetwork) - { - HttpsClient::httpsGet( - theApp->getIOService(), - VALIDATORS_SITE, - 443, - VALIDATORS_FILE_PATH, - VALIDATORS_FILE_BYTES_MAX, - boost::posix_time::seconds(VALIDATORS_FETCH_SECONDS), - boost::bind(&RPCServer::validatorsResponse, this, _1, _2)); - - return "fetching " VALIDATORS_FILE_NAME; - } - else - { - theApp->getUNL().nodeDefault(strValidators); - - return "processing " VALIDATORS_FILE_NAME; - } - } - else - return RPCError(rpcINVALID_PARAMS); -} - // unl_delete | -Json::Value RPCServer::doUnlDelete(Json::Value& params) +Json::Value RPCServer::doUnlDelete(Json::Value& params) { std::string strNode = params[0u].asString(); @@ -1992,22 +1914,45 @@ Json::Value RPCServer::doUnlDelete(Json::Value& params) } } -Json::Value RPCServer::doUnlList(Json::Value& params) +Json::Value RPCServer::doUnlList(Json::Value& params) { Json::Value obj(Json::objectValue); + obj["unl"]=theApp->getUNL().getUnlJson(); + return obj; } +// Populate the UNL from a local validators.txt file. +Json::Value RPCServer::doUnlLoad(Json::Value& params) +{ + if (!theApp->getUNL().nodeLoad()) + { + return RPCError(rpcLOAD_FAILED); + } + + return "loading"; +} + +// Populate the UNL from newcoin.org's validators.txt file. +Json::Value RPCServer::doUnlNetwork(Json::Value& params) +{ + theApp->getUNL().nodeNetwork(); + + return "fetching"; +} + // unl_reset -Json::Value RPCServer::doUnlReset(Json::Value& params) { +Json::Value RPCServer::doUnlReset(Json::Value& params) +{ theApp->getUNL().nodeReset(); return "removing nodes"; } // unl_score -Json::Value RPCServer::doUnlScore(Json::Value& params) { +Json::Value RPCServer::doUnlScore(Json::Value& params) +{ theApp->getUNL().nodeScore(); return "scoring requested"; @@ -2053,9 +1998,10 @@ Json::Value RPCServer::doCommand(const std::string& command, Json::Value& params { "tx", &RPCServer::doTx, 1, 1, }, { "unl_add", &RPCServer::doUnlAdd, 1, 2, }, - { "unl_default", &RPCServer::doUnlDefault, 0, 1, }, { "unl_delete", &RPCServer::doUnlDelete, 1, 1, }, { "unl_list", &RPCServer::doUnlList, 0, 0, }, + { "unl_load", &RPCServer::doUnlLoad, 0, 0, }, + { "unl_network", &RPCServer::doUnlNetwork, 0, 0, }, { "unl_reset", &RPCServer::doUnlReset, 0, 0, }, { "unl_score", &RPCServer::doUnlScore, 0, 0, }, diff --git a/src/RPCServer.h b/src/RPCServer.h index 4a3c6ece9..af9274e51 100644 --- a/src/RPCServer.h +++ b/src/RPCServer.h @@ -17,6 +17,9 @@ public: enum { rpcSUCCESS, + // Misc failure + rpcLOAD_FAILED, + // Networking rpcNO_CLOSED, rpcNO_CURRENT, @@ -40,7 +43,7 @@ public: rpcINVALID_PARAMS, rpcUNKNOWN_COMMAND, - // Bad paramater + // Bad parameter rpcACT_MALFORMED, rpcBAD_SEED, rpcDST_ACT_MALFORMED, @@ -131,10 +134,11 @@ private: Json::Value doTx(Json::Value& params); Json::Value doUnlAdd(Json::Value& params); - Json::Value doUnlDefault(Json::Value& params); Json::Value doUnlDelete(Json::Value& params); Json::Value doUnlFetch(Json::Value& params); Json::Value doUnlList(Json::Value& params); + Json::Value doUnlLoad(Json::Value& params); + Json::Value doUnlNetwork(Json::Value& params); Json::Value doUnlReset(Json::Value& params); Json::Value doUnlScore(Json::Value& params); @@ -151,8 +155,6 @@ private: Json::Value doWalletUnlock(Json::Value& params); Json::Value doWalletVerify(Json::Value& params); - void validatorsResponse(const boost::system::error_code& err, std::string strResponse); - public: typedef boost::shared_ptr pointer; diff --git a/src/UniqueNodeList.cpp b/src/UniqueNodeList.cpp index 1427e3db8..e52b4e504 100644 --- a/src/UniqueNodeList.cpp +++ b/src/UniqueNodeList.cpp @@ -17,6 +17,13 @@ #include #include +#include +#include + +#define VALIDATORS_FETCH_SECONDS 30 +#define VALIDATORS_FILE_PATH "/" VALIDATORS_FILE_NAME +#define VALIDATORS_FILE_BYTES_MAX (50 << 10) + // Gather string constants. #define SECTION_CURRENCIES "currencies" #define SECTION_DOMAIN "domain" @@ -1409,6 +1416,116 @@ Json::Value UniqueNodeList::getUnlJson() return ret; } +bool UniqueNodeList::nodeLoad() +{ + if (theConfig.UNL_DEFAULT.empty()) + { + std::cerr << "UNL_DEFAULT not specified." << std::endl; + + return false; + } + + if (!boost::filesystem::exists(theConfig.UNL_DEFAULT)) + { + std::cerr << str(boost::format("UNL_DEFAULT not found: '%s'") % theConfig.UNL_DEFAULT) << std::endl; + + return false; + } + + if (!boost::filesystem::is_regular_file(theConfig.UNL_DEFAULT)) + { + std::cerr << str(boost::format("UNL_DEFAULT not regular file: '%s'") % theConfig.UNL_DEFAULT) << std::endl; + + return false; + } + + std::ifstream ifsDefault(theConfig.UNL_DEFAULT.native().c_str(), std::ios::in); + + if (!ifsDefault) + { + std::cerr << str(boost::format("Failed to open: '%s'") % theConfig.UNL_DEFAULT) << std::endl; + + return false; + } + + std::string strValidators; + + strValidators.assign((std::istreambuf_iterator(ifsDefault)), + std::istreambuf_iterator()); + + if (ifsDefault.bad()) + { + std::cerr << str(boost::format("Failed to read: '%s'") % theConfig.UNL_DEFAULT) << std::endl; + + return false; + } + + nodeDefault(strValidators); + + std::cerr << str(boost::format("Processing: '%s'") % theConfig.UNL_DEFAULT) << std::endl; + + return true; +} + +void UniqueNodeList::validatorsResponse(const boost::system::error_code& err, std::string strResponse) +{ + std::cerr << "Fetch '" VALIDATORS_FILE_NAME "' complete." << std::endl; + + if (!err) + { + nodeDefault(strResponse); + } + else + { + std::cerr << "Error: " << err.message() << std::endl; + } +} + +void UniqueNodeList::nodeNetwork() +{ + HttpsClient::httpsGet( + theApp->getIOService(), + VALIDATORS_SITE, + 443, + VALIDATORS_FILE_PATH, + VALIDATORS_FILE_BYTES_MAX, + boost::posix_time::seconds(VALIDATORS_FETCH_SECONDS), + boost::bind(&UniqueNodeList::validatorsResponse, this, _1, _2)); +} + +void UniqueNodeList::nodeBootstrap() +{ + int iDomains = 0; + int iNodes = 0; + + { + Database* db=theApp->getWalletDB()->getDB(); + + ScopedLock sl(theApp->getWalletDB()->getDBLock()); + + if (db->executeSQL("SELECT COUNT(*) AS Count FROM SeedDomains;") && db->startIterRows()) + iDomains = db->getInt("Count"); + + if (db->executeSQL("SELECT COUNT(*) AS Count FROM SeedNodes;") && db->startIterRows()) + iNodes = db->getInt("Count"); + } + + bool bLoaded = iDomains || iNodes; + + if (!bLoaded && !theConfig.UNL_DEFAULT.empty()) + { + std::cerr << "Bootstrapping UNL: loading from file." << std::endl; + + bLoaded = nodeLoad(); + } + + if (!bLoaded) + { + std::cerr << "Bootstrapping UNL: loading from " VALIDATORS_SITE "." << std::endl; + nodeNetwork(); + } +} + // Process a validators.txt. // --> strValidators: a validators.txt void UniqueNodeList::nodeDefault(const std::string& strValidators) { diff --git a/src/UniqueNodeList.h b/src/UniqueNodeList.h index 622fd589c..d8c3853ca 100644 --- a/src/UniqueNodeList.h +++ b/src/UniqueNodeList.h @@ -131,6 +131,8 @@ private: bool getSeedNodes(const NewcoinAddress& naNodePublic, seedNode& dstSeedNode); void setSeedNodes(const seedNode& snSource, bool bNext); + void validatorsResponse(const boost::system::error_code& err, std::string strResponse); + public: UniqueNodeList(boost::asio::io_service& io_service); @@ -148,6 +150,10 @@ public: bool nodeInUNL(const NewcoinAddress& naNodePublic); + void nodeBootstrap(); + bool nodeLoad(); + void nodeNetwork(); + Json::Value getUnlJson(); }; diff --git a/src/main.cpp b/src/main.cpp index e60720feb..8ec61c211 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -61,6 +61,8 @@ void printHelp(const po::options_description& desc) cout << " unl_add | []" << endl; cout << " unl_delete |" << endl; cout << " unl_list" << endl; + cout << " unl_load" << endl; + cout << " unl_network" << endl; cout << " unl_reset" << endl; cout << " validation_create [||]" << endl; cout << " validation_seed [||]" << endl; From 160c625216b5f515dcafcb0d776779f2986acc97 Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Sat, 16 Jun 2012 12:17:40 -0700 Subject: [PATCH 10/19] Get rid of DH_der_gen_hex and DH_der_load_hex. --- src/Wallet.cpp | 27 ++++++++++----------------- src/utils.cpp | 25 +++++-------------------- src/utils.h | 4 +--- 3 files changed, 16 insertions(+), 40 deletions(-) diff --git a/src/Wallet.cpp b/src/Wallet.cpp index eaecc2f30..9f12140f0 100644 --- a/src/Wallet.cpp +++ b/src/Wallet.cpp @@ -53,13 +53,8 @@ bool Wallet::nodeIdentityLoad() mNodePublicKey.setNodePublic(strPublicKey); mNodePrivateKey.setNodePrivate(strPrivateKey); - std::string strDh512, strDh1024; - - db->getStr("Dh512", strDh512); - db->getStr("Dh1024", strDh1024); - - mDh512 = DH_der_load_hex(strDh512); - mDh1024 = DH_der_load_hex(strDh1024); + mDh512 = DH_der_load(db->getStrBinary("Dh512")); + mDh1024 = DH_der_load(db->getStrBinary("Dh1024")); db->endIterRows(); bSuccess = true; @@ -81,13 +76,11 @@ bool Wallet::nodeIdentityCreate() { // Make new key. - std::string strDh512, strDh1024; - - DH_der_gen_hex(strDh512, 512); // Using hex as db->escape in insufficient. + std::string strDh512 = DH_der_gen(512); #if 1 - strDh1024 = strDh512; // For testing and most cases 512 is fine. + std::string strDh1024 = strDh512; // For testing and most cases 512 is fine. #else - DH_der_gen_hex(strDh1024, 1024); + std::string strDh1024 = DH_der_gen(1024); #endif // @@ -96,11 +89,11 @@ bool Wallet::nodeIdentityCreate() { Database* db = theApp->getWalletDB()->getDB(); ScopedLock sl(theApp->getWalletDB()->getDBLock()); - db->executeSQL(str(boost::format("INSERT INTO NodeIdentity (PublicKey,PrivateKey,Dh512,Dh1024) VALUES (%s,%s,%s,%s);") - % db->escape(naNodePublic.humanNodePublic()) - % db->escape(naNodePrivate.humanNodePrivate()) - % db->escape(strDh512) - % db->escape(strDh1024))); + db->executeSQL(str(boost::format("INSERT INTO NodeIdentity (PublicKey,PrivateKey,Dh512,Dh1024) VALUES ('%s','%s',%s,%s);") + % naNodePublic.humanNodePublic() + % naNodePrivate.humanNodePrivate() + % sqlEscape(strDh512) + % sqlEscape(strDh1024))); // XXX Check error result. std::cerr << "NodeIdentity: Created." << std::endl; diff --git a/src/utils.cpp b/src/utils.cpp index 92f2379c2..d70caeb72 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -95,10 +95,11 @@ std::string strCopy(const std::vector& vucSrc) // DH support // -void DH_der_gen(std::string& strDer, int iKeyLength) +std::string DH_der_gen(int iKeyLength) { - DH* dh = 0; - int iCodes; + DH* dh = 0; + int iCodes; + std::string strDer; do { dh = DH_generate_parameters(iKeyLength, DH_GENERATOR_5, NULL, NULL); @@ -111,15 +112,8 @@ void DH_der_gen(std::string& strDer, int iKeyLength) unsigned char* next = reinterpret_cast(&strDer[0]); (void) i2d_DHparams(dh, &next); -} -void DH_der_gen_hex(std::string& strDer, int iKeyLength) -{ - std::string strBuf; - - DH_der_gen(strBuf, iKeyLength); - - strDer = strHex(strBuf); + return strDer; } DH* DH_der_load(const std::string& strDer) @@ -129,15 +123,6 @@ DH* DH_der_load(const std::string& strDer) return d2i_DHparams(NULL, &pbuf, strDer.size()); } -DH* DH_der_load_hex(const std::string& strDer) -{ - std::string strBuf; - - strUnHex(strBuf, strDer); - - return DH_der_load(strBuf); -} - /* void intIPtoStr(int ip,std::string& retStr) { diff --git a/src/utils.h b/src/utils.h index cb4a1706f..a069c96fc 100644 --- a/src/utils.h +++ b/src/utils.h @@ -105,9 +105,7 @@ std::vector strCopy(const std::string& strSrc); std::string strCopy(const std::vector& vucSrc); DH* DH_der_load(const std::string& strDer); -DH* DH_der_load_hex(const std::string& strDer); -void DH_der_gen(std::string& strDer, int iKeyLength); -void DH_der_gen_hex(std::string& strDer, int iKeyLength); +std::string DH_der_gen(int iKeyLength); inline std::string strGetEnv(const std::string& strKey) { From d6ca0389c9954d083e301da7bf471e34b674d8be Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Sat, 16 Jun 2012 12:18:02 -0700 Subject: [PATCH 11/19] Make UNL bootstrapping more robust. --- src/Application.cpp | 7 +++++-- src/UniqueNodeList.cpp | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Application.cpp b/src/Application.cpp index d451e9164..9e4cdd5ec 100644 --- a/src/Application.cpp +++ b/src/Application.cpp @@ -70,6 +70,11 @@ void Application::run() // mWallet.start(); + // + // Set up UNL. + // + getUNL().nodeBootstrap(); + // // Allow peer connections. // @@ -122,8 +127,6 @@ void Application::run() mNetOps.setStateTimer(0); - getUNL().nodeBootstrap(); - mIOService.run(); // This blocks std::cout << "Done." << std::endl; diff --git a/src/UniqueNodeList.cpp b/src/UniqueNodeList.cpp index e52b4e504..6e10d93ed 100644 --- a/src/UniqueNodeList.cpp +++ b/src/UniqueNodeList.cpp @@ -1503,10 +1503,10 @@ void UniqueNodeList::nodeBootstrap() ScopedLock sl(theApp->getWalletDB()->getDBLock()); - if (db->executeSQL("SELECT COUNT(*) AS Count FROM SeedDomains;") && db->startIterRows()) + if (db->executeSQL(str(boost::format("SELECT COUNT(*) AS Count FROM SeedDomains WHERE Source='%s' OR Source='%c';") % vsManual % vsValidator)) && db->startIterRows()) iDomains = db->getInt("Count"); - if (db->executeSQL("SELECT COUNT(*) AS Count FROM SeedNodes;") && db->startIterRows()) + if (db->executeSQL(str(boost::format("SELECT COUNT(*) AS Count FROM SeedNodes WHERE Source='%s' OR Source='%c';") % vsManual % vsValidator)) && db->startIterRows()) iNodes = db->getInt("Count"); } From 63a3e110c11f293c9980a0a77c774b9a362f3ee7 Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Sat, 16 Jun 2012 12:24:00 -0700 Subject: [PATCH 12/19] Fix default newcoind.cfg example for unl_default. --- newcoind.cfg | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/newcoind.cfg b/newcoind.cfg index e08702b46..916275fee 100644 --- a/newcoind.cfg +++ b/newcoind.cfg @@ -37,14 +37,16 @@ # starts up, if the databases are missing or are obsolete due to an upgrade # of newcoind, newcoind will reconstruct the UNL list as specified here. # -# If this field is not present or empty, newcoind will look for a validators.txt in the +# If this entry is not present or empty, newcoind will look for a validators.txt in the # config directory. If not found there, it will attempt to retrieve the file # from the newcoin foundation's web site. # +# This entry is also used by the RPC command unl_load. +# # Specify the file by specifying its full path. # -# Examples: C:/home/johndoe/newcoin/newcoind.cfg -# /home/johndoe/newcoin/newcoind.cfg +# Examples: C:/home/johndoe/newcoin/validators.txt +# /home/johndoe/newcoin/validators.txt # # [peer_ip]: # IP address or domain to bind to allow external connections from peers. From 2ed33307b468b223445ea23c505c290fe22ee204 Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Sat, 16 Jun 2012 12:35:24 -0700 Subject: [PATCH 13/19] Cosmetic changes. --- src/UniqueNodeList.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/UniqueNodeList.cpp b/src/UniqueNodeList.cpp index 6e10d93ed..6d8cb0a23 100644 --- a/src/UniqueNodeList.cpp +++ b/src/UniqueNodeList.cpp @@ -1427,14 +1427,14 @@ bool UniqueNodeList::nodeLoad() if (!boost::filesystem::exists(theConfig.UNL_DEFAULT)) { - std::cerr << str(boost::format("UNL_DEFAULT not found: '%s'") % theConfig.UNL_DEFAULT) << std::endl; + std::cerr << str(boost::format("UNL_DEFAULT not found: %s") % theConfig.UNL_DEFAULT) << std::endl; return false; } if (!boost::filesystem::is_regular_file(theConfig.UNL_DEFAULT)) { - std::cerr << str(boost::format("UNL_DEFAULT not regular file: '%s'") % theConfig.UNL_DEFAULT) << std::endl; + std::cerr << str(boost::format("UNL_DEFAULT not regular file: %s") % theConfig.UNL_DEFAULT) << std::endl; return false; } @@ -1443,7 +1443,7 @@ bool UniqueNodeList::nodeLoad() if (!ifsDefault) { - std::cerr << str(boost::format("Failed to open: '%s'") % theConfig.UNL_DEFAULT) << std::endl; + std::cerr << str(boost::format("Failed to open: %s") % theConfig.UNL_DEFAULT) << std::endl; return false; } @@ -1455,14 +1455,14 @@ bool UniqueNodeList::nodeLoad() if (ifsDefault.bad()) { - std::cerr << str(boost::format("Failed to read: '%s'") % theConfig.UNL_DEFAULT) << std::endl; + std::cerr << str(boost::format("Failed to read: %s") % theConfig.UNL_DEFAULT) << std::endl; return false; } nodeDefault(strValidators); - std::cerr << str(boost::format("Processing: '%s'") % theConfig.UNL_DEFAULT) << std::endl; + std::cerr << str(boost::format("Processing: %s") % theConfig.UNL_DEFAULT) << std::endl; return true; } From b1b8788b98da8ca39618ec5ec0467eef1c128733 Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Sat, 16 Jun 2012 12:37:51 -0700 Subject: [PATCH 14/19] Update example validators.txt. --- validators.txt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/validators.txt b/validators.txt index a94d0831c..45fc882be 100644 --- a/validators.txt +++ b/validators.txt @@ -9,8 +9,14 @@ # All other lines should be domain names. # # [validators]: -# To acquire a UNL, newcoind will probe for https web servers at the domains -# listed below in the following order: newcoin.DOMAIN, www.DOMAIN, DOMAIN +# List of nodes to accept as validators speficied by public key or domain. +# +# For domains, newcoind will probe for https web servers at the specied +# domain in the following order: newcoin.DOMAIN, www.DOMAIN, DOMAIN +# +# Examples: redstem.com +# n9KorY8QtTdRx7TVDpwnG9NvyxsDwHUKUEeDLY3AkiGncVaSXZi5 +# n9MqiExBcoG19UXwoLjBJnhsxEhAZMuWwJDRdkyDz1EkEkwzQTNt John Doe # [validators] From 2825942db056a2e9f7f6d0e41f4d4efe91e75328 Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Sat, 16 Jun 2012 13:23:40 -0700 Subject: [PATCH 15/19] Fixes for UNL bootstrapping. --- src/UniqueNodeList.cpp | 63 ++++++++++++++++++++++++------------------ src/UniqueNodeList.h | 4 +-- 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/src/UniqueNodeList.cpp b/src/UniqueNodeList.cpp index 6d8cb0a23..c445593df 100644 --- a/src/UniqueNodeList.cpp +++ b/src/UniqueNodeList.cpp @@ -628,10 +628,14 @@ void UniqueNodeList::processIps(const std::string& strSite, const NewcoinAddress } // Persist ValidatorReferrals. -void UniqueNodeList::processValidators(const std::string& strSite, const std::string& strValidatorsSrc, const NewcoinAddress& naNodePublic, section::mapped_type* pmtVecStrValidators) +// --> strSite: source site for display +// --> strValidatorsSrc: source details for display +// --> naNodePublic: remote source public key - not valid for local +// --> vsWhy: reason for adding validator to SeedDomains or SeedNodes. +void UniqueNodeList::processValidators(const std::string& strSite, const std::string& strValidatorsSrc, const NewcoinAddress& naNodePublic, validatorSource vsWhy, section::mapped_type* pmtVecStrValidators) { Database* db = theApp->getWalletDB()->getDB(); - std::string strNodePublic = naNodePublic.humanNodePublic(); + std::string strNodePublic = naNodePublic.isValid() ? naNodePublic.humanNodePublic() : "local"; std::cerr << str(boost::format("Validator: '%s' : '%s' : processing %d validators.") @@ -665,7 +669,7 @@ void UniqueNodeList::processValidators(const std::string& strSite, const std::st // domain comment? // public_key comment? - static boost::regex reReferral("\\`\\s*(\\S+)(?:\\s+(\\d+))?\\s*\\'"); + static boost::regex reReferral("\\`\\s*(\\S+)(?:\\s+(.+))?\\s*\\'"); if (!boost::regex_match(strReferral, smMatch, reReferral)) { @@ -680,35 +684,40 @@ void UniqueNodeList::processValidators(const std::string& strSite, const std::st std::string strComment = smMatch[2]; NewcoinAddress naValidator; + // std::cerr << str(boost::format("Validator: '%s' : '%s'") % strRefered % strComment) << std::endl; + if (naValidator.setNodePublic(strRefered)) { // A public key. // XXX Schedule for CAS lookup. - nodeAddPublic(naValidator, vsReferral, strComment); + nodeAddPublic(naValidator, vsWhy, strComment); - vstrValues.push_back(str(boost::format("('%s',%d,'%s')") % strNodePublic % iValues % naValidator.humanNodePublic())); + if (naNodePublic.isValid()) + vstrValues.push_back(str(boost::format("('%s',%d,'%s')") % strNodePublic % iValues % naValidator.humanNodePublic())); } else { // A domain: need to look it up. - boost::trim(strRefered); - boost::to_lower(strRefered); - nodeAddDomain(strRefered, vsReferral, strComment); + nodeAddDomain(strRefered, vsWhy, strComment); - vstrValues.push_back(str(boost::format("('%s',%d,%s)") % strNodePublic % iValues % sqlEscape(strRefered))); + if (naNodePublic.isValid()) + vstrValues.push_back(str(boost::format("('%s',%d,%s)") % strNodePublic % iValues % sqlEscape(strRefered))); } iValues++; } } - std::string strSql = str(boost::format("INSERT INTO ValidatorReferrals (Validator,Entry,Referral) VALUES %s;") - % strJoin(vstrValues.begin(), vstrValues.end(), ",")); + if (!vstrValues.empty()) + { + std::string strSql = str(boost::format("INSERT INTO ValidatorReferrals (Validator,Entry,Referral) VALUES %s;") + % strJoin(vstrValues.begin(), vstrValues.end(), ",")); - ScopedLock sl(theApp->getWalletDB()->getDBLock()); + ScopedLock sl(theApp->getWalletDB()->getDBLock()); - db->executeSQL(strSql); - // XXX Check result. + db->executeSQL(strSql); + // XXX Check result. + } } fetchDirty(); @@ -761,7 +770,7 @@ void UniqueNodeList::responseValidators(const std::string& strValidatorsUrl, con { section secFile = ParseSection(strValidatorsFile, true); - processValidators(strSite, strValidatorsUrl, naNodePublic, sectionEntries(secFile, SECTION_VALIDATORS)); + processValidators(strSite, strValidatorsUrl, naNodePublic, vsValidator, sectionEntries(secFile, SECTION_VALIDATORS)); } getIpsUrl(naNodePublic, secSite); @@ -799,7 +808,7 @@ void UniqueNodeList::processFile(const std::string strDomain, const NewcoinAddre // // Process Validators // - processValidators(strDomain, NODE_FILE_NAME, naNodePublic, sectionEntries(secSite, SECTION_VALIDATORS)); + processValidators(strDomain, NODE_FILE_NAME, naNodePublic, vsReferral, sectionEntries(secSite, SECTION_VALIDATORS)); // // Process ips @@ -1336,15 +1345,15 @@ void UniqueNodeList::nodeAddPublic(const NewcoinAddress& naNodePublic, validator // Promote source, if needed. if (!bFound || iSourceScore(vsWhy) >= iSourceScore(snCurrent.vsSource)) { - snCurrent.vsSource = vsWhy; - bChanged = true; + snCurrent.vsSource = vsWhy; + snCurrent.strComment = strComment; + bChanged = true; } if (vsManual == vsWhy) { // A manual add forces immediate scan. snCurrent.tpNext = boost::posix_time::second_clock::universal_time(); - snCurrent.strComment = strComment; bChanged = true; } @@ -1460,7 +1469,7 @@ bool UniqueNodeList::nodeLoad() return false; } - nodeDefault(strValidators); + nodeDefault(strValidators, theConfig.UNL_DEFAULT.native()); std::cerr << str(boost::format("Processing: %s") % theConfig.UNL_DEFAULT) << std::endl; @@ -1473,7 +1482,7 @@ void UniqueNodeList::validatorsResponse(const boost::system::error_code& err, st if (!err) { - nodeDefault(strResponse); + nodeDefault(strResponse, VALIDATORS_SITE); } else { @@ -1527,17 +1536,17 @@ void UniqueNodeList::nodeBootstrap() } // Process a validators.txt. -// --> strValidators: a validators.txt -void UniqueNodeList::nodeDefault(const std::string& strValidators) { +// --> strValidators: contents of a validators.txt +void UniqueNodeList::nodeDefault(const std::string& strValidators, const std::string& strSource) { section secValidators = ParseSection(strValidators, true); section::mapped_type* pmtEntries = sectionEntries(secValidators, SECTION_VALIDATORS); if (pmtEntries) { - BOOST_FOREACH(std::string strValidator, *pmtEntries) - { - nodeAddDomain(strValidator, vsValidator); - } + NewcoinAddress naInvalid; // Don't want a referrer on added entries. + + // YYY Unspecified might be bootstrap or rpc command + processValidators("unspecified", strSource, naInvalid, vsValidator, pmtEntries); } else { diff --git a/src/UniqueNodeList.h b/src/UniqueNodeList.h index d8c3853ca..8ef9b88f0 100644 --- a/src/UniqueNodeList.h +++ b/src/UniqueNodeList.h @@ -121,7 +121,7 @@ private: void responseValidators(const std::string& strValidatorsUrl, const NewcoinAddress& naNodePublic, section secSite, const std::string& strSite, const boost::system::error_code& err, const std::string strValidatorsFile); void processIps(const std::string& strSite, const NewcoinAddress& naNodePublic, section::mapped_type* pmtVecStrIps); - void processValidators(const std::string& strSite, const std::string& strValidatorsSrc, const NewcoinAddress& naNodePublic, section::mapped_type* pmtVecStrValidators); + void processValidators(const std::string& strSite, const std::string& strValidatorsSrc, const NewcoinAddress& naNodePublic, validatorSource vsWhy, section::mapped_type* pmtVecStrValidators); void processFile(const std::string strDomain, const NewcoinAddress& naNodePublic, section secSite); @@ -132,6 +132,7 @@ private: void setSeedNodes(const seedNode& snSource, bool bNext); void validatorsResponse(const boost::system::error_code& err, std::string strResponse); + void nodeDefault(const std::string& strValidators, const std::string& strSource); public: UniqueNodeList(boost::asio::io_service& io_service); @@ -143,7 +144,6 @@ public: void nodeAddDomain(std::string strDomain, validatorSource vsWhy, const std::string& strComment=""); void nodeRemovePublic(const NewcoinAddress& naNodePublic); void nodeRemoveDomain(std::string strDomain); - void nodeDefault(const std::string& strValidators); void nodeReset(); void nodeScore(); From 291951b0317b5fc0de10a9d1305edf5c88c1d3a8 Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Sat, 16 Jun 2012 15:08:36 -0700 Subject: [PATCH 16/19] Use SeedNodes when compute TrustedNodes scores. --- src/UniqueNodeList.cpp | 78 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 69 insertions(+), 9 deletions(-) diff --git a/src/UniqueNodeList.cpp b/src/UniqueNodeList.cpp index c445593df..fbee42321 100644 --- a/src/UniqueNodeList.cpp +++ b/src/UniqueNodeList.cpp @@ -197,22 +197,84 @@ void UniqueNodeList::scoreCompute() std::string strDomain = db->getStrBinary("Domain"); std::string strPublicKey = db->getStrBinary("PublicKey"); std::string strSource = db->getStrBinary("Source"); + int iScore = iSourceScore(static_cast(strSource[0])); + strIndex::iterator siOld = umPulicIdx.find(strPublicKey); + if (siOld == umPulicIdx.end()) + { + // New node + int iNode = vsnNodes.size(); + + umPulicIdx[strPublicKey] = iNode; + umDomainIdx[strDomain] = iNode; + + scoreNode snCurrent; + + snCurrent.strValidator = strPublicKey; + snCurrent.iScore = iScore; + snCurrent.iRoundSeed = snCurrent.iScore; + snCurrent.iRoundScore = 0; + snCurrent.iSeen = -1; + + vsnNodes.push_back(snCurrent); + } + else + { + scoreNode& snOld = vsnNodes[siOld->second]; + + if (snOld.iScore < iScore) + { + // Update old node + + snOld.iScore = iScore; + snOld.iRoundSeed = snOld.iScore; + } + } + } + } + } + + // For each entry in SeedNodes: + // - Add an entry in umPulicIdx, umDomainIdx, and vsnNodes. + { + ScopedLock sl(theApp->getWalletDB()->getDBLock()); + + SQL_FOREACH(db, "SELECT PublicKey,Source FROM SeedNodes;") + { + std::string strPublicKey = db->getStrBinary("PublicKey"); + std::string strSource = db->getStrBinary("Source"); + int iScore = iSourceScore(static_cast(strSource[0])); + strIndex::iterator siOld = umPulicIdx.find(strPublicKey); + + if (siOld == umPulicIdx.end()) + { + // New node int iNode = vsnNodes.size(); umPulicIdx[strPublicKey] = iNode; - umDomainIdx[strDomain] = iNode; scoreNode snCurrent; snCurrent.strValidator = strPublicKey; - snCurrent.iScore = iSourceScore(static_cast(strSource[0])); + snCurrent.iScore = iScore; snCurrent.iRoundSeed = snCurrent.iScore; snCurrent.iRoundScore = 0; snCurrent.iSeen = -1; vsnNodes.push_back(snCurrent); } + else + { + scoreNode& snOld = vsnNodes[siOld->second]; + + if (snOld.iScore < iScore) + { + // Update old node + + snOld.iScore = iScore; + snOld.iRoundSeed = snOld.iScore; + } + } } } @@ -227,8 +289,6 @@ void UniqueNodeList::scoreCompute() << std::endl; } - // XXX When we have a CAS do SeedNodes here. - // std::cerr << str(boost::format("vsnNodes.size=%d") % vsnNodes.size()) << std::endl; // Step through growing list of nodes adding each validation list. @@ -1203,22 +1263,22 @@ void UniqueNodeList::nodeAddDomain(std::string strDomain, validatorSource vsWhy, if (!bFound) { - sdCurrent.strDomain = strDomain; - sdCurrent.tpNext = boost::posix_time::second_clock::universal_time(); + sdCurrent.strDomain = strDomain; + sdCurrent.tpNext = boost::posix_time::second_clock::universal_time(); } // Promote source, if needed. if (!bFound || iSourceScore(vsWhy) >= iSourceScore(sdCurrent.vsSource)) { - sdCurrent.vsSource = vsWhy; - bChanged = true; + sdCurrent.vsSource = vsWhy; + sdCurrent.strComment = strComment; + bChanged = true; } if (vsManual == vsWhy) { // A manual add forces immediate scan. sdCurrent.tpNext = boost::posix_time::second_clock::universal_time(); - sdCurrent.strComment = strComment; bChanged = true; } From bb6867345e64c649680a590bec89163bd5e80f94 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 18 Jun 2012 00:32:51 -0700 Subject: [PATCH 17/19] Remove FIXME. --- src/Ledger.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Ledger.cpp b/src/Ledger.cpp index 7565d2a1c..aa57d6937 100644 --- a/src/Ledger.cpp +++ b/src/Ledger.cpp @@ -300,7 +300,6 @@ void Ledger::saveAcceptedLedger(Ledger::pointer ledger) "UPDATE Transactions SET LedgerSeq = '%d', Status = '%c' WHERE TransID = '%s';") % ledger->getLedgerSeq() % TXN_SQL_VALIDATED % txn.getTransactionID().GetHex())); } - // FIXME: If above updates no rows, modify seq/status (upsert) } db->executeSQL("COMMIT TRANSACTION;"); } From 562d0a658eb8b2bdbab2922bf4ca417a5f58d024 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 18 Jun 2012 00:44:27 -0700 Subject: [PATCH 18/19] Tiny bugfix. --- src/LedgerConsensus.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index bf17cd74a..b93875ea5 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -363,7 +363,7 @@ int LedgerConsensus::timerEntry() case lcsCUTOFF: return stateCutoff(sinceClose); case lcsFINISHED: return stateFinished(sinceClose); case lcsACCEPTED: return stateAccepted(sinceClose); - case lcsABORTED: return stateAccepted(sinceClose); + case lcsABORTED: return 1; } assert(false); return 1; @@ -702,7 +702,6 @@ void LedgerConsensus::accept(SHAMap::pointer set) #endif ScopedLock sl = theApp->getMasterLedger().getLock(); - applyTransactions(theApp->getMasterLedger().getCurrentLedger()->peekTransactionMap(), newOL, failedTransactions); theApp->getMasterLedger().pushLedger(newLCL, newOL); mState = lcsACCEPTED; From 2e1254cda68cd2141279858ef1487f5304f9f034 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 18 Jun 2012 00:44:33 -0700 Subject: [PATCH 19/19] Remove a FIXME, bug is fixed. --- src/Transaction.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Transaction.cpp b/src/Transaction.cpp index 56a95a5db..355a6a2fa 100644 --- a/src/Transaction.cpp +++ b/src/Transaction.cpp @@ -513,7 +513,7 @@ bool Transaction::save() if ((mStatus == INVALID) || (mStatus == REMOVED)) return false; char status; - switch(mStatus) + switch (mStatus) { case NEW: status = TXN_SQL_NEW; break; case INCLUDED: status = TXN_SQL_INCLUDED; break; @@ -523,8 +523,6 @@ bool Transaction::save() default: status = TXN_SQL_UNKNOWN; } - // FIXME: This needs to check if the transaction is already there and not - // de-confirm it Database *db = theApp->getTxnDB()->getDB(); ScopedLock dbLock = theApp->getTxnDB()->getDBLock(); return db->executeSQL(mTransaction->getSQLInsertHeader() + mTransaction->getSQL(getLedger(), status) + ";");