Merge branch 'master' of github.com:jedmccaleb/NewCoin

src/DBInit.cpp

@@ -145,38 +145,44 @@ const char *WalletDBInit[] = {
 	// - There may be multiple sources for a Validator.  The last source is used.
 	// Validator:
 	//  Public key of referrer.
-	// Index:
+	// Entry:
 	//  Entry index in [validators] table.
 	// Referral:
 	//  This is the form provided by the newcoin.txt:
 	//  - Public key for CAS based referral.
 	//  - Domain for domain based referral.
-	// XXX Might keep a reference count for garbage collection.
+	// XXX Do garbage collection when validators have no references.
 	"CREATE TABLE ValidatorReferrals (				\
-		Validator		CHARACTER(53),				\
-		Entry			INTEGER,					\
-		Referral		TEXT,						\
+		Validator		CHARACTER(53) NOT NULL,		\
+		Entry			INTEGER NOT NULL,			\
+		Referral		TEXT NOT NULL,				\
 		PRIMARY KEY (Validator,Entry)				\
 	);",
 
-	// List of referrals.
+	// List of referrals from newcoin.txt files.
 	// Validator:
 	//  Public key of referree.
-	// Index:
+	// Entry:
 	//  Entry index in [validators] table.
 	// IP:
 	//  IP of referred.
-	"CREATE TABLE IpReferrals (						\
-		Validator		CHARACTER(53),				\
-		Index			INTEGER,					\
-		IP				TEXT						\
+	// Port:
+	//  -1 = Default
+	// XXX Do garbage collection when ips have no references.
+	"CREATE TABLE IpReferrals (							\
+		Validator		CHARACTER(53) NOT NULL,			\
+		Entry			INTEGER NOT NULL,				\
+		IP				TEXT NOT NULL,					\
+		Port			INTEGER NOT NULL DEFAULT -1,	\
+		PRIMARY KEY (Validator,Entry)					\
 	);",
 
 	// Table of IPs to contact the nextwork.
 	// IP:
 	//  IP address to contact.
 	// Port:
-	//  Port to contact. 0=default.
+	//  Port to contact.
+	//  -1 = Default
 	// Score:
 	//  Computed trust score.  Higher is better.
 	// Source:
@@ -187,12 +193,13 @@ const char *WalletDBInit[] = {
 	// Contact:
 	//  Time of last contact.
 	//  XXX Update on connect and hourly.
-	"CREATE TABLE PeerIps (							\
-		IP				TEXT PRIMARY KEY,			\
-		PORT			INTEGER,					\
-		Score			INTEGER,					\
-		Source			CHARACTER(1),				\
-		Contact			DATETIME					\
+	"CREATE TABLE PeerIps (								\
+		IP				TEXT NOT NULL,					\
+		Port			INTEGER NOT NULL DEFAULT -1,	\
+		Score			INTEGER NOT NULL,				\
+		Source			CHARACTER(1) NOT NULL,			\
+		Contact			DATETIME,						\
+		PRIMARY KEY (IP,PORT)							\
 	);",
 };
 
@@ -217,6 +224,7 @@ const char *HashNodeDBInit[] = {
 		LedgerIndex	BIGINT UNSIGNED,				\
 		Object		BLOB							\
 	);",
+
 	"CREATE INDEX ObjectLocate ON					\
 		CommittedObjects(LedgerIndex, ObjType);"
 };
@@ -231,7 +239,7 @@ const char *NetNodeDBInit[] = {
 		LastSeen		TEXT,						\
 		HaveContactInfo	CHARACTER(1),				\
 		ContactObject	BLOB						\
-		);"
+	);"
 };
 
 

src/ParseSection.cpp

@@ -43,10 +43,7 @@ section ParseSection(const std::string strInput, const bool bTrim)
 		{
 			// Another line in a section.
 			if (bTrim)
-			{
-				boost::algorithm::trim_right_if(strValue, boost::algorithm::is_space());
-				boost::algorithm::trim_left_if(strValue, boost::algorithm::is_space());
-			}
+				boost::algorithm::trim(strValue);
 
 			secResult[strSection].push_back(strValue);
 		}

src/RPCServer.cpp

@@ -694,6 +694,17 @@ Json::Value RPCServer::doUnlReset(Json::Value& params) {
 	else return "invalid params";
 }
 
+// unl_score
+Json::Value RPCServer::doUnlScore(Json::Value& params) {
+	if(!params.size())
+	{
+		theApp->getUNL().nodeScore();
+
+		return "scoring requested";
+	}
+	else return "invalid params";
+}
+
 Json::Value RPCServer::doCommand(const std::string& command, Json::Value& params)
 {
 	std::cerr << "RPC:" << command << std::endl;
@@ -710,6 +721,7 @@ Json::Value RPCServer::doCommand(const std::string& command, Json::Value& params
 	if(command=="unl_delete") return doUnlDelete(params);
 	if(command=="unl_list") return doUnlList(params);
 	if(command=="unl_reset") return doUnlReset(params);
+	if(command=="unl_score") return doUnlScore(params);
 
 	if(command=="validation_create") return doValidatorCreate(params);
 

src/RPCServer.h

@@ -52,6 +52,7 @@ class RPCServer  : public boost::enable_shared_from_this<RPCServer>
 	Json::Value doUnlFetch(Json::Value& params);
 	Json::Value doUnlList(Json::Value& params);
 	Json::Value doUnlReset(Json::Value& params);
+	Json::Value doUnlScore(Json::Value& params);
 
 	Json::Value doValidatorCreate(Json::Value& params);
 

src/UniqueNodeList.cpp

@@ -14,6 +14,7 @@
 #include <boost/foreach.hpp>
 #include <boost/format.hpp>
 #include <boost/mem_fn.hpp>
+#include <boost/regex.hpp>
 
 // Gather string constants.
 #define SECTION_CURRENCIES		"currencies"
@@ -28,6 +29,10 @@
 #define REFERRAL_VALIDATORS_MAX	50
 #define REFERRAL_IPS_MAX		50
 
+#define SQL_FOREACH(_db, _strQuery) \
+	if ((_db)->executeSQL(_strQuery))	\
+		for (bool _bMore = (db)->startIterRows(); _bMore; _bMore = (_db)->getNextRow())
+
 UniqueNodeList::UniqueNodeList(boost::asio::io_service& io_service) :
 	mdtScoreTimer(io_service),
 	mFetchActive(0),
@@ -43,7 +48,7 @@ void UniqueNodeList::start()
 	std::cerr << "Validator score updated: " << mtpScoreUpdated << std::endl;
 
 	fetchNext();			// Start fetching.
-	scoreNext();			// Start scoring.
+	scoreNext(false);		// Start scoring.
 }
 
 bool UniqueNodeList::miscLoad()
@@ -82,17 +87,18 @@ bool UniqueNodeList::scoreRound(std::vector<scoreNode>& vsnNodes)
 
     // For each node, distribute roundSeed to roundScores.
     BOOST_FOREACH(scoreNode& sn, vsnNodes) {
-		long	iEntries	= sn.viReferrals.size();
+		int		iEntries	= sn.viReferrals.size();
 
 		if (sn.iRoundSeed && iEntries)
 		{
-			long	iTotal	= (iEntries + 1) * iEntries / 2;
-			long	iBase	= sn.iRoundSeed * iEntries / iTotal;
+			score	iTotal	= (iEntries + 1) * iEntries / 2;
+			score	iBase	= sn.iRoundSeed * iEntries / iTotal;
 
 			// Distribute the current entires' seed score to validators prioritized by mention order.
 			for (int i=0; i != iEntries; i++) {
-				vsnNodes[sn.viReferrals[i]].iRoundScore	+=
-					iBase * (iEntries - i) / iEntries;
+				score	iPoints	= iBase * (iEntries - i) / iEntries;
+
+				vsnNodes[sn.viReferrals[i]].iRoundScore	+= iPoints;
 			}
 		}
     }
@@ -150,45 +156,37 @@ void UniqueNodeList::scoreCompute()
 	{
 		ScopedLock	sl(theApp->getWalletDB()->getDBLock());
 
-		if (db->executeSQL("SELECT Domain,PublicKey,Source FROM SeedDomains;"))
+		SQL_FOREACH(db, "SELECT Domain,PublicKey,Source FROM SeedDomains;")
 		{
-			bool	bMore	= db->startIterRows();
-			while (bMore)
+			if (db->getNull("PublicKey"))
 			{
-				if (db->getNull("PublicKey"))
-				{
-					nothing();	// We ignore entrys we don't have public keys for.
-				}
-				else
-				{
-					std::string	strDomain;
-					std::string	strPublicKey;
-					std::string	strSource;
-
-					db->getStr("Domain", strDomain);
-					db->getStr("PublicKey", strPublicKey);
-					db->getStr("Source", strSource);
-
-					int			iNode		= vsnNodes.size();
-
-					umPulicIdx[strPublicKey]	= iNode;
-					umDomainIdx[strDomain]		= iNode;
-
-					scoreNode	snCurrent;
-
-					snCurrent.strValidator	= strPublicKey;
-					snCurrent.iScore		= iSourceScore(static_cast<validatorSource>(strSource[0]));
-					snCurrent.iRoundSeed	= snCurrent.iScore;
-					snCurrent.iRoundScore	= 0;
-					snCurrent.iSeen			= -1;
-
-					vsnNodes.push_back(snCurrent);
-				}
-
-				bMore	= db->getNextRow();
+				nothing();	// We ignore entrys we don't have public keys for.
 			}
+			else
+			{
+				std::string	strDomain;
+				std::string	strPublicKey;
+				std::string	strSource;
 
-			db->endIterRows();
+				db->getStr("Domain", strDomain);
+				db->getStr("PublicKey", strPublicKey);
+				db->getStr("Source", strSource);
+
+				int			iNode		= vsnNodes.size();
+
+				umPulicIdx[strPublicKey]	= iNode;
+				umDomainIdx[strDomain]		= iNode;
+
+				scoreNode	snCurrent;
+
+				snCurrent.strValidator	= strPublicKey;
+				snCurrent.iScore		= iSourceScore(static_cast<validatorSource>(strSource[0]));
+				snCurrent.iRoundSeed	= snCurrent.iScore;
+				snCurrent.iRoundScore	= 0;
+				snCurrent.iSeen			= -1;
+
+				vsnNodes.push_back(snCurrent);
+			}
 		}
 	}
 
@@ -213,73 +211,63 @@ void UniqueNodeList::scoreCompute()
 		std::string&		strValidator	= sn.strValidator;
 		std::vector<int>&	viReferrals		= sn.viReferrals;
 
-		strSql	= str(boost::format("SELECT Referral FROM ValidatorReferrals WHERE Validator=%s ORDER BY Entry;")
-					% db->escape(strValidator));
-
 		ScopedLock	sl(theApp->getWalletDB()->getDBLock());
 
-		if (db->executeSQL(strSql))
+		SQL_FOREACH(db, str(boost::format("SELECT Referral FROM ValidatorReferrals WHERE Validator=%s ORDER BY Entry;")
+					% db->escape(strValidator)))
 		{
-			bool	bMore	= db->startIterRows();
-			while (bMore)
+			std::string	strReferral;
+			int			iReferral;
+
+			db->getStr("Referral", strReferral);
+
+			strIndex::iterator	itEntry;
+
+			NewcoinAddress		na;
+
+			if (na.setNodePublic(strReferral))
 			{
-				std::string	strReferral;
-				int			iReferral;
+				// Referring a public key.
+				itEntry		= umPulicIdx.find(strReferral);
 
-				db->getStr("Referral", strReferral);
-
-				strIndex::iterator	itEntry;
-
-				NewcoinAddress		na;
-
-				if (na.setNodePublic(strReferral))
+				if (itEntry == umPulicIdx.end())
 				{
-					// Referring a public key.
-					itEntry		= umPulicIdx.find(strReferral);
+					// Not found add public key to list of nodes.
+					iReferral				= vsnNodes.size();
 
-					if (itEntry == umPulicIdx.end())
-					{
-						// Not found add public key to list of nodes.
-						iReferral				= vsnNodes.size();
+					umPulicIdx[strReferral]	= iReferral;
 
-						umPulicIdx[strReferral]	= iReferral;
+					scoreNode	snCurrent;
 
-						scoreNode	snCurrent;
-
-						snCurrent.strValidator	= strReferral;
-						snCurrent.iScore		= iSourceScore(vsReferral);
-						snCurrent.iRoundSeed	= snCurrent.iScore;
-						snCurrent.iRoundScore	= 0;
-						snCurrent.iSeen			= -1;
-
-						vsnNodes.push_back(snCurrent);
-					}
-					else
-					{
-						iReferral	=  itEntry->second;
-					}
-
-					// std::cerr << str(boost::format("%s: Public=%s iReferral=%d") % strValidator % strReferral % iReferral) << std::endl;
+					snCurrent.strValidator	= strReferral;
+					snCurrent.iScore		= iSourceScore(vsReferral);
+					snCurrent.iRoundSeed	= snCurrent.iScore;
+					snCurrent.iRoundScore	= 0;
+					snCurrent.iSeen			= -1;
 
+					vsnNodes.push_back(snCurrent);
 				}
 				else
 				{
-					// Referring a domain.
-					itEntry		= umDomainIdx.find(strReferral);
-					iReferral	= itEntry == umDomainIdx.end()
-									? -1			// We ignore domains we can't find entires for.
-									: itEntry->second;
-
-					// std::cerr << str(boost::format("%s: Domain=%s iReferral=%d") % strValidator % strReferral % iReferral) << std::endl;
+					iReferral	=  itEntry->second;
 				}
 
-				if (iReferral >= 0 && iNode != iReferral)
-					viReferrals.push_back(iReferral);
+				// std::cerr << str(boost::format("%s: Public=%s iReferral=%d") % strValidator % strReferral % iReferral) << std::endl;
 
-				bMore	= db->getNextRow();
+			}
+			else
+			{
+				// Referring a domain.
+				itEntry		= umDomainIdx.find(strReferral);
+				iReferral	= itEntry == umDomainIdx.end()
+								? -1			// We ignore domains we can't find entires for.
+								: itEntry->second;
+
+				// std::cerr << str(boost::format("%s: Domain=%s iReferral=%d") % strValidator % strReferral % iReferral) << std::endl;
 			}
 
-			db->endIterRows();
+			if (iReferral >= 0 && iNode != iReferral)
+				viReferrals.push_back(iReferral);
 		}
 	}
 
@@ -308,7 +296,7 @@ void UniqueNodeList::scoreCompute()
 
 	if (vsnNodes.size())
 	{
-		// Merge existing Seens.
+		// Load existing Seens from DB.
 		std::vector<std::string>	vstrPublicKeys;
 
 		vstrPublicKeys.resize(vsnNodes.size());
@@ -318,55 +306,124 @@ void UniqueNodeList::scoreCompute()
 			vstrPublicKeys[iNode]	= db->escape(vsnNodes[iNode].strValidator);
 		}
 
-		if (db->executeSQL(str(boost::format("SELECT PublicKey,Seen FROM TrustedNodes WHERE PublicKey IN (%s);")
-				% strJoin(vstrPublicKeys.begin(), vstrPublicKeys.end(), ","))))
+		SQL_FOREACH(db, str(boost::format("SELECT PublicKey,Seen FROM TrustedNodes WHERE PublicKey IN (%s);")
+				% strJoin(vstrPublicKeys.begin(), vstrPublicKeys.end(), ",")))
 		{
-			bool	bMore	= db->startIterRows();
-			while (bMore)
-			{
-				std::string	strPublicKey;
+			std::string	strPublicKey;
 
-				db->getStr("PublicKey", strPublicKey);
+			db->getStr("PublicKey", strPublicKey);
 
-				vsnNodes[umPulicIdx[strPublicKey]].iSeen	= db->getNull("Seen") ? -1 : db->getInt("Seen");
-
-				bMore	= db->getNextRow();
-			}
-
-			db->endIterRows();
+			vsnNodes[umPulicIdx[strPublicKey]].iSeen	= db->getNull("Seen") ? -1 : db->getInt("Seen");
 		}
 	}
 
 	if (vsnNodes.size())
 	{
-		// Update old PublicKeys and add new ones.
+		// Update old entries and add new ones.
 		std::vector<std::string>	vstrValues;
 
 		vstrValues.resize(vsnNodes.size());
 
 		for (int iNode=vsnNodes.size(); iNode--;)
 		{
-			scoreNode& sn	= vsnNodes[iNode];
+			scoreNode&	sn		= vsnNodes[iNode];
+			std::string	strSeen	= sn.iSeen >= 0 ? str(boost::format("%d") % sn.iSeen) : "NULL";
 
-			if (sn.iSeen >= 0)
-			{
-				vstrValues[iNode]	= str(boost::format("(%s,%s,%s)")
-					% db->escape(sn.strValidator)
-					% sn.iScore
-					% sn.iSeen);
-			}
-			else
-			{
-				vstrValues[iNode]	= str(boost::format("(%s,%s,NULL)")
-					% db->escape(sn.strValidator)
-					% sn.iScore);
-			}
+			vstrValues[iNode]	= str(boost::format("(%s,%s,%s)")
+				% db->escape(sn.strValidator)
+				% sn.iScore
+				% strSeen);
 		}
 
 		db->executeSQL(str(boost::format("REPLACE INTO TrustedNodes (PublicKey,Score,Seen) VALUES %s;")
 				% strJoin(vstrValues.begin(), vstrValues.end(), ",")));
 	}
 
+	// Score IPs.
+	db->executeSQL("UPDATE PeerIps SET Score = 0 WHERE Score != 0;");
+
+	boost::unordered_map<std::string, int>	umValidators;
+
+	if (vsnNodes.size())
+	{
+		std::vector<std::string> vstrPublicKeys;
+
+		// For every IpReferral add a score for the IP and PORT.
+		SQL_FOREACH(db, "SELECT Validator,COUNT(*) AS Count FROM IpReferrals GROUP BY Validator;")
+		{
+			std::string	strValidator;
+
+			db->getStr("Validator", strValidator);
+
+			umValidators[strValidator]	= db->getInt("Count");
+
+			// std::cerr << strValidator << ":" << db->getInt("Count") << std::endl;
+		}
+	}
+
+	// For each validator, get each referal and add its score to ip's score.
+	// map of pair<IP,Port> :: score
+	epScore	umScore;
+
+	std::pair< std::string, int> vc;
+	BOOST_FOREACH(vc, umValidators)
+	{
+		std::string	strValidator	= vc.first;
+
+		strIndex::iterator	itIndex	= umPulicIdx.find(strValidator);
+		if (itIndex != umPulicIdx.end()) {
+			int			iSeed			= vsnNodes[itIndex->second].iScore;
+			int			iEntries		= vc.second;
+			score		iTotal			= (iEntries + 1) * iEntries / 2;
+			score		iBase			= iSeed * iEntries / iTotal;
+			int			iEntry			= 0;
+
+			SQL_FOREACH(db, str(boost::format("SELECT IP,Port FROM IpReferrals WHERE Validator=%s ORDER BY Entry;")
+				% db->escape(strValidator)))
+			{
+				score		iPoints	= iBase * (iEntries - iEntry) / iEntries;
+				std::string	strIP;
+				int			iPort;
+
+				db->getStr("IP", strIP);
+				iPort		= db->getNull("Port") ? -1 : db->getInt("Port");
+
+				std::pair< std::string, int>	ep	= std::make_pair(strIP, iPort);
+
+				epScore::iterator	itEp	= umScore.find(ep);
+
+				umScore[ep]	= itEp == umScore.end() ? iPoints :  itEp->second + iPoints;
+				iEntry++;
+			}
+		}
+	}
+
+	// Apply validator scores to each IP.
+	if (umScore.size())
+	{
+		std::vector<std::string>	vstrValues;
+
+		vstrValues.reserve(umScore.size());
+
+		std::pair< ipPort, score>	ipScore;
+		BOOST_FOREACH(ipScore, umScore)
+		{
+			ipPort		ipEndpoint	= ipScore.first;
+			std::string	strIP		= ipEndpoint.first;
+			int			iPort		= ipEndpoint.second;
+			score		iPoints		= ipScore.second;
+
+			vstrValues.push_back(str(boost::format("(%s,%d,%d,'V')")
+				% db->escape(strIP)
+				% iPort
+				% iPoints));
+		}
+
+		// Set scores for each IP.
+		db->executeSQL(str(boost::format("REPLACE INTO PeerIps (IP,Port,Score,Source) VALUES %s;")
+				% strJoin(vstrValues.begin(), vstrValues.end(), ",")));
+	}
+
 	db->executeSQL("COMMIT;");
 }
 
@@ -378,11 +435,11 @@ void UniqueNodeList::scoreTimerHandler(const boost::system::error_code& err)
 		mtpScoreNext	= boost::posix_time::ptime(boost::posix_time::not_a_date_time);	// Timer not set.
 		mtpScoreStart	= boost::posix_time::second_clock::universal_time();			// Scoring.
 
-		std::cerr << "Scoring start." << std::endl;
+		std::cerr << "Scoring: Start" << std::endl;
 
 		scoreCompute();
 
-		std::cerr << "Scoring end." << std::endl;
+		std::cerr << "Scoring: End" << std::endl;
 
 		// Save update time.
 		mtpScoreUpdated	= mtpScoreStart;
@@ -391,32 +448,44 @@ void UniqueNodeList::scoreTimerHandler(const boost::system::error_code& err)
 		mtpScoreStart	= boost::posix_time::ptime(boost::posix_time::not_a_date_time);	// Not scoring.
 
 		// Score again if needed.
-		scoreNext();
+		scoreNext(false);
 	}
 }
 
 // Start a timer to update scores.
-void UniqueNodeList::scoreNext()
+void UniqueNodeList::scoreNext(bool bNow)
 {
-	std::cerr << "scoreNext>" << std::endl;
+	// std::cerr << str(boost::format("scoreNext: mtpFetchUpdated=%s mtpScoreStart=%s mtpScoreUpdated=%s mtpScoreNext=%s") % mtpFetchUpdated % mtpScoreStart % mtpScoreUpdated % mtpScoreNext)  << std::endl;
+	bool	bCanScore	= mtpScoreStart.is_not_a_date_time()							// Not scoring.
+							&& !mtpFetchUpdated.is_not_a_date_time();					// Something to score.
 
-	std::cerr << str(boost::format("scoreNext: mtpFetchUpdated=%s mtpScoreStart=%s mtpScoreUpdated=%s mtpScoreNext=%s") % mtpFetchUpdated % mtpScoreStart % mtpScoreUpdated % mtpScoreNext)  << std::endl;
-	if (mtpScoreStart.is_not_a_date_time()												// Not scoring
-		&& !mtpFetchUpdated.is_not_a_date_time()										// Need to score.
-		&& (mtpScoreUpdated.is_not_a_date_time() || mtpScoreUpdated <= mtpFetchUpdated)	// Not already scored.
+	bool	bDirty	=
+		(mtpScoreUpdated.is_not_a_date_time() || mtpScoreUpdated <= mtpFetchUpdated)	// Not already scored.
 		&& (mtpScoreNext.is_not_a_date_time()											// Timer is not fine.
-			|| mtpScoreNext < mtpFetchUpdated + boost::posix_time::seconds(SCORE_DELAY_SECONDS)))
+			|| mtpScoreNext < mtpFetchUpdated + boost::posix_time::seconds(SCORE_DELAY_SECONDS));
+
+	if (!bCanScore)
+	{
+		nothing();
+	}
+	else if (bNow || bDirty)
 	{
 		// Need to update or set timer.
 		mtpScoreNext = boost::posix_time::second_clock::universal_time()					// Past now too.
-							+ boost::posix_time::seconds(SCORE_DELAY_SECONDS);
+							+ boost::posix_time::seconds(bNow ? 0 : SCORE_DELAY_SECONDS);
 
-	std::cerr << str(boost::format("scoreNext: @%s") % mtpScoreNext)  << std::endl;
+		// std::cerr << str(boost::format("scoreNext: @%s") % mtpScoreNext)  << std::endl;
 		mdtScoreTimer.expires_at(mtpScoreNext);
 		mdtScoreTimer.async_wait(boost::bind(&UniqueNodeList::scoreTimerHandler, this, _1));
 	}
 }
 
+// For debugging, schedule forced scoring.
+void UniqueNodeList::nodeScore()
+{
+	scoreNext(true);
+}
+
 void UniqueNodeList::fetchFinish()
 {
 	{
@@ -427,14 +496,97 @@ void UniqueNodeList::fetchFinish()
 	fetchNext();
 }
 
-void UniqueNodeList::processIps(const std::string& strSite, section::mapped_type* pmtVecStrIps)
+void UniqueNodeList::fetchDirty()
 {
+	// Note update.
+	mtpFetchUpdated	= boost::posix_time::second_clock::universal_time();
+	miscSave();
+
+	// Update scores.
+	scoreNext(false);
+}
+
+void UniqueNodeList::processIps(const std::string& strSite, NewcoinAddress naNodePublic, section::mapped_type* pmtVecStrIps)
+{
+	Database*	db=theApp->getWalletDB()->getDB();
+
+	std::string strEscNodePublic	= db->escape(naNodePublic.humanNodePublic());
+
 	std::cerr
 		<< str(boost::format("Validator: '%s' processing %d ips.")
 			% strSite % ( pmtVecStrIps ? pmtVecStrIps->size() : 0))
 		<< std::endl;
 
-	// XXX Do something with ips.
+	// Remove all current Validator's entries in IpReferrals
+	{
+		ScopedLock	sl(theApp->getWalletDB()->getDBLock());
+		db->executeSQL(str(boost::format("DELETE FROM IpReferrals WHERE Validator=%s;") % strEscNodePublic));
+		// XXX Check result.
+	}
+
+	// Add new referral entries.
+	if (pmtVecStrIps && pmtVecStrIps->size()) {
+		std::vector<std::string> vstrValues;
+
+		vstrValues.resize(MIN(pmtVecStrIps->size(), REFERRAL_IPS_MAX));
+
+		int	iValues = 0;
+		BOOST_FOREACH(std::string strReferral, *pmtVecStrIps)
+		{
+			boost::smatch	smMatch;
+			std::string		strIP;
+			int				iPort;
+			bool			bValid	= false;
+
+			if (iValues == REFERRAL_VALIDATORS_MAX)
+				break;
+
+			static boost::regex	reEndpoint("\\`\\s*(\\S+)(?:\\s+(\\d+))?\\s*\\'");
+
+			// XXX Filter out private network ips.
+			// XXX http://en.wikipedia.org/wiki/Private_network
+			if (boost::regex_match(strReferral, smMatch, reEndpoint))
+			{
+				boost::system::error_code	err;
+				std::string					strIPRaw	= smMatch[1];
+				std::string					strPortRaw	= smMatch[2];
+
+				iPort		= strPortRaw.empty() ? -1 : boost::lexical_cast<int>(strPortRaw);
+
+				boost::asio::ip::address	addrIP		= boost::asio::ip::address::from_string(strIPRaw, err);
+
+				bValid	= !err;
+				if (bValid)
+					strIP	= addrIP.to_string();
+			}
+
+			if (bValid)
+			{
+				vstrValues[iValues]	= str(boost::format("(%s,%d,%s,%d)")
+					% strEscNodePublic % iValues % db->escape(strIP) % iPort);
+				iValues++;
+			}
+			else
+			{
+				std::cerr
+					<< str(boost::format("Validator: '%s' ["SECTION_IPS"]: rejecting '%s'")
+						% strSite % strReferral)
+					<< std::endl;
+			}
+		}
+
+		if (iValues)
+		{
+			vstrValues.resize(iValues);
+
+			ScopedLock sl(theApp->getWalletDB()->getDBLock());
+			db->executeSQL(str(boost::format("INSERT INTO IpReferrals (Validator,Entry,IP,Port) VALUES %s;")
+				% strJoin(vstrValues.begin(), vstrValues.end(), ",")));
+			// XXX Check result.
+		}
+	}
+
+	fetchDirty();
 }
 
 void UniqueNodeList::processValidators(const std::string& strSite, const std::string& strValidatorsSrc, NewcoinAddress naNodePublic, section::mapped_type* pmtVecStrValidators)
@@ -450,14 +602,10 @@ void UniqueNodeList::processValidators(const std::string& strSite, const std::st
 			% ( pmtVecStrValidators ? pmtVecStrValidators->size() : 0))
 		<< std::endl;
 
-	// Remove all current entries Validator in ValidatorReferrals
-	// XXX INDEX BY ValidatorReferralsIndex
+	// Remove all current Validator's entries in ValidatorReferrals
 	{
-		std::string strSql	= str(boost::format("DELETE FROM ValidatorReferrals WHERE Validator=%s;")
-			% strEscNodePublic);
-
 		ScopedLock	sl(theApp->getWalletDB()->getDBLock());
-		db->executeSQL(strSql);
+		db->executeSQL(str(boost::format("DELETE FROM ValidatorReferrals WHERE Validator=%s;") % strEscNodePublic));
 		// XXX Check result.
 	}
 
@@ -485,7 +633,7 @@ void UniqueNodeList::processValidators(const std::string& strSite, const std::st
 			}
 			else
 			{
-				// A domain.
+				// A domain: need to look it up.
 				nodeAddDomain(strReferral, vsReferral);
 			}
 		}
@@ -499,21 +647,16 @@ void UniqueNodeList::processValidators(const std::string& strSite, const std::st
 		// XXX Check result.
 	}
 
-	// Note update.
-	mtpFetchUpdated	= boost::posix_time::second_clock::universal_time();
-	miscSave();
-
-	// Update scores.
-	scoreNext();
+	fetchDirty();
 }
 
-void UniqueNodeList::responseIps(const std::string& strSite, const boost::system::error_code& err, const std::string strIpsFile)
+void UniqueNodeList::responseIps(const std::string& strSite, NewcoinAddress naNodePublic, const boost::system::error_code& err, const std::string strIpsFile)
 {
 	if (!err)
 	{
 		section			secFile		= ParseSection(strIpsFile, true);
 
-		processIps(strSite, sectionEntries(secFile, SECTION_IPS));
+		processIps(strSite, naNodePublic, sectionEntries(secFile, SECTION_IPS));
 	}
 
 	fetchFinish();
@@ -522,7 +665,7 @@ void UniqueNodeList::responseIps(const std::string& strSite, const boost::system
 //
 // Process [ips_url]
 //
-void UniqueNodeList::getIpsUrl(section secSite)
+void UniqueNodeList::getIpsUrl(NewcoinAddress naNodePublic, section secSite)
 {
 	std::string	strIpsUrl;
 	std::string	strDomain;
@@ -539,7 +682,7 @@ void UniqueNodeList::getIpsUrl(section secSite)
 			strPath,
 			NODE_FILE_BYTES_MAX,
 			boost::posix_time::seconds(NODE_FETCH_SECONDS),
-			boost::bind(&UniqueNodeList::responseIps, this, strDomain, _1, _2));
+			boost::bind(&UniqueNodeList::responseIps, this, strDomain, naNodePublic, _1, _2));
 	}
 	else
 	{
@@ -556,7 +699,7 @@ void UniqueNodeList::responseValidators(const std::string& strValidatorsUrl, New
 		processValidators(strSite, strValidatorsUrl, naNodePublic, sectionEntries(secFile, SECTION_VALIDATORS));
 	}
 
-	getIpsUrl(secSite);
+	getIpsUrl(naNodePublic, secSite);
 }
 
 //
@@ -583,7 +726,7 @@ void UniqueNodeList::getValidatorsUrl(NewcoinAddress naNodePublic, section secSi
 	}
 	else
 	{
-		getIpsUrl(secSite);
+		getIpsUrl(naNodePublic, secSite);
 	}
 }
 
@@ -598,7 +741,7 @@ void UniqueNodeList::processFile(const std::string strDomain, NewcoinAddress naN
 	//
 	// Process ips
 	//
-	processIps(strDomain, sectionEntries(secSite, SECTION_IPS));
+	processIps(strDomain, naNodePublic, sectionEntries(secSite, SECTION_IPS));
 
 	//
 	// Process currencies
@@ -739,14 +882,11 @@ void UniqueNodeList::fetchProcess(std::string strDomain)
 	std::cerr << "Fetching '" NODE_FILE_NAME "' from '" << strDomain << "'." << std::endl;
 
 	std::deque<std::string>	deqSites;
-	std::ostringstream		ossNewcoin;
-	std::ostringstream		ossWeb;
 
-	ossNewcoin << SYSTEM_NAME "." << strDomain;
-	ossWeb << "www." << strDomain;
-
-	deqSites.push_back(ossNewcoin.str());
-	deqSites.push_back(ossWeb.str());
+	// Order searching from most specifically for purpose to generic.
+	// This order allows the client to take the most burden rather than the servers.
+	deqSites.push_back(str(boost::format(SYSTEM_NAME ".%s") % strDomain));
+	deqSites.push_back(str(boost::format("www.%s") % strDomain));
 	deqSites.push_back(strDomain);
 
 	HttpsClient::httpsGet(
@@ -949,8 +1089,14 @@ bool UniqueNodeList::getSeedDomains(const std::string& strDomain, seedDomain& ds
 			dstSeedDomain.tpScan	= ptFromSeconds(iScan);
 		iFetch	= db->getInt("Fetch");
 			dstSeedDomain.tpFetch	= ptFromSeconds(iFetch);
-		db->getStr("Sha256", strSha256);
+		if (!db->getNull("Sha256") && db->getStr("Sha256", strSha256))
+		{
 			dstSeedDomain.iSha256.SetHex(strSha256);
+		}
+		else
+		{
+			dstSeedDomain.iSha256.zero();
+		}
 		db->getStr("Comment", dstSeedDomain.strComment);
 
 		db->endIterRows();
@@ -1032,28 +1178,20 @@ Json::Value UniqueNodeList::getUnlJson()
     Json::Value ret(Json::arrayValue);
 
 	ScopedLock sl(theApp->getWalletDB()->getDBLock());
-	if (db->executeSQL("SELECT * FROM TrustedNodes;"))
+	SQL_FOREACH(db, "SELECT * FROM TrustedNodes;")
 	{
-		bool	more	= db->startIterRows();
-		while (more)
-		{
-			std::string	strPublicKey;
-			std::string	strComment;
+		std::string	strPublicKey;
+		std::string	strComment;
 
-			db->getStr("PublicKey", strPublicKey);
-			db->getStr("Comment", strComment);
+		db->getStr("PublicKey", strPublicKey);
+		db->getStr("Comment", strComment);
 
-			Json::Value node(Json::objectValue);
+		Json::Value node(Json::objectValue);
 
-			node["PublicKey"]	= strPublicKey;
-			node["Comment"]		= strComment;
+		node["PublicKey"]	= strPublicKey;
+		node["Comment"]		= strComment;
 
-			ret.append(node);
-
-			more	= db->getNextRow();
-		}
-
-		db->endIterRows();
+		ret.append(node);
 	}
 
 	return ret;

src/UniqueNodeList.h

@@ -39,6 +39,8 @@ public:
 		vsReferral	= 'R',
 	} validatorSource;
 
+	typedef long score;
+
 private:
 	// Misc persistent information
 	boost::posix_time::ptime		mtpScoreUpdated;
@@ -68,6 +70,8 @@ private:
 	} scoreNode;
 
 	typedef boost::unordered_map<std::string,int> strIndex;
+	typedef std::pair<std::string,int> ipPort;
+	typedef boost::unordered_map<std::pair< std::string, int>, score>	epScore;
 
 	bool scoreRound(std::vector<scoreNode>& vsnNodes);
 	int iSourceScore(validatorSource vsWhy);
@@ -78,7 +82,7 @@ private:
 	boost::posix_time::ptime		mtpScoreStart;		// Time currently started scoring.
 	boost::asio::deadline_timer		mdtScoreTimer;		// Timer to start scoring.
 
-	void scoreNext();									// Update scoring timer.
+	void scoreNext(bool bNow);							// Update scoring timer.
 	void scoreCompute();
 	void scoreTimerHandler(const boost::system::error_code& err);
 
@@ -89,16 +93,17 @@ private:
 	boost::asio::deadline_timer		mdtFetchTimer;		// Timer to start fetching.
 
 	void fetchNext();
+	void fetchDirty();
 	void fetchFinish();
 	void fetchProcess(std::string strDomain);
 	void fetchTimerHandler(const boost::system::error_code& err);
 
 	void getValidatorsUrl(NewcoinAddress naNodePublic, section secSite);
-	void getIpsUrl(section secSite);
-	void responseIps(const std::string& strSite, const boost::system::error_code& err, const std::string strIpsFile);
+	void getIpsUrl(NewcoinAddress naNodePublic, section secSite);
+	void responseIps(const std::string& strSite, NewcoinAddress naNodePublic, const boost::system::error_code& err, const std::string strIpsFile);
 	void responseValidators(const std::string& strValidatorsUrl, NewcoinAddress naNodePublic, section secSite, const std::string& strSite, const boost::system::error_code& err, const std::string strValidatorsFile);
 
-	void processIps(const std::string& strSite, section::mapped_type* pmtVecStrIps);
+	void processIps(const std::string& strSite, NewcoinAddress naNodePublic, section::mapped_type* pmtVecStrIps);
 	void processValidators(const std::string& strSite, const std::string& strValidatorsSrc, NewcoinAddress naNodePublic, section::mapped_type* pmtVecStrValidators);
 
 	void processFile(const std::string strDomain, NewcoinAddress naNodePublic, section secSite);
@@ -118,6 +123,8 @@ public:
 	void nodeDefault(std::string strValidators);
 	void nodeReset();
 
+	void nodeScore();
+
 	Json::Value getUnlJson();
 };