Initial support for fetch newcoin.txt via https.

src/Application.cpp

@@ -45,12 +45,18 @@ Application::Application() :
 	mPeerDoor(NULL), mRPCDoor(NULL)
 {
 	theConfig.load();
-
 }
 
 extern const char *TxnDBInit[], *LedgerDBInit[], *WalletDBInit[], *HashNodeDBInit[], *NetNodeDBInit[];
 extern int TxnDBCount, LedgerDBCount, WalletDBCount, HashNodeDBCount, NetNodeDBCount;
 
+void Application::stop()
+{
+	mIOService.stop();
+
+    std::cerr << "Stopped: " << mIOService.stopped() << std::endl;
+}
+
 void Application::run()
 {
 	assert(mTxnDB==NULL);

src/Application.h

@@ -57,7 +57,6 @@ class Application
 
 	boost::asio::io_service mIOService;
 
-
 public:
 	Application();
 	~Application();
@@ -86,12 +85,11 @@ public:
 	//Serializer* getSerializer(){ return(mSerializer); }
 	//void setSerializer(Serializer* ser){ mSerializer=ser; }
 
-
 	void run();
-
+	void stop();
-	
 };
 
 extern Application* theApp;
 
 #endif
+// vim:ts=4

src/HttpsClient.cpp

@@ -0,0 +1,309 @@
+//
+// Fetch a web page via https.
+//
+
+#include "HttpsClient.h"
+
+#include <iostream>
+#include <boost/bind.hpp>
+#include <boost/date_time/posix_time/posix_time.hpp>
+#include <boost/smart_ptr/shared_ptr.hpp>
+#include <boost/system/error_code.hpp>
+
+#include <boost/regex.hpp>
+
+using namespace boost::system;
+using namespace boost::asio;
+
+HttpsClient::HttpsClient(
+    boost::asio::io_service& io_service,
+    const std::string strDomain,
+    const std::string strPath,
+    unsigned short port,
+    std::size_t responseMax
+    ) :
+		mCtx(boost::asio::ssl::context::sslv23),
+		mResolver(io_service),
+		mQuery(strDomain, boost::lexical_cast<std::string>(port),
+			ip::resolver_query_base::numeric_service|ip::resolver_query_base::numeric_service),
+		mSocketSsl(io_service, mCtx),
+		mResponse(responseMax),
+		mStrDomain(strDomain),
+		mDeadline(io_service)
+{
+    std::ostream		osRequest(&mRequest);
+
+    osRequest <<
+		"GET " << strPath << " HTTP/1.0\r\n"
+		"Host: " << mStrDomain << "\r\n"
+		"Accept: */*\r\n"						// YYY Do we need this line?
+		"Connection: close\r\n\r\n";
+}
+
+void HttpsClient::httpsGet(
+	boost::posix_time::time_duration timeout,
+	boost::function<void(const boost::system::error_code& ecResult, std::string& strData)> complete) {
+
+	mComplete	= complete;
+
+	mCtx.set_default_verify_paths(mShutdown);
+	if (!mShutdown)
+	{
+		std::cerr << "set_default_verify_paths: " << mShutdown.message() << std::endl;
+	}
+
+	if (!mShutdown)
+	{
+		mDeadline.expires_from_now(timeout, mShutdown);
+
+		std::cerr << "expires_from_now: " << mShutdown.message() << std::endl;
+	}
+
+	if (!mShutdown)
+	{
+		mDeadline.async_wait(
+			boost::bind(
+				&HttpsClient::handleDeadline,
+				shared_from_this(),
+				boost::asio::placeholders::error));
+	}
+
+    if (!mShutdown)
+    {
+		std::cerr << "Resolving: " << mStrDomain << std::endl;
+
+		mResolver.async_resolve(mQuery,
+			boost::bind(
+				&HttpsClient::handleResolve,
+				shared_from_this(),
+				boost::asio::placeholders::error,
+				boost::asio::placeholders::iterator));
+    }
+
+	if (mShutdown)
+		invokeComplete(mShutdown);
+}
+
+void HttpsClient::handleDeadline(const boost::system::error_code& ecResult)
+{
+	if (ecResult == boost::asio::error::operation_aborted)
+	{
+		// Timer canceled because deadline no longer needed.
+		std::cerr << "Deadline cancelled." << std::endl;
+
+		// Do nothing.
+	}
+	else if (ecResult)
+    {
+		std::cerr << "Deadline error: " << mStrDomain << ": " << ecResult.message() << std::endl;
+
+		// Can't do anything sound.
+		abort();
+    }
+    else
+    {
+		boost::system::error_code ec_shutdown;
+
+		std::cerr << "Deadline arrived." << std::endl;
+
+		// Mark us as shutting down.
+		// XXX Use our own error code.
+		mShutdown	= boost::system::error_code(errc::bad_address, system_category());
+
+		// Cancel any resolving.
+		mResolver.cancel();
+
+		// Stop the transaction.
+		mSocketSsl.shutdown(ec_shutdown);
+
+		if (ec_shutdown)
+		{
+			std::cerr << "Shutdown error: " << mStrDomain << ": " << ec_shutdown.message() << std::endl;
+		}
+    }
+}
+
+void HttpsClient::handleResolve(
+	const boost::system::error_code& ecResult,
+	boost::asio::ip::tcp::resolver::iterator itrEndpoint
+	)
+{
+	if (!mShutdown)
+		mShutdown	= ecResult;
+
+    if (mShutdown)
+    {
+		std::cerr << "Resolve error: " << mStrDomain << ": " << mShutdown.message() << std::endl;
+
+		invokeComplete(mShutdown);
+    }
+    else
+	{
+		std::cerr << "Resolve complete." << std::endl;
+
+		boost::asio::async_connect(
+			mSocketSsl.lowest_layer(),
+			itrEndpoint,
+			boost::bind(
+				&HttpsClient::handleConnect,
+				shared_from_this(),
+				boost::asio::placeholders::error));
+    }
+}
+
+void HttpsClient::handleConnect(const boost::system::error_code& ecResult)
+{
+	if (!mShutdown)
+		mShutdown	= ecResult;
+
+    if (mShutdown)
+    {
+		std::cerr << "Connect error: " << mShutdown.message() << std::endl;
+    }
+
+    if (!mShutdown)
+	{
+		std::cerr << "Connected." << std::endl;
+
+	    mSocketSsl.lowest_layer().set_option(boost::asio::ip::tcp::no_delay(true));
+	    mSocketSsl.set_verify_mode(boost::asio::ssl::verify_peer);
+
+		// XXX Verify semantics of RFC 2818 are what we want.
+	    mSocketSsl.set_verify_callback(boost::asio::ssl::rfc2818_verification(mStrDomain), mShutdown);
+
+	    if (mShutdown)
+		{
+			std::cerr << "set_verify_callback: " << mStrDomain << ": " << mShutdown.message() << std::endl;
+		}
+	}
+
+	if (!mShutdown)
+	{
+	    mSocketSsl.async_handshake(boost::asio::ssl::stream<boost::asio::ip::tcp::socket>::client,
+			boost::bind(&HttpsClient::handleRequest,
+				shared_from_this(),
+				boost::asio::placeholders::error));
+    }
+	else
+    {
+		invokeComplete(mShutdown);
+    }
+}
+
+void HttpsClient::handleRequest(const boost::system::error_code& ecResult)
+{
+	if (!mShutdown)
+		mShutdown	= ecResult;
+
+    if (mShutdown)
+    {
+		std::cerr << "Handshake error:" << mShutdown.message() << std::endl;
+
+		invokeComplete(mShutdown);
+    }
+	else
+	{
+		std::cerr << "SSL session started." << std::endl;
+
+		boost::asio::async_write(
+			mSocketSsl,
+			mRequest,
+			boost::bind(&HttpsClient::handleWrite,
+				shared_from_this(),
+				boost::asio::placeholders::error));
+    }
+}
+
+void HttpsClient::handleWrite(const boost::system::error_code& ecResult)
+{
+	if (!mShutdown)
+		mShutdown	= ecResult;
+
+    if (mShutdown)
+    {
+		std::cerr << "Write error: " << mShutdown.message() << std::endl;
+
+		invokeComplete(mShutdown);
+    }
+    else
+    {
+		std::cerr << "Wrote." << std::endl;
+
+		boost::asio::async_read(
+			mSocketSsl,
+			mResponse,
+			boost::asio::transfer_all(),
+			boost::bind(&HttpsClient::handleData,
+				shared_from_this(),
+				boost::asio::placeholders::error));
+    }
+}
+
+void HttpsClient::handleData(const boost::system::error_code& ecResult)
+{
+	if (!mShutdown)
+		mShutdown	= ecResult;
+
+    if (mShutdown && mShutdown != boost::asio::error::eof)
+    {
+		std::cerr << "Read error: " << mShutdown.message() << std::endl;
+
+		invokeComplete(mShutdown);
+    }
+    else
+    {
+		if (mShutdown)
+		{
+			std::cerr << "Complete." << std::endl;
+		}
+		else
+		{
+			// XXX According to boost example code, this is what we should expect for success.
+			std::cerr << "Complete, no eof." << std::endl;
+		}
+
+		parseData();
+    }
+}
+
+// Call cancel the deadline timer and invoke the completion routine.
+void HttpsClient::invokeComplete(const boost::system::error_code& ecResult, std::string strData)
+{
+	boost::system::error_code ecCancel;
+
+	(void) mDeadline.cancel(ecCancel);
+
+	if (ecCancel)
+	{
+		std::cerr << "Deadline cancel error: " << ecCancel.message() << std::endl;
+	}
+
+	mComplete(ecResult ? ecResult : ecCancel, strData);
+}
+
+void HttpsClient::parseData()
+{
+	// AHB How does this work?
+	// http://stackoverflow.com/questions/877652/copy-a-streambufs-contents-to-a-string
+	std::string		strData((std::istreambuf_iterator<char>(&mResponse)), std::istreambuf_iterator<char>());
+
+	// Match status code on a line.
+	boost::regex	reStatus("\\`HTTP/1\\S+ (\\d{3}) .*\\'");		// HTTP/1.1 200 OK
+
+	// Match body.
+	boost::regex	reBody("\\`(?:.*\\r\\n\\r\\n){1,1}(.*)\\'");
+
+	boost::smatch	smMatch;
+
+	bool	bMatch	= boost::regex_match(strData, smMatch, reStatus)
+						&& !smMatch[1].compare("200")
+						&& boost::regex_match(strData, smMatch, reBody);
+
+	std::cerr << "Match: " << bMatch << std::endl;
+	std::cerr << "Body:" << smMatch[1] << std::endl;
+
+	boost::system::error_code	noErr;
+
+	invokeComplete(noErr, smMatch[1]);
+}
+// vim:ts=4

src/HttpsClient.h

@@ -0,0 +1,65 @@
+#ifndef _HTTPS_CLIENT_
+#define _HTTPS_CLIENT_
+
+#include <string>
+
+#include <boost/asio.hpp>
+#include <boost/asio/ssl.hpp>
+#include <boost/date_time/posix_time/posix_time_types.hpp>
+#include <boost/function.hpp>
+#include <boost/shared_ptr.hpp>
+#include <boost/enable_shared_from_this.hpp>
+
+//
+// Async https client.
+//
+
+class HttpsClient : public boost::enable_shared_from_this<HttpsClient>
+{
+private:
+    boost::asio::ssl::context								mCtx;
+    boost::asio::ip::tcp::resolver							mResolver;
+    boost::asio::ip::tcp::resolver::query					mQuery;
+    boost::asio::ssl::stream<boost::asio::ip::tcp::socket>	mSocketSsl;
+    boost::asio::streambuf									mRequest;
+    boost::asio::streambuf									mResponse;
+    std::string												mStrDomain;
+    boost::function<void(const boost::system::error_code& ecResult, std::string& strData)> mComplete;
+
+	boost::asio::deadline_timer								mDeadline;
+
+	// If not success, we are shutting down.
+	boost::system::error_code								mShutdown;
+
+	void handleDeadline(const boost::system::error_code& ecResult);
+
+    void handleResolve(
+		const boost::system::error_code& ecResult,
+		boost::asio::ip::tcp::resolver::iterator endpoint_iterator
+		);
+
+    void handleConnect(const boost::system::error_code& ecResult);
+	void handleRequest(const boost::system::error_code& ecResult);
+    void handleWrite(const boost::system::error_code& ecResult);
+    void handleData(const boost::system::error_code& ecResult);
+
+	void invokeComplete(const boost::system::error_code& ecResult, std::string strData = "");
+
+	void parseData();
+
+public:
+
+    HttpsClient(
+		boost::asio::io_service& io_service,
+		const std::string strDomain,
+		const std::string strPath,
+		unsigned short port,
+		std::size_t responseMax
+		);
+
+	void httpsGet(
+		boost::posix_time::time_duration timeout,
+		boost::function<void(const boost::system::error_code& ecResult, std::string& strData)> complete);
+};
+#endif
+// vim:ts=4

src/RPCServer.cpp

@@ -550,7 +550,15 @@ Json::Value RPCServer::doUnlDelete(Json::Value& params) {
 }
 
 Json::Value RPCServer::doUnlFetch(Json::Value& params) {
-	return "not implemented";
+	if(params.size() == 1)
+	{
+		std::string	strDomain=params[0u].asString();
+
+		theApp->getUNL().fetchNode(strDomain);
+
+		return "fetching domain";
+	}
+	else return "invalid params";
 }
 
 Json::Value RPCServer::doUnlList(Json::Value& params) {
@@ -562,6 +570,7 @@ Json::Value RPCServer::doUnlReset(Json::Value& params) {
 	if(!params.size())
 	{
 		theApp->getUNL().reset();
+
 		return "removing nodes";
 	}
 	else return "invalid params";
@@ -573,7 +582,8 @@ Json::Value RPCServer::doCommand(const std::string& command, Json::Value& params
 
 	if(command== "stop")
 	{
-		mSocket.get_io_service().stop();
+		theApp->stop();
+
 		return "newcoin server stopping";
 	}
 

src/UniqueNodeList.cpp

@@ -1,13 +1,94 @@
 #include "UniqueNodeList.h"
 #include "Application.h"
 #include "Conversion.h"
+#include "HttpsClient.h"
 
-void UniqueNodeList::addNode(NewcoinAddress nodePublic, std::string strComment)
+#include <boost/asio.hpp>
+#include <boost/bind.hpp>
+#include <boost/mem_fn.hpp>
+
+UniqueNodeList::UniqueNodeList() : mFetchActive(0)
+{
+}
+
+void UniqueNodeList::fetchResponse(const boost::system::error_code& err, std::string strResponse)
+{
+	std::cerr << "Fetch complete." << std::endl;
+	std::cerr << "Error: " << err.message() << std::endl;
+
+    // std::cerr << &response << std::endl;
+	// HTTP/1.1 200 OK
+
+	{
+		boost::mutex::scoped_lock sl(mFetchLock);
+		mFetchActive--;
+	}
+
+	std::cerr << "Fetch active: " << mFetchActive << std::endl;
+	fetchNext();
+}
+
+// Get the newcoin.txt and process it.
+void UniqueNodeList::fetchProcess(std::string strDomain)
+{
+	std::cerr << "Fetching '" NODE_FILE_NAME "' from '" << strDomain << "'." << std::endl;
+
+	{
+		boost::mutex::scoped_lock sl(mFetchLock);
+		mFetchActive++;
+	}
+
+    boost::shared_ptr<HttpsClient> client(new HttpsClient(
+		theApp->getIOService(),
+		strDomain,
+		NODE_FILE_PATH,
+		443,
+		NODE_FILE_BYTES_MAX
+		));
+
+	client->httpsGet(
+		boost::posix_time::seconds(NODE_FETCH_SECONDS),
+		boost::bind(&UniqueNodeList::fetchResponse, this, _1, _2));
+}
+
+// Try to process the next fetch.
+void UniqueNodeList::fetchNext()
+{
+	bool		work;
+	std::string	strDomain;
+	{
+		boost::mutex::scoped_lock sl(mFetchLock);
+		work	= mFetchActive != NODE_FETCH_JOBS && !mFetchPending.empty();
+		if (work) {
+			strDomain	= mFetchPending.front();
+			mFetchPending.pop_front();
+		}
+	}
+
+	if (!strDomain.empty())
+	{
+		fetchProcess(strDomain);
+	}
+}
+
+// Get newcoin.txt from a domain's web server.
+void UniqueNodeList::fetchNode(std::string strDomain)
+{
+	{
+		boost::mutex::scoped_lock sl(mFetchLock);
+
+		mFetchPending.push_back(strDomain);
+	}
+
+	fetchNext();
+}
+
+void UniqueNodeList::addNode(NewcoinAddress naNodePublic, std::string strComment)
 {
 	Database* db=theApp->getWalletDB()->getDB();
 
-	std::string strHanko	    = nodePublic.humanHanko();
+	std::string strHanko		= naNodePublic.humanHanko();
-	std::string strPublicKey   = nodePublic.humanNodePublic();
+	std::string strPublicKey	= naNodePublic.humanNodePublic();
 	std::string strTmp;
 
 	std::string strSql="INSERT INTO TrustedNodes (Hanko,PublicKey,Comment) values (";
@@ -25,11 +106,11 @@ void UniqueNodeList::addNode(NewcoinAddress nodePublic, std::string strComment)
 	db->executeSQL(strSql.c_str());
 }
 
-void UniqueNodeList::removeNode(NewcoinAddress hanko)
+void UniqueNodeList::removeNode(NewcoinAddress naHanko)
 {
 	Database* db=theApp->getWalletDB()->getDB();
 
-	std::string strHanko	= hanko.humanHanko();
+	std::string strHanko	= naHanko.humanHanko();
 	std::string strTmp;
 
 	std::string strSql		= "DELETE FROM TrustedNodes where Hanko=";

src/UniqueNodeList.h

@@ -3,22 +3,48 @@
 
 #include "../json/value.h"
 
-#include "uint256.h"
 #include "NewcoinAddress.h"
 
+#include "HttpsClient.h"
+
+#include <boost/thread/mutex.hpp>
+#include <boost/container/deque.hpp>
+
+// Guarantees minimum thoughput of 1 node per second.
+#define NODE_FETCH_JOBS			10
+#define NODE_FETCH_SECONDS		10
+#define NODE_FILE_BYTES_MAX		(50<<10)	// 50k
+#define NODE_FILE_NAME			"newcoin.txt"
+#define NODE_FILE_PATH			"/" NODE_FILE_NAME
+
 class UniqueNodeList
 {
+private:
+	void fetchResponse(const boost::system::error_code& err, std::string strResponse);
+
 	// hanko to public key
 	//std::map<uint160, uint256> mUNL;
+
+	boost::mutex							mFetchLock;
+	int										mFetchActive;	// count of active fetches
+	boost::container::deque<std::string>	mFetchPending;
+
+	void fetchNext();
+	void fetchProcess(std::string strDomain);
+
 public:
-	void addNode(NewcoinAddress nodePublic, std::string strComment);
+	UniqueNodeList();
-	void removeNode(NewcoinAddress hanko);
+
+	void addNode(NewcoinAddress naNodePublic, std::string strComment);
+	void fetchNode(std::string strDomain);
+	void removeNode(NewcoinAddress naHanko);
 	void reset();
 
-	// 0- we don't care, 1- we care and is valid, 2-invalid signature
+	// 2- we don't care, 1- we care and is valid, 2-invalid signature
 //	int checkValid(newcoin::Validation& valid);
 
 	Json::Value getUnlJson();
 };
 
 #endif
+// vim:ts=4