Split up and refactor HashedObject

modules/ripple_main/ripple_main.cpp

@@ -67,10 +67,10 @@
 //------------------------------------------------------------------------------
 
 // VFALCO: TODO, prepare a unity header for LevelDB
-#ifdef USE_LEVELDB
 #include "leveldb/cache.h"
 #include "leveldb/filter_policy.h"
-#endif
+#include "leveldb/db.h"
+#include "leveldb/write_batch.h"
 
 //------------------------------------------------------------------------------
 
@@ -93,6 +93,7 @@
 #include "src/cpp/ripple/Ledger.h"
 #include "src/cpp/ripple/SerializedValidation.h"
 #include "src/cpp/database/SqliteDatabase.h"
+#include "src/cpp/ripple/LoadManager.h"
 
 //------------------------------------------------------------------------------
 //
@@ -105,6 +106,9 @@
 #include "src/cpp/ripple/ripple_ProofOfWork.h"
 #include "src/cpp/ripple/ripple_Job.h"
 #include "src/cpp/ripple/ripple_JobQueue.h"
+#include "src/cpp/ripple/ripple_UptimeTimerAdapter.h"
+#include "src/cpp/ripple/ripple_HashedObject.h"
+#include "src/cpp/ripple/ripple_HashedObjectStore.h"
 
 #include "src/cpp/ripple/ripple_IFeatures.h"
 #include "src/cpp/ripple/ripple_IFeeVote.h"
@@ -139,7 +143,6 @@
 #include "src/cpp/ripple/ChangeTransactor.h"
 #include "src/cpp/ripple/HTTPRequest.h"
 #include "src/cpp/ripple/HashPrefixes.h"
-#include "src/cpp/ripple/HashedObject.h"
 #include "src/cpp/ripple/HttpsClient.h"
 #include "src/cpp/ripple/Ledger.h"
 #include "src/cpp/ripple/LedgerAcquire.h"
@@ -149,7 +152,6 @@
 #include "src/cpp/ripple/LedgerMaster.h"
 #include "src/cpp/ripple/LedgerProposal.h"
 #include "src/cpp/ripple/LedgerTiming.h"
-#include "src/cpp/ripple/LoadManager.h"
 #include "src/cpp/ripple/NetworkOPs.h"
 #include "src/cpp/ripple/NicknameState.h"
 #include "src/cpp/ripple/Offer.h"
@@ -224,7 +226,6 @@ static DH* handleTmpDh(SSL* ssl, int is_export, int iKeyLength)
 #include "src/cpp/ripple/ChangeTransactor.cpp" // no log
 #include "src/cpp/ripple/Contract.cpp" // no log
 #include "src/cpp/ripple/DBInit.cpp"
-#include "src/cpp/ripple/HashedObject.cpp"
 #include "src/cpp/ripple/HTTPRequest.cpp"
 #include "src/cpp/ripple/HttpsClient.cpp"
 #include "src/cpp/ripple/Interpreter.cpp" // no log
@@ -288,10 +289,12 @@ static DH* handleTmpDh(SSL* ssl, int is_export, int iKeyLength)
 
 // Refactored sources
 
-#include "src/cpp/ripple/ripple_Config.cpp" // no log
+#include "src/cpp/ripple/ripple_Config.cpp"
 #include "src/cpp/ripple/ripple_DatabaseCon.cpp"
 #include "src/cpp/ripple/ripple_Features.cpp"
 #include "src/cpp/ripple/ripple_FeeVote.cpp"
+#include "src/cpp/ripple/ripple_HashedObject.cpp"
+#include "src/cpp/ripple/ripple_HashedObjectStore.cpp"
 #include "src/cpp/ripple/ripple_HashRouter.cpp"
 #include "src/cpp/ripple/ripple_Job.cpp"
 #include "src/cpp/ripple/ripple_JobQueue.cpp"

newcoin.vcxproj

@@ -758,6 +758,18 @@
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
     </ClCompile>
+    <ClCompile Include="src\cpp\ripple\ripple_HashedObject.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
+    </ClCompile>
+    <ClCompile Include="src\cpp\ripple\ripple_HashedObjectStore.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
+    </ClCompile>
     <ClCompile Include="src\cpp\ripple\ripple_Peers.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
@@ -776,12 +788,6 @@
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
     </ClCompile>
-    <ClCompile Include="src\cpp\ripple\HashedObject.cpp">
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
-    </ClCompile>
     <ClCompile Include="src\cpp\ripple\HTTPRequest.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
@@ -1645,9 +1651,10 @@
     <ClInclude Include="src\cpp\ripple\CanonicalTXSet.h" />
     <ClInclude Include="src\cpp\ripple\ChangeTransactor.h" />
     <ClInclude Include="src\cpp\ripple\ripple_Config.h" />
+    <ClInclude Include="src\cpp\ripple\ripple_HashedObject.h" />
+    <ClInclude Include="src\cpp\ripple\ripple_HashedObjectStore.h" />
     <ClInclude Include="src\cpp\ripple\ripple_IPeers.h" />
     <ClInclude Include="src\cpp\ripple\Contract.h" />
-    <ClInclude Include="src\cpp\ripple\HashedObject.h" />
     <ClInclude Include="src\cpp\ripple\HashPrefixes.h" />
     <ClInclude Include="src\cpp\ripple\HTTPRequest.h" />
     <ClInclude Include="src\cpp\ripple\HttpsClient.h" />
@@ -1692,6 +1699,7 @@
     <ClInclude Include="src\cpp\ripple\ripple_Peer.h" />
     <ClInclude Include="src\cpp\ripple\ripple_ProofOfWork.h" />
     <ClInclude Include="src\cpp\ripple\ripple_IProofOfWorkFactory.h" />
+    <ClInclude Include="src\cpp\ripple\ripple_UptimeTimerAdapter.h" />
     <ClInclude Include="src\cpp\ripple\RPC.h" />
     <ClInclude Include="src\cpp\ripple\RPCDoor.h" />
     <ClInclude Include="src\cpp\ripple\RPCErr.h" />

newcoin.vcxproj.filters

@@ -462,9 +462,6 @@
     <ClCompile Include="src\cpp\ripple\Contract.cpp">
       <Filter>1. Modules\ripple_main\_unfactored\contracts</Filter>
     </ClCompile>
-    <ClCompile Include="src\cpp\ripple\HashedObject.cpp">
-      <Filter>1. Modules\ripple_main\_unfactored\types</Filter>
-    </ClCompile>
     <ClCompile Include="src\cpp\ripple\ParameterTable.cpp">
       <Filter>1. Modules\ripple_main\_unfactored\types</Filter>
     </ClCompile>
@@ -813,6 +810,12 @@
     <ClCompile Include="src\cpp\ripple\ripple_Peers.cpp">
       <Filter>1. Modules\ripple_main\refactored</Filter>
     </ClCompile>
+    <ClCompile Include="src\cpp\ripple\ripple_HashedObjectStore.cpp">
+      <Filter>1. Modules\ripple_main\refactored</Filter>
+    </ClCompile>
+    <ClCompile Include="src\cpp\ripple\ripple_HashedObject.cpp">
+      <Filter>1. Modules\ripple_main\refactored</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="database\sqlite3ext.h">
@@ -1130,9 +1133,6 @@
     <ClInclude Include="src\cpp\ripple\Contract.h">
       <Filter>1. Modules\ripple_main\_unfactored\contracts</Filter>
     </ClInclude>
-    <ClInclude Include="src\cpp\ripple\HashedObject.h">
-      <Filter>1. Modules\ripple_main\_unfactored\types</Filter>
-    </ClInclude>
     <ClInclude Include="src\cpp\ripple\HashPrefixes.h">
       <Filter>1. Modules\ripple_main\_unfactored\types</Filter>
     </ClInclude>
@@ -1511,6 +1511,15 @@
     <ClInclude Include="src\cpp\ripple\ripple_IPeers.h">
       <Filter>1. Modules\ripple_main\refactored</Filter>
     </ClInclude>
+    <ClInclude Include="src\cpp\ripple\ripple_HashedObjectStore.h">
+      <Filter>1. Modules\ripple_main\refactored</Filter>
+    </ClInclude>
+    <ClInclude Include="src\cpp\ripple\ripple_UptimeTimerAdapter.h">
+      <Filter>1. Modules\ripple_main\refactored</Filter>
+    </ClInclude>
+    <ClInclude Include="src\cpp\ripple\ripple_HashedObject.h">
+      <Filter>1. Modules\ripple_main\refactored</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <None Include="SConstruct" />

src/cpp/ripple/Application.cpp

@@ -1,23 +1,4 @@
 
-/*
-#include "Application.h"
-
-#include "leveldb/cache.h"
-#include "leveldb/filter_policy.h"
-
-#include "AcceptedLedger.h"
-#include "PeerDoor.h"
-#include "RPCDoor.h"
-
-#include "../database/SqliteDatabase.h"
-
-#include <iostream>
-
-#include <boost/bind.hpp>
-#include <boost/filesystem.hpp>
-#include <boost/thread.hpp>
-*/
-
 // VFALCO: TODO Replace these with beast "unsigned long long" generators
 #define SYSTEM_CURRENCY_GIFT		1000ull
 #define SYSTEM_CURRENCY_USERS		100000000ull
@@ -58,10 +39,7 @@ Application::Application ()
     , mNetNodeDB (NULL)
     , mPathFindDB (NULL)
     , mHashNodeDB (NULL)
-    // VFALCO: TODO eliminate USE_LEVELDB macro
-#ifdef USE_LEVELDB
 	, mHashNodeLDB (NULL)
-#endif
 	, mPeerDoor (NULL)
     , mRPCDoor (NULL)
     , mWSPublicDoor (NULL)

src/cpp/ripple/Application.h

@@ -33,6 +33,7 @@ class IPeers;
 
 class RPCDoor;
 class PeerDoor;
+
 typedef TaggedCache <uint256, std::vector <unsigned char>, UptimeTimerAdapter> NodeCache;
 typedef TaggedCache <uint256, SLE, UptimeTimerAdapter> SLECache;
 
@@ -92,6 +93,10 @@ public:
 	void sweep();
 
 private:	
+	void updateTables (bool);
+	void startNewLedger ();
+	bool loadOldLedger (const std::string&);
+
     boost::asio::io_service	mIOService;
     boost::asio::io_service mAuxService;
 	boost::asio::io_service::work mIOWork;
@@ -140,10 +145,6 @@ private:
 	boost::recursive_mutex	mPeerMapLock;
 
 	volatile bool			mShutdown;
-
-	void updateTables(bool);
-	void startNewLedger();
-	bool loadOldLedger(const std::string&);
 };
 
 extern Application* theApp;

src/cpp/ripple/HashedObject.h

@@ -1,114 +0,0 @@
-#ifndef __HASHEDOBJECT__
-#define __HASHEDOBJECT__
-
-#include <vector>
-
-#include <boost/thread/mutex.hpp>
-#include <boost/thread/condition_variable.hpp>
-
-// VFALCO: TODO, Move this to someplace sensible!!
-// Adapter to furnish uptime information to KeyCache via UptimeTimer singleton
-struct UptimeTimerAdapter
-{
-	inline static int getElapsedSeconds ()
-	{
-		return UptimeTimer::getInstance().getElapsedSeconds ();
-	}
-};
-
-
-
-DEFINE_INSTANCE(HashedObject);
-
-class Job;
-
-enum HashedObjectType
-{
-	hotUNKNOWN = 0,
-	hotLEDGER = 1,
-	hotTRANSACTION = 2,
-	hotACCOUNT_NODE = 3,
-	hotTRANSACTION_NODE = 4
-};
-
-class HashedObject : private IS_INSTANCE(HashedObject)
-{
-public:
-	typedef boost::shared_ptr<HashedObject> pointer;
-
-	HashedObjectType			mType;
-	uint256						mHash;
-	uint32						mLedgerIndex;
-	std::vector<unsigned char>	mData;
-
-	HashedObject(HashedObjectType type, uint32 index, const std::vector<unsigned char>& data, const uint256& hash) :
-		mType(type), mHash(hash), mLedgerIndex(index), mData(data) { ; }
-
-	HashedObject(HashedObjectType type, uint32 index, const unsigned char *data, int dlen, const uint256& hash) :
-		mType(type), mHash(hash), mLedgerIndex(index), mData(data, data + dlen) { ; }
-
-	const std::vector<unsigned char>& getData() const	{ return mData; }
-	const uint256& getHash() const						{ return mHash; }
-	HashedObjectType getType() const					{ return mType; }
-	uint32 getIndex() const								{ return mLedgerIndex; }
-};
-
-class HashedObjectStore
-{
-public:
-	HashedObjectStore(int cacheSize, int cacheAge);
-
-	bool isLevelDB()		{ return mLevelDB; }
-
-	float getCacheHitRate()	{ return mCache.getHitRate(); }
-
-	bool store(HashedObjectType type, uint32 index, const std::vector<unsigned char>& data,
-		const uint256& hash)
-	{
-		if (mLevelDB)
-			return storeLevelDB(type, index, data, hash);
-		return storeSQLite(type, index, data, hash);
-	}
-
-	HashedObject::pointer retrieve(const uint256& hash)
-	{
-		if (mLevelDB)
-			return retrieveLevelDB(hash);
-		return retrieveSQLite(hash);
-	}
-
-	bool storeSQLite(HashedObjectType type, uint32 index, const std::vector<unsigned char>& data,
-		const uint256& hash);
-	HashedObject::pointer retrieveSQLite(const uint256& hash);
-	void bulkWriteSQLite(Job&);
-
-	bool storeLevelDB(HashedObjectType type, uint32 index, const std::vector<unsigned char>& data,
-		const uint256& hash);
-	HashedObject::pointer retrieveLevelDB(const uint256& hash);
-	void bulkWriteLevelDB(Job&);
-
-
-	void waitWrite();
-	void tune(int size, int age);
-	void sweep() { mCache.sweep(); mNegativeCache.sweep(); }
-	int getWriteLoad();
-
-	int import(const std::string& fileName);
-
-private:
-	TaggedCache<uint256, HashedObject, UptimeTimerAdapter>	mCache;
-	KeyCache <uint256, UptimeTimerAdapter> mNegativeCache;
-
-	boost::mutex				mWriteMutex;
-	boost::condition_variable	mWriteCondition;
-	int							mWriteGeneration;
-	int							mWriteLoad;
-
-	std::vector< boost::shared_ptr<HashedObject> > mWriteSet;
-	bool mWritePending;
-	bool mLevelDB;
-	bool mEphemeralDB;
-};
-
-#endif
-// vim:ts=4

src/cpp/ripple/HttpsClient.cpp

@@ -2,32 +2,17 @@
 // Fetch a web page via https.
 //
 
-#include "HttpsClient.h"
-
-#include <iostream>
-#include <boost/date_time/posix_time/posix_time.hpp>
-#include <boost/regex.hpp>
-#include <boost/smart_ptr/shared_ptr.hpp>
-#include <boost/system/error_code.hpp>
-
-#define CLIENT_MAX_HEADER (32*1024)
-
 SETUP_LOG (HttpsClient)
 
-using namespace boost::system;
+HttpsClient::HttpsClient (boost::asio::io_service& io_service,
-using namespace boost::asio;
-
-HttpsClient::HttpsClient(
-    boost::asio::io_service& io_service,
                           const unsigned short port,
-    std::size_t responseMax
+                          std::size_t responseMax)
-    ) :
+    : mSocket (io_service, theConfig.SSL_CONTEXT)
-		mSocket(io_service, theConfig.SSL_CONTEXT),
+    , mResolver (io_service)
-		mResolver(io_service),
+    , mHeader (maxClientHeaderBytes)
-		mHeader(CLIENT_MAX_HEADER),
+    , mPort (port)
-		mPort(port),
+    , mResponseMax (responseMax)
-		mResponseMax(responseMax),
+    , mDeadline (io_service)
-		mDeadline(io_service)
 {
 	if (!theConfig.SSL_VERIFY)
 		mSocket.SSLSocket().set_verify_mode (boost::asio::ssl::verify_none);
@@ -82,8 +67,11 @@ void HttpsClient::httpsNext()
 {
 	WriteLog (lsTRACE, HttpsClient) << "Fetch: " << mDeqSites[0];
 
-    boost::shared_ptr<boost::asio::ip::tcp::resolver::query>	query(new boost::asio::ip::tcp::resolver::query(mDeqSites[0], boost::lexical_cast<std::string>(mPort),
+    boost::shared_ptr <boost::asio::ip::tcp::resolver::query> query (
-			ip::resolver_query_base::numeric_service));
+        new boost::asio::ip::tcp::resolver::query (
+            mDeqSites[0],
+            boost::lexical_cast <std::string>(mPort),
+			boost::asio::ip::resolver_query_base::numeric_service));
 	mQuery	= query;
 
 	mDeadline.expires_from_now(mTimeout, mShutdown);
@@ -477,7 +465,7 @@ void HttpsClient::sendSMS(boost::asio::io_service& io_service, const std::string
 		if (iPort < 0)
 			iPort = bSSL ? 443 : 80;
 
-		boost::shared_ptr<HttpsClient> client(new HttpsClient(io_service, iPort, CLIENT_MAX_HEADER));
+		boost::shared_ptr<HttpsClient> client(new HttpsClient(io_service, iPort, maxClientHeaderBytes));
 
 		client->httpsGet(bSSL, deqSites, strURI, boost::posix_time::seconds(SMS_TIMEOUT),
 			BIND_TYPE(&responseSMS, P_1, P_2, P_3));

src/cpp/ripple/HttpsClient.h

@@ -18,54 +18,7 @@
 
 class HttpsClient : public boost::enable_shared_from_this<HttpsClient>
 {
-private:
-	typedef boost::shared_ptr<HttpsClient> pointer;
-
-	bool														mSSL;
-    AutoSocket													mSocket;
-    boost::asio::ip::tcp::resolver								mResolver;
-	boost::shared_ptr<boost::asio::ip::tcp::resolver::query>	mQuery;
-	boost::asio::streambuf										mRequest;
-	boost::asio::streambuf										mHeader;
-    boost::asio::streambuf										mResponse;
-    std::string													mBody;
-	const unsigned short										mPort;
-	int															mResponseMax;
-	int															mStatus;
-	FUNCTION_TYPE<void(boost::asio::streambuf& sb, const std::string& strHost)>			mBuild;
-    FUNCTION_TYPE<bool(const boost::system::error_code& ecResult, int iStatus, const std::string& strData)>	mComplete;
-
-	boost::asio::deadline_timer									mDeadline;
-
-	// If not success, we are shutting down.
-	boost::system::error_code									mShutdown;
-
-	std::deque<std::string>										mDeqSites;
-	boost::posix_time::time_duration							mTimeout;
-
-	void handleDeadline(const boost::system::error_code& ecResult);
-
-    void handleResolve(const boost::system::error_code& ecResult, boost::asio::ip::tcp::resolver::iterator endpoint_iterator);
-
-    void handleConnect(const boost::system::error_code& ecResult);
-
-	void handleRequest(const boost::system::error_code& ecResult);
-
-    void handleWrite(const boost::system::error_code& ecResult, std::size_t bytes_transferred);
-
-    void handleHeader(const boost::system::error_code& ecResult, std::size_t bytes_transferred);
-
-    void handleData(const boost::system::error_code& ecResult, std::size_t bytes_transferred);
-
-	void handleShutdown(const boost::system::error_code& ecResult);
-
-	void httpsNext();
-
-	void invokeComplete(const boost::system::error_code& ecResult, int iStatus = 0, const std::string& strData = "");
-
-	void makeGet(const std::string& strPath, boost::asio::streambuf& sb, const std::string& strHost);
 public:
-
     HttpsClient(
 		boost::asio::io_service& io_service,
 		const unsigned short port,
@@ -117,6 +70,56 @@ public:
 		FUNCTION_TYPE<bool(const boost::system::error_code& ecResult, int iStatus, const std::string& strData)> complete);
 
 	static void sendSMS(boost::asio::io_service& io_service, const std::string& strText);
+
+private:
+    static const int maxClientHeaderBytes = 32 * 1024;
+
+    typedef boost::shared_ptr<HttpsClient> pointer;
+
+	bool														mSSL;
+    AutoSocket													mSocket;
+    boost::asio::ip::tcp::resolver								mResolver;
+	boost::shared_ptr<boost::asio::ip::tcp::resolver::query>	mQuery;
+	boost::asio::streambuf										mRequest;
+	boost::asio::streambuf										mHeader;
+    boost::asio::streambuf										mResponse;
+    std::string													mBody;
+	const unsigned short										mPort;
+	int															mResponseMax;
+	int															mStatus;
+	FUNCTION_TYPE<void(boost::asio::streambuf& sb, const std::string& strHost)>			mBuild;
+    FUNCTION_TYPE<bool(const boost::system::error_code& ecResult, int iStatus, const std::string& strData)>	mComplete;
+
+	boost::asio::deadline_timer									mDeadline;
+
+	// If not success, we are shutting down.
+	boost::system::error_code									mShutdown;
+
+	std::deque<std::string>										mDeqSites;
+	boost::posix_time::time_duration							mTimeout;
+
+	void handleDeadline(const boost::system::error_code& ecResult);
+
+    void handleResolve(const boost::system::error_code& ecResult, boost::asio::ip::tcp::resolver::iterator endpoint_iterator);
+
+    void handleConnect(const boost::system::error_code& ecResult);
+
+	void handleRequest(const boost::system::error_code& ecResult);
+
+    void handleWrite(const boost::system::error_code& ecResult, std::size_t bytes_transferred);
+
+    void handleHeader(const boost::system::error_code& ecResult, std::size_t bytes_transferred);
+
+    void handleData(const boost::system::error_code& ecResult, std::size_t bytes_transferred);
+
+	void handleShutdown(const boost::system::error_code& ecResult);
+
+	void httpsNext();
+
+	void invokeComplete(const boost::system::error_code& ecResult, int iStatus = 0, const std::string& strData = "");
+
+	void makeGet(const std::string& strPath, boost::asio::streambuf& sb, const std::string& strHost);
 };
+
 #endif
 // vim:ts=4

src/cpp/ripple/Ledger.h

@@ -1,13 +1,9 @@
-#ifndef __LEDGER__
+#ifndef RIPPLE_LEDGER_H
-#define __LEDGER__
+#define RIPPLE_LEDGER_H
 
-#include <map>
+// VFALCO: TODO Get this include out of here!
-#include <list>
+#include "ripple_HashedObject.h"
 
-#include <boost/shared_ptr.hpp>
-#include <boost/unordered_set.hpp>
-#include <boost/enable_shared_from_this.hpp>
-#include <boost/date_time/posix_time/posix_time.hpp>
 
 #include "Transaction.h"
 #include "TransactionMeta.h"
@@ -15,6 +11,8 @@
 #include "NicknameState.h"
 #include "SHAMap.h"
 
+class Job;
+
 enum LedgerStateParms
 {
 	lepNONE			= 0,	// no special flags

src/cpp/ripple/OrderBook.h

@@ -2,26 +2,14 @@
 #ifndef ORDERBOOK_H
 #define ORDERBOOK_H
 
-
 #include "SerializedLedger.h"
 #include "NetworkOPs.h"
-#include <boost/shared_ptr.hpp>
 
 /*
 	Encapsulates the SLE for an orderbook
 */
 class OrderBook
 {
-	uint256 mBookBase;
-
-	uint160 mCurrencyIn;
-	uint160 mCurrencyOut;
-	uint160 mIssuerIn;
-	uint160 mIssuerOut;
-
-	//SerializedLedgerEntry::pointer	mLedgerEntry;
-	OrderBook(SerializedLedgerEntry::ref ledgerEntry);	// For accounts in a ledger
-
 public:
 	typedef boost::shared_ptr<OrderBook> pointer;
 	typedef const boost::shared_ptr<OrderBook>& ref;
@@ -42,6 +30,17 @@ public:
 
 	// looks through the best offers to see how much it would cost to take the given amount
 	STAmount& getTakePrice(STAmount& takeAmount);
+
+private:
+	uint256 mBookBase;
+
+	uint160 mCurrencyIn;
+	uint160 mCurrencyOut;
+	uint160 mIssuerIn;
+	uint160 mIssuerOut;
+
+	//SerializedLedgerEntry::pointer	mLedgerEntry;
+	OrderBook(SerializedLedgerEntry::ref ledgerEntry);	// For accounts in a ledger
 };
 
 #endif

src/cpp/ripple/OrderBookDB.h

@@ -25,28 +25,20 @@ typedef std::pair<uint160, uint160> currencyIssuer_ct; // C++ defect 106
 
 class BookListeners
 {
-	boost::unordered_map<uint64, InfoSub::wptr> mListeners;
-	boost::recursive_mutex mLock;
-
 public:
 	typedef boost::shared_ptr<BookListeners> pointer;
 
 	void addSubscriber(InfoSub::ref sub);
 	void removeSubscriber(uint64 sub);
 	void publish(Json::Value& jvObj);
+
+private:
+	boost::unordered_map<uint64, InfoSub::wptr> mListeners;
+	boost::recursive_mutex mLock;
 };
 
 class OrderBookDB
 {
-	boost::unordered_map< currencyIssuer_t, std::vector<OrderBook::pointer> > mSourceMap;	// by ci/ii
-	boost::unordered_map< currencyIssuer_t, std::vector<OrderBook::pointer> > mDestMap;		// by co/io
-
-	// issuerPays, issuerGets, currencyPays, currencyGets
-	std::map<uint160, std::map<uint160, std::map<uint160, std::map<uint160, BookListeners::pointer> > > > mListeners;
-
-	uint32 mSeq;
-	boost::recursive_mutex mLock;
-
 public:
 	OrderBookDB();
 	void setup(Ledger::ref ledger);
@@ -60,11 +52,23 @@ public:
 
 	BookListeners::pointer getBookListeners(const uint160& currencyPays, const uint160& currencyGets,
 		const uint160& issuerPays, const uint160& issuerGets);
+
     BookListeners::pointer makeBookListeners(const uint160& currencyPays, const uint160& currencyGets,
 		const uint160& issuerPays, const uint160& issuerGets);
 
 	// see if this txn effects any orderbook
 	void processTxn(Ledger::ref ledger, const ALTransaction& alTx, Json::Value& jvObj);
+
+private:
+	boost::unordered_map< currencyIssuer_t, std::vector<OrderBook::pointer> > mSourceMap;	// by ci/ii
+	boost::unordered_map< currencyIssuer_t, std::vector<OrderBook::pointer> > mDestMap;		// by co/io
+
+	// issuerPays, issuerGets, currencyPays, currencyGets
+	std::map<uint160, std::map<uint160, std::map<uint160, std::map<uint160, BookListeners::pointer> > > > mListeners;
+
+	uint32 mSeq;
+	boost::recursive_mutex mLock;
+
 };
 
 #endif

src/cpp/ripple/SHAMap.cpp

@@ -1,6 +1,4 @@
 
-#include "SHAMap.h"
-
 #include <stack>
 
 #include <boost/foreach.hpp>
@@ -9,9 +7,6 @@
 #include <boost/test/unit_test.hpp>
 #include <iostream>
 
-#include "SHAMap.h"
-#include "Application.h"
-
 #ifndef STATE_MAP_BUCKETS
 #define STATE_MAP_BUCKETS 1024
 #endif

src/cpp/ripple/SHAMap.h

@@ -1,15 +1,8 @@
-#ifndef __SHAMAP__
+#ifndef RIPPLE_SHAMAP_H
-#define __SHAMAP__
+#define RIPPLE_SHAMAP_H
 
-#include <list>
+// VFALCO: TODO Get this include out of here!
-#include <map>
+#include "ripple_UptimeTimerAdapter.h"
-#include <stack>
-
-#include <boost/shared_ptr.hpp>
-#include <boost/enable_shared_from_this.hpp>
-#include <boost/unordered_map.hpp>
-
-#include "HashedObject.h"
 
 DEFINE_INSTANCE(SHAMap);
 DEFINE_INSTANCE(SHAMapItem);

src/cpp/ripple/SHAMapDiff.cpp

@@ -1,7 +1,3 @@
-#include "SHAMap.h"
-
-#include <stack>
-
 // This code is used to compare another node's transaction tree
 // to our own. It returns a map containing all items that are different
 // between two SHA maps. It is optimized not to descend down tree

src/cpp/ripple/SHAMapNodes.cpp

@@ -1,18 +1,4 @@
 
-#include "SHAMap.h"
-
-#include <cstring>
-#include <iostream>
-
-#include <boost/foreach.hpp>
-#include <boost/format.hpp>
-#include <boost/lexical_cast.hpp>
-#include <boost/smart_ptr/make_shared.hpp>
-
-#include <openssl/sha.h>
-
-#include "HashPrefixes.h"
-
 SETUP_LOG (SHAMapNode)
 
 std::string SHAMapNode::getString() const

src/cpp/ripple/ripple_HashedObject.cpp

@@ -0,0 +1,14 @@
+
+/*
+#include "HashedObject.h"
+#include "leveldb/db.h"
+#include "leveldb/write_batch.h"
+#include "../database/SqliteDatabase.h"
+#include "Application.h"
+*/
+
+SETUP_LOG (HashedObject)
+
+DECLARE_INSTANCE(HashedObject);
+
+// vim:ts=4

src/cpp/ripple/ripple_HashedObject.h

@@ -0,0 +1,105 @@
+#ifndef RIPPLE_HASHEDOBJECT_H
+#define RIPPLE_HASHEDOBJECT_H
+
+/** The types of hashed objects.
+*/
+enum HashedObjectType
+{
+	hotUNKNOWN = 0,
+	hotLEDGER = 1,
+	hotTRANSACTION = 2,
+	hotACCOUNT_NODE = 3,
+	hotTRANSACTION_NODE = 4
+};
+
+DEFINE_INSTANCE (HashedObject);
+
+/** A blob of data with associated metadata, referenced by hash.
+
+    The metadata includes the following:
+
+    - Type of the blob
+    - The ledger index in which it appears
+    - The SHA 256 hash
+
+    @note No checking is performed to make sure the hash matches the data.
+    @see SHAMap
+*/
+// VFALCO: TODO, consider making the instance a private member of SHAMap
+//         since its the primary user.
+//
+class HashedObject : private IS_INSTANCE (HashedObject)
+{
+public:
+	typedef boost::shared_ptr <HashedObject> pointer;
+
+    /** Create from a vector of data.
+
+        @note A copy of the data is created.
+    */
+	HashedObject (HashedObjectType type,
+                  uint32 ledgerIndex,
+                  std::vector <unsigned char> const& binaryDataToCopy,
+                  uint256 const& hash)
+        : mType (type)
+        , mHash (hash)
+        , mLedgerIndex (ledgerIndex)
+        , mData (binaryDataToCopy)
+    {
+    }
+
+    /** Create from an area of memory.
+
+        @note A copy of the data is created.
+    */
+	HashedObject (HashedObjectType type,
+                  uint32 ledgerIndex,
+                  void const* bufferToCopy,
+                  int bytesInBuffer,
+                  uint256 const& hash)
+        : mType (type)
+        , mHash (hash)
+        , mLedgerIndex (ledgerIndex)
+        , mData (static_cast <unsigned char const*> (bufferToCopy),
+                 static_cast <unsigned char const*> (bufferToCopy) + bytesInBuffer)
+    {
+    }
+
+    /** Retrieve the type of this object.
+    */
+    HashedObjectType getType () const
+    {
+        return mType;
+    }
+
+    /** Retrieve the hash metadata.
+    */
+    uint256 const& getHash() const
+    {
+        return mHash;
+    }
+
+    /** Retrieve the ledger index in which this object appears.
+    */
+    // VFALCO: TODO rename to getLedgerIndex or getLedgerId
+    uint32 getIndex () const
+    {
+        return mLedgerIndex;
+    }
+
+    /** Retrieve the binary data.
+    */
+	std::vector <unsigned char> const& getData() const
+    {
+        return mData;
+    }
+	
+private:
+	HashedObjectType const mType;
+	uint256 const mHash;
+	uint32 const mLedgerIndex;
+	std::vector <unsigned char> const mData;
+};
+
+#endif
+// vim:ts=4

src/cpp/ripple/ripple_HashedObjectStore.cpp

@@ -1,18 +1,11 @@
-#include "HashedObject.h"
 
+/*
+#include "HashedObject.h"
 #include "leveldb/db.h"
 #include "leveldb/write_batch.h"
-
-#include <boost/lexical_cast.hpp>
-#include <boost/foreach.hpp>
-
 #include "../database/SqliteDatabase.h"
-
 #include "Application.h"
-
+*/
-SETUP_LOG (HashedObject)
-
-DECLARE_INSTANCE(HashedObject);
 
 HashedObjectStore::HashedObjectStore(int cacheSize, int cacheAge) :
 	mCache("HashedObjectStore", cacheSize, cacheAge), mNegativeCache("HashedObjectNegativeCache", 0, 120),
@@ -53,8 +46,9 @@ int HashedObjectStore::getWriteLoad()
 	return std::max(mWriteLoad, static_cast<int>(mWriteSet.size()));
 }
 
-static HashedObject::pointer LLRetrieve(const uint256& hash, leveldb::DB* db)
+// low-level retrieve
-{ // low-level retrieve
+HashedObject::pointer HashedObjectStore::LLRetrieve(const uint256& hash, leveldb::DB* db)
+{
 	std::string sData;
 
 	leveldb::Status st = db->Get(leveldb::ReadOptions(),
@@ -73,19 +67,20 @@ static HashedObject::pointer LLRetrieve(const uint256& hash, leveldb::DB* db)
 		bufPtr + 9, sData.size() - 9, hash);
 }
 
-static void LLWrite(boost::shared_ptr<HashedObject> ptr, leveldb::DB* db)
+// low-level write single
-{ // low-level write single
+void HashedObjectStore::LLWrite(boost::shared_ptr<HashedObject> ptr, leveldb::DB* db)
+{
 	HashedObject& obj = *ptr;
-	std::vector<unsigned char> rawData(9 + obj.mData.size());
+	std::vector<unsigned char> rawData(9 + obj.getData ().size());
 	unsigned char* bufPtr = &rawData.front();
 
-	*reinterpret_cast<uint32*>(bufPtr + 0) = ntohl(obj.mLedgerIndex);
+	*reinterpret_cast<uint32*>(bufPtr + 0) = ntohl(obj.getIndex ());
-	*reinterpret_cast<uint32*>(bufPtr + 4) = ntohl(obj.mLedgerIndex);
+	*reinterpret_cast<uint32*>(bufPtr + 4) = ntohl(obj.getIndex ());
-	*(bufPtr + 8) = static_cast<unsigned char>(obj.mType);
+	*(bufPtr + 8) = static_cast<unsigned char>(obj.getType ());
-	memcpy(bufPtr + 9, &obj.mData.front(), obj.mData.size());
+	memcpy(bufPtr + 9, &obj.getData ().front(), obj.getData ().size());
 
 	leveldb::Status st = db->Put(leveldb::WriteOptions(),
-		leveldb::Slice(reinterpret_cast<const char *>(obj.mHash.begin()), obj.mHash.size()),
+		leveldb::Slice(reinterpret_cast<const char *>(obj.getHash ().begin()), obj.getHash ().size()),
 		leveldb::Slice(reinterpret_cast<const char *>(bufPtr), rawData.size()));
 	if (!st.ok())
 	{
@@ -94,22 +89,23 @@ static void LLWrite(boost::shared_ptr<HashedObject> ptr, leveldb::DB* db)
 	}
 }
 
-static void LLWrite(const std::vector< boost::shared_ptr<HashedObject> >& set, leveldb::DB* db)
+// low-level write set
-{ // low-level write set
+void HashedObjectStore::LLWrite(const std::vector< boost::shared_ptr<HashedObject> >& set, leveldb::DB* db)
+{
 	leveldb::WriteBatch batch;
 
 	BOOST_FOREACH(const boost::shared_ptr<HashedObject>& it, set)
 	{
 		const HashedObject& obj = *it;
-		std::vector<unsigned char> rawData(9 + obj.mData.size());
+		std::vector<unsigned char> rawData(9 + obj.getData ().size());
 		unsigned char* bufPtr = &rawData.front();
 
-		*reinterpret_cast<uint32*>(bufPtr + 0) = ntohl(obj.mLedgerIndex);
+		*reinterpret_cast<uint32*>(bufPtr + 0) = ntohl(obj.getIndex ());
-		*reinterpret_cast<uint32*>(bufPtr + 4) = ntohl(obj.mLedgerIndex);
+		*reinterpret_cast<uint32*>(bufPtr + 4) = ntohl(obj.getIndex ());
-		*(bufPtr + 8) = static_cast<unsigned char>(obj.mType);
+		*(bufPtr + 8) = static_cast<unsigned char>(obj.getType ());
-		memcpy(bufPtr + 9, &obj.mData.front(), obj.mData.size());
+		memcpy(bufPtr + 9, &obj.getData ().front(), obj.getData ().size());
 
-		batch.Put(leveldb::Slice(reinterpret_cast<const char *>(obj.mHash.begin()), obj.mHash.size()),
+		batch.Put(leveldb::Slice(reinterpret_cast<const char *>(obj.getHash ().begin()), obj.getHash ().size()),
 			leveldb::Slice(reinterpret_cast<const char *>(bufPtr), rawData.size()));
 	}
 

src/cpp/ripple/ripple_HashedObjectStore.h

@@ -0,0 +1,76 @@
+#ifndef RIPPLE_HASHEDOBJECTSTORE_H
+#define RIPPLE_HASHEDOBJECTSTORE_H
+
+/** Persistency layer for hashed objects.
+*/
+class HashedObjectStore
+{
+public:
+	HashedObjectStore (int cacheSize, int cacheAge);
+
+	bool isLevelDB()
+    {
+        return mLevelDB;
+    }
+
+	float getCacheHitRate ()
+    {
+        return mCache.getHitRate();
+    }
+
+	bool store (HashedObjectType type, uint32 index, const std::vector<unsigned char>& data,
+		const uint256& hash)
+	{
+		if (mLevelDB)
+			return storeLevelDB(type, index, data, hash);
+
+        return storeSQLite(type, index, data, hash);
+	}
+
+	HashedObject::pointer retrieve(const uint256& hash)
+	{
+		if (mLevelDB)
+			return retrieveLevelDB(hash);
+		return retrieveSQLite(hash);
+	}
+
+	bool storeSQLite(HashedObjectType type, uint32 index, const std::vector<unsigned char>& data,
+		const uint256& hash);
+	HashedObject::pointer retrieveSQLite(const uint256& hash);
+	void bulkWriteSQLite(Job&);
+
+	bool storeLevelDB(HashedObjectType type, uint32 index, const std::vector<unsigned char>& data,
+		const uint256& hash);
+	HashedObject::pointer retrieveLevelDB(const uint256& hash);
+	void bulkWriteLevelDB(Job&);
+
+
+	void waitWrite();
+	void tune(int size, int age);
+	void sweep() { mCache.sweep(); mNegativeCache.sweep(); }
+	int getWriteLoad();
+
+	int import(const std::string& fileName);
+
+private:
+    static HashedObject::pointer LLRetrieve(const uint256& hash, leveldb::DB* db);
+    static void LLWrite(boost::shared_ptr<HashedObject> ptr, leveldb::DB* db);
+    static void LLWrite(const std::vector< boost::shared_ptr<HashedObject> >& set, leveldb::DB* db);
+
+private:
+	TaggedCache<uint256, HashedObject, UptimeTimerAdapter>	mCache;
+	KeyCache <uint256, UptimeTimerAdapter> mNegativeCache;
+
+	boost::mutex				mWriteMutex;
+	boost::condition_variable	mWriteCondition;
+	int							mWriteGeneration;
+	int							mWriteLoad;
+
+	std::vector< boost::shared_ptr<HashedObject> > mWriteSet;
+	bool mWritePending;
+	bool mLevelDB;
+	bool mEphemeralDB;
+};
+
+#endif
+// vim:ts=4

src/cpp/ripple/ripple_Peer.h

@@ -1,17 +1,6 @@
 #ifndef __PEER__
 #define __PEER__
 
-#include <bitset>
-#include <boost/asio.hpp>
-#include <boost/asio/ssl.hpp>
-#include <boost/enable_shared_from_this.hpp>
-#include <boost/make_shared.hpp>
-#include <boost/shared_ptr.hpp>
-
-#include "Ledger.h"
-#include "Transaction.h"
-#include "LoadManager.h"
-
 typedef std::pair <std::string,int> ipPort;
 
 class Peer : public boost::enable_shared_from_this <Peer>

src/cpp/ripple/ripple_UptimeTimerAdapter.h

@@ -0,0 +1,14 @@
+#ifndef RIPPLE_UPTIMETIMERADAPTER_H
+#define RIPPLE_UPTIMETIMERADAPTER_H
+
+/** Adapter providing uptime measurements for template classes.
+*/
+struct UptimeTimerAdapter
+{
+	inline static int getElapsedSeconds ()
+	{
+		return UptimeTimer::getInstance().getElapsedSeconds ();
+	}
+};
+
+#endif