Merge branch 'develop' of github.com:jedmccaleb/NewCoin into develop

src/cpp/ripple/Application.cpp

@@ -1,5 +1,10 @@
 
 #include "Application.h"
+
+#ifdef USE_LEVELDB
+#include "leveldb/cache.h"
+#endif
+
 #include "AcceptedLedger.h"
 #include "Config.h"
 #include "PeerDoor.h"
@@ -79,6 +84,11 @@ void Application::stop()
 	mAuxService.stop();
 	mJobQueue.shutdown();
 
+#ifdef HAVE_LEVELDB
+	delete mHashNodeDB:
+	mHashNodeDB = NULL;
+#endif
+
 	cLog(lsINFO) << "Stopped: " << mIOService.stopped();
 	Instance::shutdown();
 }
@@ -157,6 +167,7 @@ void Application::setup()
 #ifdef USE_LEVELDB
 	leveldb::Options options;
 	options.create_if_missing = true;
+	options.block_cache = leveldb::NewLRUCache(theConfig.getSize(siHashNodeDBCache) * 1024 * 1024);
 	leveldb::Status status = leveldb::DB::Open(options, (theConfig.DATA_DIR / "hashnode").string(), &mHashNodeDB);
 	if (!status.ok() || !mHashNodeDB)
 	{

src/cpp/ripple/HashedObject.cpp

@@ -37,7 +37,7 @@ bool HashedObjectStore::store(HashedObjectType type, uint32 index,
 { // return: false = already in cache, true = added to cache
 	if (!theApp->getHashNodeDB())
 	{
-		cLog(lsTRACE) << "HOS: no db";
+		cLog(lsWARNING) << "HOS: no db";
 		return true;
 	}
 	if (mCache.touch(hash))
@@ -50,7 +50,7 @@ bool HashedObjectStore::store(HashedObjectType type, uint32 index,
 	HashedObject::pointer object = boost::make_shared<HashedObject>(type, index, data, hash);
 	if (!mCache.canonicalize(hash, object))
 	{
-		Serializer s(1 + (32 / 8) + (32 / 8) + data.size());
+		Serializer s(9 + data.size());
 		s.add8(static_cast<unsigned char>(type));
 		s.add32(index);
 		s.add32(index);
@@ -63,6 +63,8 @@ bool HashedObjectStore::store(HashedObjectType type, uint32 index,
 			assert(false);
 		}
 	}
+	else
+		cLog(lsDEBUG) << "HOS: store race";
 	return true;
 }
 
@@ -77,12 +79,18 @@ HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash)
 		return obj;
 
 	if (!theApp || !theApp->getHashNodeDB())
+	{
+		cLog(lsWARNING) << "HOS: no db";
 		return obj;
+	}
 
 	std::string sData;
 	leveldb::Status st = theApp->getHashNodeDB()->Get(leveldb::ReadOptions(), hash.GetHex(), &sData);
 	if (!st.ok())
+	{
+		assert(st.IsNotFound());
 		return obj;
+	}
 
 	Serializer s(sData);
 
@@ -132,6 +140,7 @@ bool HashedObjectStore::store(HashedObjectType type, uint32 index,
 	}
 //	else
 //		cLog(lsTRACE) << "HOS: already had " << hash;
+	mNegativeCache.del(hash);
 	return true;
 }
 

src/cpp/ripple/LedgerAcquire.cpp

@@ -960,7 +960,10 @@ void LedgerAcquireMaster::gotLedgerData(Job&, uint256 hash,
 			cLog(lsWARNING) << "Included TXbase invalid";
 		}
 		if (!san.isInvalid())
+		{
+			ledger->progress();
 			ledger->trigger(peer);
+		}
 		else
 			cLog(lsDEBUG) << "Peer sends invalid base data";
 		return;
@@ -996,7 +999,10 @@ void LedgerAcquireMaster::gotLedgerData(Job&, uint256 hash,
 		else
 			ledger->takeAsNode(nodeIDs, nodeData, ret);
 		if (!ret.isInvalid())
-				ledger->trigger(peer);
+		{
+			ledger->progress();
+			ledger->trigger(peer);
+		}
 		else
 			cLog(lsDEBUG) << "Peer sends invalid node data";
 		return;

src/cpp/ripple/LedgerConsensus.cpp

@@ -267,6 +267,7 @@ void LedgerConsensus::checkLCL()
 
 void LedgerConsensus::handleLCL(const uint256& lclHash)
 {
+	assert((lclHash != mPrevLedgerHash) || (mPreviousLedger->getHash() != lclHash));
 	if (mPrevLedgerHash != lclHash)
 	{ // first time switching to this ledger
 		mPrevLedgerHash = lclHash;
@@ -286,30 +287,32 @@ void LedgerConsensus::handleLCL(const uint256& lclHash)
 		playbackProposals();
 	}
 
-	if (mPreviousLedger->getHash() != mPrevLedgerHash)
-	{ // we need to switch the ledger we're working from
-		Ledger::pointer newLCL = theApp->getLedgerMaster().getLedgerByHash(lclHash);
-		if (newLCL)
-		{
-			mPreviousLedger = newLCL;
-			mPrevLedgerHash = newLCL->getHash();
-		}
-		else if (!mAcquiringLedger || (mAcquiringLedger->getHash() != mPrevLedgerHash))
-		{ // need to start acquiring the correct consensus LCL
-			cLog(lsWARNING) << "Need consensus ledger " << mPrevLedgerHash;
+	if (mPreviousLedger->getHash() == mPrevLedgerHash)
+		return;
 
-			mAcquiringLedger = theApp->getMasterLedgerAcquire().findCreate(mPrevLedgerHash, 0);
-			mHaveCorrectLCL = false;
-			return;
-		}
+	// we need to switch the ledger we're working from
+	Ledger::pointer newLCL = theApp->getLedgerMaster().getLedgerByHash(lclHash);
+	if (newLCL)
+	{
+		assert(newLCL->isClosed());
+		assert(newLCL->isImmutable());
+		assert(newLCL->getHash() == lclHash);
+		mPreviousLedger = newLCL;
+		mPrevLedgerHash = lclHash;
+	}
+	else if (!mAcquiringLedger || (mAcquiringLedger->getHash() != mPrevLedgerHash))
+	{ // need to start acquiring the correct consensus LCL
+		cLog(lsWARNING) << "Need consensus ledger " << mPrevLedgerHash;
+		if (mAcquiringLedger)
+			theApp->getMasterLedgerAcquire().dropLedger(mAcquiringLedger->getHash());
+		mAcquiringLedger = theApp->getMasterLedgerAcquire().findCreate(mPrevLedgerHash, 0);
+		mHaveCorrectLCL = false;
+		return;
 	}
 
 	cLog(lsINFO) << "Have the consensus ledger " << mPrevLedgerHash;
 	mHaveCorrectLCL = true;
-#if 0 // FIXME: can trigger early
-	if (mAcquiringLedger && mAcquiringLedger->isComplete())
-		theApp->getOPs().clearNeedNetworkLedger();
-#endif
+
 	mCloseResolution = ContinuousLedgerTiming::getNextLedgerTimeResolution(
 		mPreviousLedger->getCloseResolution(), mPreviousLedger->getCloseAgree(),
 		mPreviousLedger->getLedgerSeq() + 1);

src/cpp/ripple/LedgerMaster.cpp

@@ -284,7 +284,7 @@ bool LedgerMaster::acquireMissingLedger(Ledger::ref origLedger, const uint256& l
 		}
 	}
 
-	if (theApp->getOPs().shouldFetchPack() && (ledgerSeq > 40000))
+	if (theApp->getOPs().shouldFetchPack(ledgerSeq) && (ledgerSeq > 40000))
 	{ // refill our fetch pack
 		Ledger::pointer nextLedger = mLedgerHistory.getLedgerBySeq(ledgerSeq + 1);
 		if (nextLedger)

src/cpp/ripple/NetworkOPs.cpp

@@ -36,7 +36,7 @@ NetworkOPs::NetworkOPs(boost::asio::io_service& io_service, LedgerMaster* pLedge
 	mMode(omDISCONNECTED), mNeedNetworkLedger(false), mProposing(false), mValidating(false),
 	mNetTimer(io_service), mLedgerMaster(pLedgerMaster), mCloseTimeOffset(0), mLastCloseProposers(0),
 	mLastCloseConvergeTime(1000 * LEDGER_IDLE_INTERVAL), mLastValidationTime(0),
-	mFetchPack("FetchPack", 2048, 30), mLastFetchPack(0),
+	mFetchPack("FetchPack", 2048, 20), mLastFetchPack(0), mFetchSeq(static_cast<uint32>(-1)),
 	mLastLoadBase(256), mLastLoadFactor(256)
 {
 }
@@ -2091,13 +2091,19 @@ bool NetworkOPs::getFetchPack(const uint256& hash, std::vector<unsigned char>& d
 	return true;
 }
 
-bool NetworkOPs::shouldFetchPack()
+bool NetworkOPs::shouldFetchPack(uint32 seq)
 {
 	uint32 now = getNetworkTimeNC();
 	if ((mLastFetchPack == now) || ((mLastFetchPack + 1) == now))
 		 return false;
-	mFetchPack.sweep();
-	if (mFetchPack.getCacheSize() > 384)
+	if (seq < mFetchSeq) // fetch pack has only data for ledgers ahead of where we are
+		mFetchPack.clear();
+	else
+		mFetchPack.sweep();
+	int size = mFetchPack.getCacheSize();
+	if (size == 0)
+		mFetchSeq = static_cast<uint32>(-1);
+	else if (mFetchPack.getCacheSize() > 64)
 		return false;
 	mLastFetchPack = now;
 	return true;
@@ -2108,9 +2114,10 @@ int NetworkOPs::getFetchSize()
 	return mFetchPack.getCacheSize();
 }
 
-void NetworkOPs::gotFetchPack(bool progress)
+void NetworkOPs::gotFetchPack(bool progress, uint32 seq)
 {
 	mLastFetchPack = 0;
+	mFetchSeq = seq;		// earliest pack we have data on
 	theApp->getJobQueue().addJob(jtLEDGER_DATA, "gotFetchPack",
 		boost::bind(&LedgerAcquireMaster::gotFetchPack, &theApp->getMasterLedgerAcquire(), _1));
 }

src/cpp/ripple/NetworkOPs.h

@@ -130,6 +130,7 @@ protected:
 
 	TaggedCache< uint256, std::vector<unsigned char> >	mFetchPack;
 	uint32												mLastFetchPack;
+	uint32												mFetchSeq;
 
 	uint32												mLastLoadBase;
 	uint32												mLastLoadFactor;
@@ -262,8 +263,8 @@ public:
 	bool stillNeedTXSet(const uint256& hash);
 	void makeFetchPack(Job&, boost::weak_ptr<Peer> peer, boost::shared_ptr<ripple::TMGetObjectByHash> request,
 		Ledger::pointer wantLedger, Ledger::pointer haveLedger);
-	bool shouldFetchPack();
-	void gotFetchPack(bool progress);
+	bool shouldFetchPack(uint32 seq);
+	void gotFetchPack(bool progress, uint32 seq);
 	void addFetchPack(const uint256& hash, boost::shared_ptr< std::vector<unsigned char> >& data);
 	bool getFetchPack(const uint256& hash, std::vector<unsigned char>& data);
 	int getFetchSize();

src/cpp/ripple/PathDB.h

@@ -1,6 +1,8 @@
 #ifndef PATHDB__H
 #define PATHBD__H
 
+#include <set>
+
 #include "uint256.h"
 #include "TaggedCache.h"
 
@@ -45,7 +47,7 @@ protected:
 	boost::recursive_mutex						mLock;
 	TaggedCache<currencyIssuer_t, PathDBEntry>	mFromCache;
 	TaggedCache<currencyIssuer_t, PathDBEntry>	mToCache;
-	std::set<PathDBEntry::pointer>				mDirtyPaths;
+//	std::set<PathDBEntry::pointer>				mDirtyPaths;
 
 public:
 

src/cpp/ripple/Peer.cpp

@@ -1274,7 +1274,7 @@ void Peer::recvGetObjectByHash(const boost::shared_ptr<ripple::TMGetObjectByHash
 		}
 		tLog(pLDo && (pLSeq != 0), lsDEBUG) << "Received partial fetch pack for " << pLSeq;
 		if (packet.type() == ripple::TMGetObjectByHash::otFETCH_PACK)
-			theApp->getOPs().gotFetchPack(progress);
+			theApp->getOPs().gotFetchPack(progress, pLSeq);
 	}
 }
 

src/cpp/ripple/SHAMap.cpp

@@ -236,6 +236,31 @@ SHAMapTreeNode* SHAMap::getNodePointer(const SHAMapNode& id, const uint256& hash
 	return fetchNodeExternal(id, hash).get();
 }
 
+SHAMapTreeNode* SHAMap::getNodePointer(const SHAMapNode& id, const uint256& hash, SHAMapSyncFilter* filter)
+{
+	try
+	{
+		return getNodePointer(id, hash);
+	}
+	catch (SHAMapMissingNode)
+	{
+		if (filter)
+		{
+			std::vector<unsigned char> nodeData;
+			if (filter->haveNode(id, hash, nodeData))
+			{
+				SHAMapTreeNode::pointer node = boost::make_shared<SHAMapTreeNode>(
+					boost::cref(id), boost::cref(nodeData), mSeq - 1, snfPREFIX, boost::cref(hash), true);
+				mTNByID[id] = node;
+				filter->gotNode(true, id, hash, nodeData, node->getType());
+				return node.get();
+			}
+		}
+		throw;
+	}
+}
+
+
 void SHAMap::returnNode(SHAMapTreeNode::pointer& node, bool modify)
 { // make sure the node is suitable for the intended operation (copy on write)
 	assert(node->isValid());

src/cpp/ripple/SHAMap.h

@@ -369,6 +369,7 @@ protected:
 	SHAMapTreeNode::pointer getNode(const SHAMapNode& id);
 	SHAMapTreeNode::pointer getNode(const SHAMapNode& id, const uint256& hash, bool modify);
 	SHAMapTreeNode* getNodePointer(const SHAMapNode& id, const uint256& hash);
+	SHAMapTreeNode* getNodePointer(const SHAMapNode& id, const uint256& hash, SHAMapSyncFilter* filter);
 	SHAMapTreeNode* firstBelow(SHAMapTreeNode*);
 	SHAMapTreeNode* lastBelow(SHAMapTreeNode*);
 

src/cpp/ripple/SHAMapSync.cpp

@@ -58,38 +58,21 @@ void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint2
 					SHAMapTreeNode* d = NULL;
 					try
 					{
-						d = getNodePointer(childID, childHash);
+						d = getNodePointer(childID, childHash, filter);
+						if (d->isInner() && !d->isFullBelow())
+						{
+							have_all = false;
+							stack.push(d);
+						}
 					}
 					catch (SHAMapMissingNode&)
 					{ // node is not in the map
-						if (filter != NULL)
-						{
-							std::vector<unsigned char> nodeData;
-							if (filter->haveNode(childID, childHash, nodeData))
-							{
-								assert(mSeq >= 1);
-								SHAMapTreeNode::pointer ptr =
-									boost::make_shared<SHAMapTreeNode>(childID, nodeData, mSeq - 1,
-										snfPREFIX, childHash, true);
-								mTNByID[*ptr] = ptr;
-								d = ptr.get();
-								filter->gotNode(true, childID, childHash, nodeData, ptr->getType());
-							}
-						}
-					}
-					if (!d)
-					{ // we need this node
 						nodeIDs.push_back(childID);
 						hashes.push_back(childHash);
 						if (--max <= 0)
 							return;
 						have_all = false;
 					}
-					else if (d->isInner() && !d->isFullBelow()) // we might need children of this node
-					{
-						have_all = false;
-						stack.push(d);
-					}
 				}
 			}
 		}
@@ -142,32 +125,15 @@ std::vector<uint256> SHAMap::getNeededHashes(int max, SHAMapSyncFilter* filter)
 					SHAMapTreeNode* d = NULL;
 					try
 					{
-						d = getNodePointer(childID, childHash);
-						assert(d);
-					}
-					catch (SHAMapMissingNode&)
-					{ // node is not in the map
-						std::vector<unsigned char> nodeData;
-						if (filter && filter->haveNode(childID, childHash, nodeData))
-						{
-							SHAMapTreeNode::pointer ptr =
-								boost::make_shared<SHAMapTreeNode>(childID, nodeData, mSeq -1,
-									snfPREFIX, childHash, true);
-							mTNByID[*ptr] = ptr;
-							d = ptr.get();
-							filter->gotNode(true, childID, childHash, nodeData, ptr->getType());
-						}
-					}
-					if (d)
-					{
+						d = getNodePointer(childID, childHash, filter);
 						if (d->isInner() && !d->isFullBelow())
 						{
 							have_all = false;
 							stack.push(d);
 						}
 					}
-					else
-					{
+					catch (SHAMapMissingNode&)
+					{ // node is not in the map
 						have_all = false;
 						ret.push_back(childHash);
 						if (--max <= 0)
@@ -191,14 +157,6 @@ std::vector<uint256> SHAMap::getNeededHashes(int max, SHAMapSyncFilter* filter)
 	return ret;
 }
 
-std::list< std::pair<uint256, std::vector<unsigned char> > >
-	getSyncInfo(SHAMap::pointer have, SHAMap::pointer want, int max)
-{
-	std::list< std::pair< uint256, std::vector<unsigned char> > > ret;
-	// WRITEME
-	return ret;
-}
-
 bool SHAMap::getNodeFat(const SHAMapNode& wanted, std::vector<SHAMapNode>& nodeIDs,
 	std::list<std::vector<unsigned char> >& rawNodes, bool fatRoot, bool fatLeaves)
 { // Gets a node and some of its children
@@ -340,11 +298,8 @@ SMAddNode SHAMap::addKnownNode(const SHAMapNode& node, const std::vector<unsigne
 		return SMAddNode::okay();
 
 	SHAMapTreeNode* iNode = root.get();
-	while (!iNode->isLeaf() && !iNode->isFullBelow())
+	while (!iNode->isLeaf() && !iNode->isFullBelow() && (iNode->getDepth() < node.getDepth()))
 	{
-		if (iNode->isLeaf() || iNode->isFullBelow() || (iNode->getDepth() >= node.getDepth()))
-			return SMAddNode::okay();
-
 		int branch = iNode->selectBranch(node.getNodeID());
 		assert(branch >= 0);
 
@@ -358,7 +313,7 @@ SMAddNode SHAMap::addKnownNode(const SHAMapNode& node, const std::vector<unsigne
 
 		try
 		{
-			iNode = getNodePointer(iNode->getChildNodeID(branch), iNode->getChildHash(branch));
+			iNode = getNodePointer(iNode->getChildNodeID(branch), iNode->getChildHash(branch), filter);
 		}
 		catch (SHAMapMissingNode)
 		{
@@ -374,6 +329,7 @@ SMAddNode SHAMap::addKnownNode(const SHAMapNode& node, const std::vector<unsigne
 				boost::make_shared<SHAMapTreeNode>(node, rawNode, mSeq - 1, snfWIRE, uZero, false);
 			if (iNode->getChildHash(branch) != newNode->getNodeHash())
 			{
+				cLog(lsWARNING) << "Corrupt node recevied";
 				return SMAddNode::invalid();
 			}
 
@@ -384,41 +340,11 @@ SMAddNode SHAMap::addKnownNode(const SHAMapNode& node, const std::vector<unsigne
 				filter->gotNode(false, node, iNode->getChildHash(branch), s.peekData(), newNode->getType());
 			}
 			mTNByID[node] = newNode;
-
-			if (!newNode->isLeaf()) // only a leaf can fill an inner node
-				return SMAddNode::useful();
-
-			try
-			{
-				for (int i = 0; i < 16; ++i)
-				{ // does the parent still need more nodes
-					if (!iNode->isEmptyBranch(i) && !fullBelowCache.isPresent(iNode->getChildHash(i)))
-					{
-						SHAMapTreeNode* d = getNodePointer(iNode->getChildNodeID(i), iNode->getChildHash(i));
-						if (d->isInner() && !d->isFullBelow()) // unfilled inner node
-							return SMAddNode::useful();
-					}
-				}
-			}
-			catch (SHAMapMissingNode)
-			{ // still missing something
-				return SMAddNode::useful();
-			}
-
-			// received leaf fills its parent
-			iNode->setFullBelow();
-			if (mType == smtSTATE)
-			{
-				fullBelowCache.add(iNode->getNodeHash());
-				dropBelow(iNode);
-			}
-			if (root->isFullBelow())
-				clearSynching();
 			return SMAddNode::useful();
 		}
 	}
 
-	cLog(lsTRACE) << "got inner node, already had it (late)";
+	cLog(lsTRACE) << "got node, already had it (late)";
 	return SMAddNode::okay();
 }
 

src/cpp/ripple/TaggedCache.h

@@ -77,6 +77,7 @@ public:
 	void setTargetSize(int size);
 	void setTargetAge(int age);
 	void sweep();
+	void clear();
 
 	bool touch(const key_type& key);
 	bool del(const key_type& key, bool valid);
@@ -128,6 +129,12 @@ template<typename c_Key, typename c_Data> int TaggedCache<c_Key, c_Data>::getTra
 	return mCache.size();
 }
 
+template<typename c_Key, typename c_Data> void TaggedCache<c_Key, c_Data>::clear()
+{
+	mCache.clear();
+	mCacheCount = 0;
+}
+
 template<typename c_Key, typename c_Data> void TaggedCache<c_Key, c_Data>::sweep()
 {
 	boost::recursive_mutex::scoped_lock sl(mLock);