Merge branch 'master' of github.com:jedmccaleb/NewCoin

src/Application.cpp

@@ -208,6 +208,8 @@ void Application::startNewLedger()
 
 void Application::loadOldLedger()
 {
+	try
+	{
 		Ledger::pointer lastLedger = Ledger::getSQL("SELECT * from Ledgers order by LedgerSeq desc limit 1;");
 
 		if (!lastLedger)
@@ -215,10 +217,16 @@ void Application::loadOldLedger()
 			std::cout << "No Ledger found?" << std::endl;
 			exit(-1);
 		}
-
 		lastLedger->setClosed();
+
 		Ledger::pointer openLedger = boost::make_shared<Ledger>(false, boost::ref(*lastLedger));
 		mMasterLedger.switchLedgers(lastLedger, openLedger);
 		mNetOps.setLastCloseTime(lastLedger->getCloseTimeNC());
+	}
+	catch (SHAMapMissingNode& mn)
+	{
+		Log(lsFATAL) << "Cannot load ledger. " << mn;
+		exit(-1);
+	}
 }
 // vim:ts=4

src/Ledger.cpp

@@ -24,7 +24,8 @@ SETUP_LOG();
 Ledger::Ledger(const NewcoinAddress& masterID, uint64 startAmount) : mTotCoins(startAmount), mLedgerSeq(1),
 	mCloseTime(0), mParentCloseTime(0), mCloseResolution(LEDGER_TIME_ACCURACY), mCloseFlags(0),
 	mClosed(false), mValidHash(false), mAccepted(false), mImmutable(false),
-	mTransactionMap(new SHAMap()), mAccountStateMap(new SHAMap())
+	mTransactionMap(boost::make_shared<SHAMap>(smtTRANSACTION)),
+	mAccountStateMap(boost::make_shared<SHAMap>(smtSTATE))
 {
 	// special case: put coins in root account
 	AccountState::pointer startAccount = boost::make_shared<AccountState>(masterID);
@@ -38,13 +39,21 @@ Ledger::Ledger(const NewcoinAddress& masterID, uint64 startAmount) : mTotCoins(s
 }
 
 Ledger::Ledger(const uint256 &parentHash, const uint256 &transHash, const uint256 &accountHash,
-	uint64 totCoins, uint32 closeTime, uint32 parentCloseTime, int closeFlags, int closeResolution, uint32 ledgerSeq,bool isMutable)
+	uint64 totCoins, uint32 closeTime, uint32 parentCloseTime, int closeFlags, int closeResolution, uint32 ledgerSeq)
 		: mParentHash(parentHash), mTransHash(transHash), mAccountHash(accountHash), mTotCoins(totCoins),
 		mLedgerSeq(ledgerSeq), mCloseTime(closeTime), mParentCloseTime(parentCloseTime),
 		mCloseResolution(closeResolution), mCloseFlags(closeFlags),
-		mClosed(false), mValidHash(false), mAccepted(false), mImmutable(isMutable)
-{
+		mClosed(false), mValidHash(false), mAccepted(false), mImmutable(true),
+		mTransactionMap(boost::make_shared<SHAMap>(smtTRANSACTION, transHash)),
+		mAccountStateMap(boost::make_shared<SHAMap>(smtSTATE, accountHash))
+{ // This will throw if the root nodes are not available locally
 	updateHash();
+	if (mTransHash.isNonZero())
+		mTransactionMap->fetchRoot(mTransHash);
+	if (mAccountHash.isNonZero())
+		mAccountStateMap->fetchRoot(mAccountHash);
+	mTransactionMap->setImmutable();
+	mAccountStateMap->setImmutable();
 }
 
 Ledger::Ledger(Ledger& ledger, bool isMutable) : mTotCoins(ledger.mTotCoins), mLedgerSeq(ledger.mLedgerSeq),
@@ -62,7 +71,8 @@ Ledger::Ledger(bool /* dummy */, Ledger& prevLedger) :
 	mTotCoins(prevLedger.mTotCoins), mLedgerSeq(prevLedger.mLedgerSeq + 1),
 	mParentCloseTime(prevLedger.mCloseTime), mCloseResolution(prevLedger.mCloseResolution),
 	mCloseFlags(0),	mClosed(false), mValidHash(false), mAccepted(false), mImmutable(false),
-	mTransactionMap(new SHAMap()), mAccountStateMap(prevLedger.mAccountStateMap->snapShot(true))
+	mTransactionMap(boost::make_shared<SHAMap>(smtTRANSACTION)),
+	mAccountStateMap(prevLedger.mAccountStateMap->snapShot(true))
 { // Create a new ledger that follows this one
 	prevLedger.updateHash();
 	mParentHash = prevLedger.getHash();
@@ -123,8 +133,8 @@ void Ledger::setRaw(const Serializer &s)
 	updateHash();
 	if(mValidHash)
 	{
-		mTransactionMap = boost::make_shared<SHAMap>(mTransHash);
-		mAccountStateMap = boost::make_shared<SHAMap>(mAccountHash);
+		mTransactionMap = boost::make_shared<SHAMap>(smtTRANSACTION, mTransHash);
+		mAccountStateMap = boost::make_shared<SHAMap>(smtSTATE, mAccountHash);
 	}
 }
 
@@ -328,10 +338,11 @@ void Ledger::saveAcceptedLedger(Ledger::ref ledger)
 		ledger->mAccountHash.GetHex() % ledger->mTransHash.GetHex()));
 
 	// write out dirty nodes
-	while(ledger->mTransactionMap->flushDirty(256, hotTRANSACTION_NODE, ledger->mLedgerSeq))
-	{ ; }
-	while(ledger->mAccountStateMap->flushDirty(256, hotACCOUNT_NODE, ledger->mLedgerSeq))
-	{ ; }
+	int fc;
+	while ((fc = ledger->mTransactionMap->flushDirty(256, hotTRANSACTION_NODE, ledger->mLedgerSeq)) > 0)
+	{ cLog(lsINFO) << "Flushed " << fc << " dirty transaction nodes"; }
+	while ((fc = ledger->mAccountStateMap->flushDirty(256, hotACCOUNT_NODE, ledger->mLedgerSeq)) > 0)
+	{ cLog(lsINFO) << "Flushed " << fc << " dirty state nodes"; }
 	ledger->disarmDirty();
 
 	SHAMap& txSet = *ledger->peekTransactionMap();
@@ -425,7 +436,7 @@ Ledger::pointer Ledger::getSQL(const std::string& sql)
 	}
 
 	Ledger::pointer ret = Ledger::pointer(new Ledger(prevHash, transHash, accountHash, totCoins,
-		closingTime, prevClosingTime, closeFlags, closeResolution, ledgerSeq, true));
+		closingTime, prevClosingTime, closeFlags, closeResolution, ledgerSeq));
 	if (ret->getHash() != ledgerHash)
 	{
 		if (sLog(lsERROR))

src/Ledger.h

@@ -93,7 +93,7 @@ public:
 
 	Ledger(const uint256 &parentHash, const uint256 &transHash, const uint256 &accountHash,
 		uint64 totCoins, uint32 closeTime, uint32 parentCloseTime, int closeFlags, int closeResolution,
-		uint32 ledgerSeq, bool immutable); // used for database ledgers
+		uint32 ledgerSeq); // used for database ledgers
 
 	Ledger(const std::vector<unsigned char>& rawLedger);
 

src/LedgerAcquire.cpp

@@ -96,6 +96,45 @@ LedgerAcquire::LedgerAcquire(const uint256& hash) : PeerSet(hash, LEDGER_ACQUIRE
 #endif
 }
 
+bool LedgerAcquire::tryLocal()
+{ // return value: true = no more work to do
+	HashedObject::pointer node = theApp->getHashedObjectStore().retrieve(mHash);
+	if (!node)
+		return false;
+
+	mLedger = boost::make_shared<Ledger>(strCopy(node->getData()));
+	assert(mLedger->getHash() == mHash);
+	mHaveBase = true;
+
+	if (!mLedger->getTransHash())
+		mHaveTransactions = true;
+	else
+	{
+		try
+		{
+			mLedger->peekTransactionMap()->fetchRoot(mLedger->getTransHash());
+		}
+		catch (SHAMapMissingNode&)
+		{
+		}
+	}
+
+	if (!mLedger->getAccountHash())
+		mHaveState = true;
+	else
+	{
+		try
+		{
+			mLedger->peekAccountStateMap()->fetchRoot(mLedger->getAccountHash());
+		}
+		catch (SHAMapMissingNode&)
+		{
+		}
+	}
+
+	return mHaveTransactions && mHaveState;
+}
+
 void LedgerAcquire::onTimer()
 {
 	if (getTimeouts() > 6)
@@ -124,7 +163,7 @@ void LedgerAcquire::done()
 	setComplete();
 	mLock.lock();
 	triggers = mOnComplete;
-	mOnComplete.empty();
+	mOnComplete.clear();
 	mLock.unlock();
 
 	if (mLedger)

src/LedgerAcquire.h

@@ -94,6 +94,7 @@ public:
 	bool takeAsNode(const std::list<SHAMapNode>& IDs, const std::list<std::vector<unsigned char> >& data);
 	bool takeAsRootNode(const std::vector<unsigned char>& data);
 	void trigger(Peer::ref, bool timer);
+	bool tryLocal();
 };
 
 class LedgerAcquireMaster

src/LedgerConsensus.cpp

@@ -27,7 +27,7 @@ SETUP_LOG();
 
 TransactionAcquire::TransactionAcquire(const uint256& hash) : PeerSet(hash, TX_ACQUIRE_TIMEOUT), mHaveRoot(false)
 {
-	mMap = boost::make_shared<SHAMap>(hash);
+	mMap = boost::make_shared<SHAMap>(smtTRANSACTION, hash);
 }
 
 void TransactionAcquire::done()
@@ -759,7 +759,7 @@ SHAMap::pointer LedgerConsensus::getTransactionTree(const uint256& hash, bool do
 			{
 				if (!hash)
 				{
-					SHAMap::pointer empty = boost::make_shared<SHAMap>();
+					SHAMap::pointer empty = boost::make_shared<SHAMap>(smtTRANSACTION);
 					mapComplete(hash, empty, false);
 					return empty;
 				}

src/NetworkOPs.cpp

@@ -585,7 +585,6 @@ bool NetworkOPs::checkLastClosedLedger(const std::vector<Peer::pointer>& peerLis
 		if (!mAcquiringLedger->isComplete())
 		{ // add more peers
 			int count = 0;
-			std::vector<Peer::pointer> peers=theApp->getConnectionPool().getPeerVector();
 			BOOST_FOREACH(Peer::ref it, peerList)
 			{
 				if (it->getClosedLedgerHash() == closedLedger)

src/PubKeyCache.cpp

@@ -69,6 +69,6 @@ CKey::pointer PubKeyCache::store(const NewcoinAddress& id, const CKey::pointer&
 void PubKeyCache::clear()
 {
 	boost::mutex::scoped_lock sl(mLock);
-	mCache.empty();
+	mCache.clear();
 }
 // vim:ts=4

src/SHAMap.cpp

@@ -41,14 +41,14 @@ std::size_t hash_value(const uint160& u)
 }
 
 
-SHAMap::SHAMap(uint32 seq) : mSeq(seq), mState(smsModifying)
+SHAMap::SHAMap(SHAMapType t, uint32 seq) : mSeq(seq), mState(smsModifying), mType(t)
 {
 	root = boost::make_shared<SHAMapTreeNode>(mSeq, SHAMapNode(0, uint256()));
 	root->makeInner();
 	mTNByID[*root] = root;
 }
 
-SHAMap::SHAMap(const uint256& hash) : mSeq(0), mState(smsSynching)
+SHAMap::SHAMap(SHAMapType t, const uint256& hash) : mSeq(0), mState(smsSynching), mType(t)
 { // FIXME: Need to acquire root node
 	root = boost::make_shared<SHAMapTreeNode>(mSeq, SHAMapNode(0, uint256()));
 	root->makeInner();
@@ -58,7 +58,7 @@ SHAMap::SHAMap(const uint256& hash) : mSeq(0), mState(smsSynching)
 SHAMap::pointer SHAMap::snapShot(bool isMutable)
 { // Return a new SHAMap that is an immutable snapshot of this one
   // Initially nodes are shared, but CoW is forced on both ledgers
-	SHAMap::pointer ret = boost::make_shared<SHAMap>();
+	SHAMap::pointer ret = boost::make_shared<SHAMap>(mType);
 	SHAMap& newMap = *ret;
 	newMap.mSeq = ++mSeq;
 	newMap.mTNByID = mTNByID;
@@ -93,7 +93,8 @@ std::stack<SHAMapTreeNode::pointer> SHAMap::getStack(const uint256& id, bool inc
 		{
 			if (partialOk)
 				return stack;
-			throw mn;
+			mn.setTargetNode(id);
+			throw;
 		}
 	}
 
@@ -153,10 +154,15 @@ SHAMapTreeNode::pointer SHAMap::walkTo(const uint256& id, bool modify)
 			return inNode;
 		uint256 childHash = inNode->getChildHash(branch);
 
-		SHAMapTreeNode::pointer nextNode = getNode(inNode->getChildNodeID(branch), childHash, false);
-		if (!nextNode)
-			throw SHAMapMissingNode(inNode->getChildNodeID(branch), childHash);
-		inNode = nextNode;
+		try
+		{
+			inNode = getNode(inNode->getChildNodeID(branch), childHash, false);
+		}
+		catch (SHAMapMissingNode& mn)
+		{
+			mn.setTargetNode(id);
+			throw;
+		}
 	}
 	if (inNode->getTag() != id)
 		return SHAMapTreeNode::pointer();
@@ -175,7 +181,7 @@ SHAMapTreeNode* SHAMap::walkToPointer(const uint256& id)
 		if (nextHash.isZero()) return NULL;
 		inNode = getNodePointer(inNode->getChildNodeID(branch), nextHash);
 		if (!inNode)
-			throw SHAMapMissingNode(inNode->getChildNodeID(branch), nextHash);
+			throw SHAMapMissingNode(mType, inNode->getChildNodeID(branch), nextHash, id);
 	}
 	return (inNode->getTag() == id) ? inNode : NULL;
 }
@@ -680,11 +686,14 @@ void SHAMapItem::dump()
 SHAMapTreeNode::pointer SHAMap::fetchNodeExternal(const SHAMapNode& id, const uint256& hash)
 {
 	if (!theApp->running())
-		throw SHAMapMissingNode(id, hash);
+		throw SHAMapMissingNode(mType, id, hash);
 
 	HashedObject::pointer obj(theApp->getHashedObjectStore().retrieve(hash));
 	if (!obj)
-		throw SHAMapMissingNode(id, hash);
+	{
+		Log(lsTRACE) << "fetchNodeExternal: missing " << hash;
+		throw SHAMapMissingNode(mType, id, hash);
+	}
 	assert(Serializer::getSHA512Half(obj->getData()) == hash);
 
 	try
@@ -698,10 +707,17 @@ SHAMapTreeNode::pointer SHAMap::fetchNodeExternal(const SHAMapNode& id, const ui
 	catch (...)
 	{
 		cLog(lsWARNING) << "fetchNodeExternal gets an invalid node: " << hash;
-		throw SHAMapMissingNode(id, hash);
+		throw SHAMapMissingNode(mType, id, hash);
 	}
 }
 
+void SHAMap::fetchRoot(const uint256& hash)
+{
+	root = fetchNodeExternal(SHAMapNode(), hash);
+	root->makeInner();
+	mTNByID[*root] = root;
+}
+
 void SHAMap::armDirty()
 { // begin saving dirty nodes
 	++mSeq;
@@ -719,6 +735,8 @@ int SHAMap::flushDirty(int maxNodes, HashedObjectType t, uint32 seq)
 		boost::unordered_map<SHAMapNode, SHAMapTreeNode::pointer>::iterator it = dirtyNodes.begin();
 		while (it != dirtyNodes.end())
 		{
+			tLog(mType == smtTRANSACTION, lsDEBUG) << "TX node write " << it->first;
+			tLog(mType == smtSTATE, lsDEBUG) << "STATE node write " << it->first;
 			s.erase();
 			it->second->addRaw(s, snfPREFIX);
 			theApp->getHashedObjectStore().store(t, seq, s.peekData(), s.getSHA512Half());
@@ -802,7 +820,7 @@ BOOST_AUTO_TEST_CASE( SHAMap_test )
 	h4.SetHex("b92891fe4ef6cee585fdc6fda2e09eb4d386363158ec3321b8123e5a772c6ca8");
 	h5.SetHex("a92891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7");
 
-	SHAMap sMap;
+	SHAMap sMap(smtFREE);
 	SHAMapItem i1(h1, IntToVUC(1)), i2(h2, IntToVUC(2)), i3(h3, IntToVUC(3)), i4(h4, IntToVUC(4)), i5(h5, IntToVUC(5));
 
 	if (!sMap.addItem(i2, true, false)) BOOST_FAIL("no add");

src/SHAMap.h

@@ -128,6 +128,13 @@ enum SHANodeFormat
 	snfWIRE		= 2, // Compressed form used on the wire
 };
 
+enum SHAMapType
+{
+	smtTRANSACTION	=1,		// A tree of transactions
+	smtSTATE		=2,		// A tree of state nodes
+	smtFREE			=3,		// A tree not part of a ledger
+};
+
 class SHAMapTreeNode : public SHAMapNode
 {
 	friend class SHAMap;
@@ -239,20 +246,35 @@ public:
 class SHAMapMissingNode : public std::runtime_error
 {
 protected:
+	SHAMapType mType;
 	SHAMapNode mNodeID;
 	uint256 mNodeHash;
+	uint256 mTargetIndex;
 
 public:
-	SHAMapMissingNode(const SHAMapNode& nodeID, const uint256& nodeHash) :
-		std::runtime_error(nodeID.getString()), mNodeID(nodeID), mNodeHash(nodeHash)
+	SHAMapMissingNode(SHAMapType t, const SHAMapNode& nodeID, const uint256& nodeHash) :
+		std::runtime_error(nodeID.getString()), mType(t), mNodeID(nodeID), mNodeHash(nodeHash)
 	{ ; }
+
+	SHAMapMissingNode(SHAMapType t, const SHAMapNode& nodeID, const uint256& nodeHash, const uint256& targetIndex) :
+		std::runtime_error(nodeID.getString()), mType(t),
+		mNodeID(nodeID), mNodeHash(nodeHash), mTargetIndex(targetIndex)
+	{ ; }
+
 	virtual ~SHAMapMissingNode() throw()
 	{ ; }
 
+	void setTargetNode(const uint256& tn)	{ mTargetIndex = tn; }
+
+	SHAMapType getMapType() const			{ return mType; }
 	const SHAMapNode& getNodeID() const		{ return mNodeID; }
 	const uint256& getNodeHash() const		{ return mNodeHash; }
+	const uint256& getTargetIndex() const	{ return mTargetIndex; }
+	bool hasTargetIndex() const				{ return !mTargetIndex.isZero(); }
 };
 
+extern std::ostream& operator<<(std::ostream&, const SHAMapMissingNode&);
+
 class SHAMap
 {
 public:
@@ -272,6 +294,8 @@ private:
 
 	SHAMapState mState;
 
+	SHAMapType mType;
+
 protected:
 
 	void dirtyUp(std::stack<SHAMapTreeNode::pointer>& stack, const uint256& target, uint256 prevHash);
@@ -296,8 +320,8 @@ protected:
 public:
 
 	// build new map
-	SHAMap(uint32 seq = 0);
-	SHAMap(const uint256& hash);
+	SHAMap(SHAMapType t, uint32 seq = 0);
+	SHAMap(SHAMapType t, const uint256& hash);
 
 	~SHAMap() { mState = smsInvalid; }
 
@@ -308,6 +332,7 @@ public:
 	ScopedLock Lock() const { return ScopedLock(mLock); }
 
 	bool hasNode(const SHAMapNode& id);
+	void fetchRoot(const uint256& hash);
 
 	// normal hash access functions
 	bool hasItem(const uint256& id);

src/SHAMapDiff.cpp

@@ -120,7 +120,7 @@ bool SHAMap::compare(SHAMap::ref otherMap, SHAMapDiff& differences, int maxCount
 		if (!ourNode || !otherNode)
 		{
 			assert(false);
-			throw SHAMapMissingNode(dNode.mNodeID, uint256());
+			throw SHAMapMissingNode(mType, dNode.mNodeID, uint256());
 		}
 
 		if (ourNode->isLeaf() && otherNode->isLeaf())

src/SHAMapNodes.cpp

@@ -18,6 +18,9 @@
 
 std::string SHAMapNode::getString() const
 {
+	if ((mDepth == 0) && (mNodeID.isZero()))
+		return "NodeID(root)";
+
 	return str(boost::format("NodeID(%s,%s)")
 			% boost::lexical_cast<std::string>(mDepth)
 			% mNodeID.GetHex());
@@ -507,4 +510,16 @@ bool SHAMapTreeNode::setChildHash(int m, const uint256 &hash)
 	return updateHash();
 }
 
+std::ostream& operator<<(std::ostream& out, const SHAMapMissingNode& mn)
+{
+	if (mn.getMapType() == smtTRANSACTION)
+		out << "Missing/TXN(" << mn.getNodeID() << ")";
+	else if (mn.getMapType() == smtSTATE)
+		out << "Missing/STA(" << mn.getNodeID() << ")";
+	else
+		out << "Missing/" << mn.getNodeID();
+}
+
+
+
 // vim:ts=4

src/SHAMapSync.cpp

@@ -53,7 +53,7 @@ void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint2
 				{
 					d = getNode(childID, childHash, false);
 				}
-				catch (SHAMapMissingNode& )
+				catch (SHAMapMissingNode&)
 				{ // node is not in the map
 					if (filter != NULL)
 					{
@@ -269,14 +269,15 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vector<unsigned cha
 					if (nextNode->isInner() && !nextNode->isFullBelow())
 						return true;
 				}
-				catch (SHAMapMissingNode)
+				catch (SHAMapMissingNode&)
 				{
 					return true;
 				}
 			}
 		iNode->setFullBelow();
 	} while (!stack.empty());
-	if (root->isFullBelow()) clearSynching();
+	if (root->isFullBelow())
+		clearSynching();
 	return true;
 }
 
@@ -420,7 +421,7 @@ BOOST_AUTO_TEST_CASE( SHAMapSync_test )
 	srand(seed);
 
 	cLog(lsTRACE) << "Constructing maps";
-	SHAMap source, destination;
+	SHAMap source(smtFREE), destination(smtFREE);
 
 	// add random data to the source map
 	cLog(lsTRACE) << "Adding random data";

src/SerializedObject.cpp

@@ -120,8 +120,8 @@ std::auto_ptr<SerializedType> STObject::makeDeserializedObject(SerializedTypeID
 
 void STObject::set(const std::vector<SOElement::ptr>& type)
 {
-	mData.empty();
-	mType.empty();
+	mData.clear();
+	mType.clear();
 
 	BOOST_FOREACH(const SOElement::ptr& elem, type)
 	{
@@ -138,7 +138,7 @@ bool STObject::setType(const std::vector<SOElement::ptr> &type)
 	boost::ptr_vector<SerializedType> newData;
 	bool valid = true;
 
-	mType.empty();
+	mType.clear();
 	BOOST_FOREACH(const SOElement::ptr& elem, type)
 	{
 		bool match = false;
@@ -204,7 +204,7 @@ bool STObject::isFieldAllowed(SField::ref field)
 
 bool STObject::set(SerializerIterator& sit, int depth)
 { // return true = terminated with end-of-object
-	mData.empty();
+	mData.clear();
 	while (!sit.empty())
 	{
 		int type, field;