Merge branch 'master' of github.com:jedmccaleb/NewCoin

src/Application.cpp

@@ -17,6 +17,8 @@
 #include <boost/filesystem.hpp>
 #include <boost/thread.hpp>
 
+SETUP_LOG();
+
 Application* theApp = NULL;
 
 DatabaseCon::DatabaseCon(const std::string& strName, const char *initStrings[], int initCount)
@@ -56,7 +58,7 @@ void Application::stop()
 	mValidations.flush();
 	mAuxService.stop();
 
-	Log(lsINFO) << "Stopped: " << mIOService.stopped();
+	cLog(lsINFO) << "Stopped: " << mIOService.stopped();
 }
 
 static void InitDB(DatabaseCon** dbCon, const char *fileName, const char *dbInit[], int dbCount)
@@ -90,12 +92,12 @@ void Application::run()
 
 	if (theConfig.START_UP == Config::FRESH)
 	{
-		Log(lsINFO) << "Starting new Ledger";
+		cLog(lsINFO) << "Starting new Ledger";
 		startNewLedger();
 	}
 	else if (theConfig.START_UP == Config::LOAD)
 	{
-		Log(lsINFO) << "Loading Old Ledger";
+		cLog(lsINFO) << "Loading Old Ledger";
 		loadOldLedger();
 	}
 	else if (theConfig.START_UP == Config::NETWORK)
@@ -155,7 +157,7 @@ void Application::run()
 
 	if (theConfig.RUN_STANDALONE)
 	{
-		Log(lsWARNING) << "Running in standalone mode";
+		cLog(lsWARNING) << "Running in standalone mode";
 		mNetOps.setStandAlone();
 	}
 	else
@@ -185,8 +187,8 @@ void Application::startNewLedger()
 	NewcoinAddress	rootAddress			= NewcoinAddress::createAccountPublic(rootGeneratorMaster, 0);
 
 	// Print enough information to be able to claim root account.
-	Log(lsINFO) << "Root master seed: " << rootSeedMaster.humanSeed();
+	cLog(lsINFO) << "Root master seed: " << rootSeedMaster.humanSeed();
-	Log(lsINFO) << "Root account: " << rootAddress.humanAccountID();
+	cLog(lsINFO) << "Root account: " << rootAddress.humanAccountID();
 
 	{
 		Ledger::pointer firstLedger = boost::make_shared<Ledger>(rootAddress, SYSTEM_CURRENCY_START);
@@ -218,13 +220,34 @@ void Application::loadOldLedger()
 		}
 		lastLedger->setClosed();
 
+		cLog(lsINFO) << "Loading ledger " << lastLedger->getHash() << " seq:" << lastLedger->getLedgerSeq();
+
+		if (lastLedger->getAccountHash().isZero())
+		{
+			cLog(lsFATAL) << "Ledger is empty.";
+			assert(false);
+			exit(-1);
+		}
+
+		if (!lastLedger->walkLedger())
+		{
+			cLog(lsFATAL) << "Ledger is missing nodes.";
+			exit(-1);
+		}
+
+		if (!lastLedger->assertSane())
+		{
+			cLog(lsFATAL) << "Ledger is not sane.";
+			exit(-1);
+		}
+
 		Ledger::pointer openLedger = boost::make_shared<Ledger>(false, boost::ref(*lastLedger));
 		mMasterLedger.switchLedgers(lastLedger, openLedger);
 		mNetOps.setLastCloseTime(lastLedger->getCloseTimeNC());
 	}
 	catch (SHAMapMissingNode& mn)
 	{
-		Log(lsFATAL) << "Cannot load ledger. " << mn;
+		cLog(lsFATAL) << "Cannot load ledger. " << mn;
 		exit(-1);
 	}
 }

src/HashedObject.cpp

@@ -21,7 +21,11 @@ bool HashedObjectStore::store(HashedObjectType type, uint32 index,
 	const std::vector<unsigned char>& data, const uint256& hash)
 { // return: false=already in cache, true = added to cache
 	assert(hash == Serializer::getSHA512Half(data));
-	if (!theApp->getHashNodeDB()) return true;
+	if (!theApp->getHashNodeDB())
+	{
+		cLog(lsTRACE) << "HOS: no db";
+		return true;
+	}
 	if (mCache.touch(hash))
 	{
 		cLog(lsTRACE) << "HOS: " << hash << " store: incache";
@@ -31,29 +35,39 @@ bool HashedObjectStore::store(HashedObjectType type, uint32 index,
 	HashedObject::pointer object = boost::make_shared<HashedObject>(type, index, data, hash);
 	if (!mCache.canonicalize(hash, object))
 	{
+//		cLog(lsTRACE) << "Queuing write for " << hash;
 		boost::recursive_mutex::scoped_lock sl(mWriteMutex);
 		mWriteSet.push_back(object);
-		if (!mWritePending && (mWriteSet.size() >= 64))
+		if (!mWritePending)
 		{
 			mWritePending = true;
 			boost::thread t(boost::bind(&HashedObjectStore::bulkWrite, this));
 			t.detach();
 		}
 	}
+//	else
+//		cLog(lsTRACE) << "HOS: already had " << hash;
 	return true;
 }
 
 void HashedObjectStore::bulkWrite()
 {
 	std::vector< boost::shared_ptr<HashedObject> > set;
+	while (1)
+	{
+		set.clear();
 		set.reserve(128);
 
 		{
 			boost::recursive_mutex::scoped_lock sl(mWriteMutex);
 			mWriteSet.swap(set);
+			if (set.empty())
+			{
 				mWritePending = false;
+				return;
 			}
-	cLog(lsINFO) << "HOS: BulkWrite " << set.size();
+		}
+//		cLog(lsINFO) << "HOS: writing " << set.size();
 
 		static boost::format fExists("SELECT ObjType FROM CommittedObjects WHERE Hash = '%s';");
 		static boost::format
@@ -85,6 +99,7 @@ void HashedObjectStore::bulkWrite()
 
 		db->executeSQL("END TRANSACTION;");
 	}
+}
 
 HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash)
 {
@@ -111,7 +126,7 @@ HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash)
 
 		if (!db->executeSQL(sql) || !db->startIterRows())
 		{
-			cLog(lsTRACE) << "HOS: " << hash << " fetch: not in db";
+//			cLog(lsTRACE) << "HOS: " << hash << " fetch: not in db";
 			return HashedObject::pointer();
 		}
 

src/Ledger.cpp

@@ -31,11 +31,12 @@ Ledger::Ledger(const NewcoinAddress& masterID, uint64 startAmount) : mTotCoins(s
 	AccountState::pointer startAccount = boost::make_shared<AccountState>(masterID);
 	startAccount->peekSLE().setFieldAmount(sfBalance, startAmount);
 	startAccount->peekSLE().setFieldU32(sfSequence, 1);
+	cLog(lsTRACE) << "root account: " << startAccount->peekSLE().getJson(0);
+
+	mAccountStateMap->armDirty();
 	writeBack(lepCREATE, startAccount->getSLE());
-#if 0
+	mAccountStateMap->flushDirty(256, hotACCOUNT_NODE, mLedgerSeq);
-	std::cerr << "Root account:";
+	mAccountStateMap->disarmDirty();
-	startAccount->dump();
-#endif
 }
 
 Ledger::Ledger(const uint256 &parentHash, const uint256 &transHash, const uint256 &accountHash,
@@ -225,18 +226,24 @@ bool Ledger::addTransaction(const uint256& txID, const Serializer& txn)
 { // low-level - just add to table
 	SHAMapItem::pointer item = boost::make_shared<SHAMapItem>(txID, txn.peekData());
 	if (!mTransactionMap->addGiveItem(item, true, false))
+	{
+		cLog(lsWARNING) << "Attempt to add transaction to ledger that already had it";
 		return false;
+	}
 	return true;
 }
 
 bool Ledger::addTransaction(const uint256& txID, const Serializer& txn, const Serializer& md)
 { // low-level - just add to table
-	Serializer s(txn.getDataLength() + md.getDataLength() + 64);
+	Serializer s(txn.getDataLength() + md.getDataLength() + 16);
 	s.addVL(txn.peekData());
 	s.addVL(md.peekData());
 	SHAMapItem::pointer item = boost::make_shared<SHAMapItem>(txID, s.peekData());
 	if (!mTransactionMap->addGiveItem(item, true, true))
+	{
+		cLog(lsFATAL) << "Attempt to add transaction+MD to ledger that already had it";
 		return false;
+	}
 	return true;
 }
 
@@ -273,6 +280,22 @@ Transaction::pointer Ledger::getTransaction(const uint256& transID) const
 	return txn;
 }
 
+SerializedTransaction::pointer Ledger::getSTransaction(SHAMapItem::ref item, SHAMapTreeNode::TNType type)
+{
+	SerializerIterator sit(item->peekSerializer());
+
+	if (type == SHAMapTreeNode::tnTRANSACTION_NM)
+		return boost::make_shared<SerializedTransaction>(boost::ref(sit));
+	else if (type == SHAMapTreeNode::tnTRANSACTION_MD)
+	{
+		Serializer sTxn(sit.getVL());
+		SerializerIterator tSit(sTxn);
+		return boost::make_shared<SerializedTransaction>(boost::ref(tSit));
+	}
+
+	return SerializedTransaction::pointer();
+}
+
 bool Ledger::getTransaction(const uint256& txID, Transaction::pointer& txn, TransactionMetaSet::pointer& meta)
 {
 	SHAMapTreeNode::TNType type;
@@ -285,7 +308,7 @@ bool Ledger::getTransaction(const uint256& txID, Transaction::pointer& txn, Tran
 		txn = theApp->getMasterTransaction().fetch(txID, false);
 		meta = TransactionMetaSet::pointer();
 		if (!txn)
-			txn = Transaction::sharedTransaction(item->getData(), true);
+			txn = Transaction::sharedTransaction(item->peekData(), true);
 	}
 	else if (type == SHAMapTreeNode::tnTRANSACTION_MD)
 	{ // in tree with metadata
@@ -313,12 +336,13 @@ bool Ledger::unitTest()
 
 uint256 Ledger::getHash()
 {
-	if(!mValidHash) updateHash();
+	if (!mValidHash)
-	return(mHash);
+		updateHash();
+	return mHash;
 }
 
-void Ledger::saveAcceptedLedger(Ledger::ref ledger)
+void Ledger::saveAcceptedLedger()
-{
+{ // can be called in a different thread
 	static boost::format ledgerExists("SELECT LedgerSeq FROM Ledgers where LedgerSeq = %d;");
 	static boost::format deleteLedger("DELETE FROM Ledgers WHERE LedgerSeq = %d;");
 	static boost::format AcctTransExists("SELECT LedgerSeq FROM AccountTransactions WHERE TransId = '%s';");
@@ -328,30 +352,43 @@ void Ledger::saveAcceptedLedger(Ledger::ref ledger)
 		"(LedgerHash,LedgerSeq,PrevHash,TotalCoins,ClosingTime,PrevClosingTime,CloseTimeRes,CloseFlags,"
 		"AccountSetHash,TransSetHash) VALUES ('%s','%u','%s','%s','%u','%u','%d','%u','%s','%s');");
 
+	if (!getAccountHash().isNonZero())
+	{
+		cLog(lsFATAL) << "AH is zero: " << getJson(0);
+		assert(false);
+	}
+
+	assert (getAccountHash() == mAccountStateMap->getHash());
+	assert (getTransHash() == mTransactionMap->getHash());
+
+	{
 		ScopedLock sl(theApp->getLedgerDB()->getDBLock());
-	if (SQL_EXISTS(theApp->getLedgerDB()->getDB(), boost::str(ledgerExists % ledger->mLedgerSeq)))
+		if (SQL_EXISTS(theApp->getLedgerDB()->getDB(), boost::str(ledgerExists % mLedgerSeq)))
-		theApp->getLedgerDB()->getDB()->executeSQL(boost::str(deleteLedger % ledger->mLedgerSeq));
+			theApp->getLedgerDB()->getDB()->executeSQL(boost::str(deleteLedger % mLedgerSeq));
 		theApp->getLedgerDB()->getDB()->executeSQL(boost::str(addLedger %
-		ledger->getHash().GetHex() % ledger->mLedgerSeq % ledger->mParentHash.GetHex() %
+			getHash().GetHex() % mLedgerSeq % mParentHash.GetHex() %
-		boost::lexical_cast<std::string>(ledger->mTotCoins) % ledger->mCloseTime % ledger->mParentCloseTime %
+			boost::lexical_cast<std::string>(mTotCoins) % mCloseTime % mParentCloseTime %
-		ledger->mCloseResolution % ledger->mCloseFlags %
+			mCloseResolution % mCloseFlags %
-		ledger->mAccountHash.GetHex() % ledger->mTransHash.GetHex()));
+			mAccountHash.GetHex() % mTransHash.GetHex()));
 
 		// write out dirty nodes
 		int fc;
-	while ((fc = ledger->mTransactionMap->flushDirty(256, hotTRANSACTION_NODE, ledger->mLedgerSeq)) > 0)
+		while ((fc = mTransactionMap->flushDirty(256, hotTRANSACTION_NODE, mLedgerSeq)) > 0)
 		{ cLog(lsINFO) << "Flushed " << fc << " dirty transaction nodes"; }
-	while ((fc = ledger->mAccountStateMap->flushDirty(256, hotACCOUNT_NODE, ledger->mLedgerSeq)) > 0)
+		while ((fc = mAccountStateMap->flushDirty(256, hotACCOUNT_NODE, mLedgerSeq)) > 0)
 		{ cLog(lsINFO) << "Flushed " << fc << " dirty state nodes"; }
-	ledger->disarmDirty();
+		disarmDirty();
 
-	SHAMap& txSet = *ledger->peekTransactionMap();
+		SHAMap& txSet = *peekTransactionMap();
 		Database *db = theApp->getTxnDB()->getDB();
 		ScopedLock dbLock = theApp->getTxnDB()->getDBLock();
 		db->executeSQL("BEGIN TRANSACTION;");
-	for (SHAMapItem::pointer item = txSet.peekFirstItem(); !!item; item = txSet.peekNextItem(item->getTag()))
+		SHAMapTreeNode::TNType type;
+		for (SHAMapItem::pointer item = txSet.peekFirstItem(type); !!item;
+			item = txSet.peekNextItem(item->getTag(), type))
 		{
-		SerializedTransaction::pointer	txn	= theApp->getMasterTransaction().fetch(item, false, ledger->mLedgerSeq);
+			SerializedTransaction::pointer txn	= getSTransaction(item, type);
+			assert(txn);
 
 			// Make sure transaction is in AccountTransactions.
 			if (!SQL_EXISTS(db, boost::str(AcctTransExists % item->getTag().GetHex())))
@@ -374,7 +411,7 @@ void Ledger::saveAcceptedLedger(Ledger::ref ledger)
 					sql += "','";
 					sql += it->humanAccountID();
 					sql += "',";
-				sql += boost::lexical_cast<std::string>(ledger->getLedgerSeq());
+					sql += boost::lexical_cast<std::string>(getLedgerSeq());
 					sql += ")";
 				}
 				sql += ";";
@@ -386,7 +423,7 @@ void Ledger::saveAcceptedLedger(Ledger::ref ledger)
 			{
 				// In Transactions, update LedgerSeq and Status.
 				db->executeSQL(boost::str(updateTx
-				% ledger->getLedgerSeq()
+					% getLedgerSeq()
 					% TXN_SQL_VALIDATED
 					% txn->getTransactionID().GetHex()));
 			}
@@ -394,12 +431,13 @@ void Ledger::saveAcceptedLedger(Ledger::ref ledger)
 			{
 				// Not in Transactions, insert the whole thing..
 				db->executeSQL(
-				txn->getSQLInsertHeader() + txn->getSQL(ledger->getLedgerSeq(), TXN_SQL_VALIDATED) + ";");
+					txn->getSQLInsertHeader() + txn->getSQL(getLedgerSeq(), TXN_SQL_VALIDATED) + ";");
 			}
 		}
 		db->executeSQL("COMMIT TRANSACTION;");
+	}
 
-	theApp->getOPs().pubLedger(ledger);
+	theApp->getOPs().pubLedger(shared_from_this());
 }
 
 Ledger::pointer Ledger::getSQL(const std::string& sql)
@@ -449,6 +487,7 @@ Ledger::pointer Ledger::getSQL(const std::string& sql)
 		assert(false);
 		return Ledger::pointer();
 	}
+	Log(lsDEBUG) << "Loaded ledger: " << ledgerHash;
 	return ret;
 }
 
@@ -469,6 +508,11 @@ Ledger::pointer Ledger::loadByHash(const uint256& ledgerHash)
 }
 
 void Ledger::addJson(Json::Value& ret, int options)
+{
+	ret["ledger"] = getJson(options);
+}
+
+Json::Value Ledger::getJson(int options)
 {
 	Json::Value ledger(Json::objectValue);
 
@@ -555,7 +599,7 @@ void Ledger::addJson(Json::Value& ret, int options)
 		ledger["accountState"] = state;
 	}
 	ledger["seqNum"] = boost::lexical_cast<std::string>(mLedgerSeq);
-	ret["ledger"] = ledger;
+	return ledger;
 }
 
 void Ledger::setAcquiring(void)
@@ -613,7 +657,7 @@ LedgerStateParms Ledger::writeBack(LedgerStateParms parms, SLE::ref entry)
 	if (create)
 	{
 		assert(!mAccountStateMap->hasItem(entry->getIndex()));
-		if(!mAccountStateMap->addGiveItem(item, false, false)) // FIXME: TX metadata
+		if(!mAccountStateMap->addGiveItem(item, false, false))
 		{
 			assert(false);
 			return lepERROR;
@@ -621,7 +665,7 @@ LedgerStateParms Ledger::writeBack(LedgerStateParms parms, SLE::ref entry)
 		return lepCREATED;
 	}
 
-	if (!mAccountStateMap->updateGiveItem(item, false, false)) // FIXME: TX metadata
+	if (!mAccountStateMap->updateGiveItem(item, false, false))
 	{
 		assert(false);
 		return lepERROR;
@@ -930,6 +974,32 @@ uint256 Ledger::getRippleStateIndex(const NewcoinAddress& naA, const NewcoinAddr
 	return s.getSHA512Half();
 }
 
+bool Ledger::walkLedger()
+{
+	std::vector<SHAMapMissingNode> missingNodes;
+	mAccountStateMap->walkMap(missingNodes, 32);
+	if (sLog(lsINFO) && !missingNodes.empty())
+	{
+		Log(lsINFO) << missingNodes.size() << " missing account node(s)";
+		Log(lsINFO) << "First: " << missingNodes[0];
+	}
+	mTransactionMap->walkMap(missingNodes, 32);
+	return missingNodes.empty();
+}
 
+bool Ledger::assertSane()
+{
+	if (mHash.isNonZero() && mAccountHash.isNonZero() && mAccountStateMap && mTransactionMap &&
+			(mAccountHash == mAccountStateMap->getHash()) && (mTransHash == mTransactionMap->getHash()))
+		return true;
+
+	Log(lsFATAL) << "ledger is not sane";
+	Json::Value j = getJson(0);
+	j["accountTreeHash"] = mAccountHash.GetHex();
+	j["transTreeHash"] = mTransHash.GetHex();
+
+	assert(false);
+	return false;
+}
 
 // vim:ts=4

src/Ledger.h

@@ -149,6 +149,7 @@ public:
 	bool hasTransaction(const uint256& TransID) const { return mTransactionMap->hasItem(TransID); }
 	Transaction::pointer getTransaction(const uint256& transID) const;
 	bool getTransaction(const uint256& transID, Transaction::pointer& txn, TransactionMetaSet::pointer& txMeta);
+	static SerializedTransaction::pointer getSTransaction(SHAMapItem::ref, SHAMapTreeNode::TNType);
 
 	// high-level functions
 	AccountState::pointer getAccountState(const NewcoinAddress& acctID);
@@ -157,7 +158,7 @@ public:
 	SLE::pointer getAccountRoot(const NewcoinAddress& naAccountID);
 
 	// database functions
-	static void saveAcceptedLedger(Ledger::ref);
+	void saveAcceptedLedger();
 	static Ledger::pointer loadByIndex(uint32 ledgerIndex);
 	static Ledger::pointer loadByHash(const uint256& ledgerHash);
 
@@ -277,8 +278,12 @@ public:
 	SLE::pointer getRippleState(const uint160& uiA, const uint160& uiB, const uint160& uCurrency)
 		{ return getRippleState(getRippleStateIndex(NewcoinAddress::createAccountID(uiA), NewcoinAddress::createAccountID(uiB), uCurrency)); }
 
+	Json::Value getJson(int options);
 	void addJson(Json::Value&, int options);
 
+	bool walkLedger();
+	bool assertSane();
+
 	static bool unitTest();
 };
 

src/LedgerConsensus.cpp

@@ -58,7 +58,6 @@ void TransactionAcquire::trigger(Peer::ref peer, bool timer)
 	}
 	if (!mHaveRoot)
 	{
-		cLog(lsINFO) << "have no root";
 		ripple::TMGetLedger tmGL;
 		tmGL.set_ledgerhash(mHash.begin(), mHash.size());
 		tmGL.set_itype(ripple::liTS_CANDIDATE);
@@ -301,17 +300,33 @@ void LedgerConsensus::checkLCL()
 
 void LedgerConsensus::handleLCL(const uint256& lclHash)
 {
+	if (mPrevLedgerHash != lclHash)
+	{ // first time switching to this ledger
 		mPrevLedgerHash = lclHash;
-	if (mPreviousLedger->getHash() == mPrevLedgerHash)
-		return;
 
+		if (mHaveCorrectLCL && mProposing && mOurPosition)
+		{
+			cLog(lsINFO) << "Bowing out of consensus";
+			mOurPosition->bowOut();
+			propose();
+		}
+		mProposing = false;
+		mValidating = false;
+		mPeerPositions.clear();
+		mPeerData.clear();
+		mDisputes.clear();
+		mCloseTimes.clear();
+		mDeadNodes.clear();
+		playbackProposals();
+	}
+
+	if (mPreviousLedger->getHash() != mPrevLedgerHash)
+	{ // we need to switch the ledger we're working from
 		Ledger::pointer newLCL = theApp->getMasterLedger().getLedgerByHash(lclHash);
 		if (newLCL)
 			mPreviousLedger = newLCL;
-	else if (mAcquiringLedger && (mAcquiringLedger->getHash() == mPrevLedgerHash))
+		else if (!mAcquiringLedger || (mAcquiringLedger->getHash() != mPrevLedgerHash))
-		return;
+		{ // need to start acquiring the correct consensus LCL
-	else
-	{
 			cLog(lsWARNING) << "Need consensus ledger " << mPrevLedgerHash;
 
 			mAcquiringLedger = theApp->getMasterLedgerAcquire().findCreate(mPrevLedgerHash);
@@ -333,22 +348,10 @@ void LedgerConsensus::handleLCL(const uint256& lclHash)
 					mAcquiringLedger->peerHas(peer);
 			}
 
-		if (mHaveCorrectLCL && mProposing && mOurPosition)
-		{
-			cLog(lsINFO) << "Bowing out of consensus";
-			mOurPosition->bowOut();
-			propose();
-		}
 			mHaveCorrectLCL = false;
-		mProposing = false;
-		mValidating = false;
-		mCloseTimes.clear();
-		mPeerPositions.clear();
-		mDisputes.clear();
-		mDeadNodes.clear();
-		playbackProposals();
 			return;
 		}
+	}
 
 	cLog(lsINFO) << "Acquired the consensus ledger " << mPrevLedgerHash;
 	mHaveCorrectLCL = true;
@@ -356,7 +359,6 @@ void LedgerConsensus::handleLCL(const uint256& lclHash)
 	mCloseResolution = ContinuousLedgerTiming::getNextLedgerTimeResolution(
 		mPreviousLedger->getCloseResolution(), mPreviousLedger->getCloseAgree(),
 		mPreviousLedger->getLedgerSeq() + 1);
-	playbackProposals();
 }
 
 void LedgerConsensus::takeInitialPosition(Ledger& initialLedger)
@@ -675,7 +677,7 @@ void LedgerConsensus::updateOurPositions()
 		for (std::map<uint32, int>::iterator it = closeTimes.begin(), end = closeTimes.end(); it != end; ++it)
 		{
 			cLog(lsINFO) << "CCTime: " << it->first << " has " << it->second << ", " << thresh << " required";
-			if (it->second > thresh)
+			if (it->second >= thresh)
 			{
 				cLog(lsINFO) << "Close time consensus reached: " << it->first;
 				mHaveCloseTimeConsensus = true;
@@ -1114,6 +1116,15 @@ void LedgerConsensus::accept(SHAMap::ref set)
 	newLCL->setAccepted(closeTime, mCloseResolution, closeTimeCorrect);
 	newLCL->updateHash();
 	uint256 newLCLHash = newLCL->getHash();
+
+	if (sLog(lsTRACE))
+	{
+		Log(lsTRACE) << "newLCL";
+		Json::Value p;
+		newLCL->addJson(p, LEDGER_JSON_DUMP_TXNS | LEDGER_JSON_DUMP_STATE);
+		Log(lsTRACE) << p;
+	}
+
 	statusChange(ripple::neACCEPTED_LEDGER, *newLCL);
 	if (mValidating)
 	{
@@ -1181,13 +1192,6 @@ void LedgerConsensus::accept(SHAMap::ref set)
 		theApp->getOPs().closeTimeOffset(offset);
 	}
 
-	if (sLog(lsTRACE))
-	{
-		Log(lsTRACE) << "newLCL";
-		Json::Value p;
-		newLCL->addJson(p, LEDGER_JSON_DUMP_TXNS | LEDGER_JSON_DUMP_STATE);
-		Log(lsTRACE) << p;
-	}
 }
 
 void LedgerConsensus::endConsensus()

src/LedgerEntrySet.cpp

@@ -310,7 +310,8 @@ bool LedgerEntrySet::threadTx(const NewcoinAddress& threadTo, Ledger::ref ledger
 	return threadTx(sle, ledger, newMods);
 }
 
-bool LedgerEntrySet::threadTx(SLE::ref threadTo, Ledger::ref ledger, boost::unordered_map<uint256, SLE::pointer>& newMods)
+bool LedgerEntrySet::threadTx(SLE::ref threadTo, Ledger::ref ledger,
+	boost::unordered_map<uint256, SLE::pointer>& newMods)
 {  // node = the node that was modified/deleted/created
    // threadTo = the node that needs to know
 	uint256 prevTxID;
@@ -323,7 +324,8 @@ bool LedgerEntrySet::threadTx(SLE::ref threadTo, Ledger::ref ledger, boost::unor
 	return false;
 }
 
-bool LedgerEntrySet::threadOwners(SLE::ref node, Ledger::ref ledger, boost::unordered_map<uint256, SLE::pointer>& newMods)
+bool LedgerEntrySet::threadOwners(SLE::ref node, Ledger::ref ledger,
+	boost::unordered_map<uint256, SLE::pointer>& newMods)
 { // thread new or modified node to owner or owners
 	if (node->hasOneOwner()) // thread to owner's account
 	{

src/LedgerHistory.cpp

@@ -36,6 +36,7 @@ void LedgerHistory::addAcceptedLedger(Ledger::pointer ledger)
 	assert(ledger->isAccepted());
 	assert(ledger->isImmutable());
 	mLedgersByIndex.insert(std::make_pair(ledger->getLedgerSeq(), ledger));
+
 	boost::thread thread(boost::bind(&Ledger::saveAcceptedLedger, ledger));
 	thread.detach();
 }

src/Peer.cpp

@@ -1035,6 +1035,9 @@ void Peer::recvGetLedger(ripple::TMGetLedger& packet)
 		if ((!ledger) || (packet.has_ledgerseq() && (packet.ledgerseq() != ledger->getLedgerSeq())))
 		{
 			punishPeer(PP_UNKNOWN_REQUEST);
+			if (ledger)
+				cLog(lsWARNING) << "Ledger has wrong sequence";
+			else
 				cLog(lsWARNING) << "Can't find the ledger they want";
 			return;
 		}
@@ -1056,7 +1059,7 @@ void Peer::recvGetLedger(ripple::TMGetLedger& packet)
 			{ // new-style root request
 				cLog(lsINFO) << "Ledger root w/map roots request";
 				SHAMap::pointer map = ledger->peekAccountStateMap();
-				if (map)
+				if (map && map->getHash().isNonZero())
 				{ // return account state root node if possible
 					Serializer rootNode(768);
 					if (map->getRootNode(rootNode, snfWIRE))
@@ -1065,7 +1068,7 @@ void Peer::recvGetLedger(ripple::TMGetLedger& packet)
 						if (ledger->getTransHash().isNonZero())
 						{
 							map = ledger->peekTransactionMap();
-							if (map)
+							if (map && map->getHash().isNonZero())
 							{
 								rootNode.resize(0);
 								if (map->getRootNode(rootNode, snfWIRE))

src/SHAMap.cpp

@@ -230,11 +230,13 @@ void SHAMap::returnNode(SHAMapTreeNode::pointer& node, bool modify)
 	assert(node->getSeq() <= mSeq);
 	if (node && modify && (node->getSeq() != mSeq))
 	{ // have a CoW
-		if (mDirtyNodes) (*mDirtyNodes)[*node] = node;
 		node = boost::make_shared<SHAMapTreeNode>(*node, mSeq);
+		if (mDirtyNodes)
+			(*mDirtyNodes)[*node] = node;
 		assert(node->isValid());
 		mTNByID[*node] = node;
-		if (node->isRoot()) root = node;
+		if (node->isRoot())
+			root = node;
 	}
 }
 
@@ -549,7 +551,7 @@ bool SHAMap::delItem(const uint256& id)
 	return true;
 }
 
-bool SHAMap::addGiveItem(const SHAMapItem::pointer& item, bool isTransaction, bool hasMeta)
+bool SHAMap::addGiveItem(SHAMapItem::ref item, bool isTransaction, bool hasMeta)
 { // add the specified item, does not update
 #ifdef ST_DEBUG
 	std::cerr << "aGI " << item->getTag() << std::endl;
@@ -647,7 +649,7 @@ bool SHAMap::addItem(const SHAMapItem& i, bool isTransaction, bool hasMetaData)
 	return addGiveItem(boost::make_shared<SHAMapItem>(i), isTransaction, hasMetaData);
 }
 
-bool SHAMap::updateGiveItem(const SHAMapItem::pointer& item, bool isTransaction, bool hasMeta)
+bool SHAMap::updateGiveItem(SHAMapItem::ref item, bool isTransaction, bool hasMeta)
 { // can't change the tag but can change the hash
 	uint256 tag = item->getTag();
 
@@ -686,12 +688,15 @@ void SHAMapItem::dump()
 SHAMapTreeNode::pointer SHAMap::fetchNodeExternal(const SHAMapNode& id, const uint256& hash)
 {
 	if (!theApp->running())
+	{
+		cLog(lsTRACE) << "Trying to fetch external node with application not running";
 		throw SHAMapMissingNode(mType, id, hash);
+	}
 
 	HashedObject::pointer obj(theApp->getHashedObjectStore().retrieve(hash));
 	if (!obj)
 	{
-		Log(lsTRACE) << "fetchNodeExternal: missing " << hash;
+//		Log(lsTRACE) << "fetchNodeExternal: missing " << hash;
 		throw SHAMapMissingNode(mType, id, hash);
 	}
 	assert(Serializer::getSHA512Half(obj->getData()) == hash);
@@ -699,9 +704,7 @@ SHAMapTreeNode::pointer SHAMap::fetchNodeExternal(const SHAMapNode& id, const ui
 	try
 	{
 		SHAMapTreeNode::pointer ret = boost::make_shared<SHAMapTreeNode>(id, obj->getData(), mSeq, snfPREFIX);
-#ifdef DEBUG
 		assert((ret->getNodeHash() == hash) && (id == *ret));
-#endif
 		return ret;
 	}
 	catch (...)
@@ -713,9 +716,18 @@ SHAMapTreeNode::pointer SHAMap::fetchNodeExternal(const SHAMapNode& id, const ui
 
 void SHAMap::fetchRoot(const uint256& hash)
 {
+	if (sLog(lsTRACE))
+	{
+		if (mType == smtTRANSACTION)
+			Log(lsTRACE) << "Fetch root TXN node " << hash;
+		else if (mType == smtSTATE)
+			Log(lsTRACE) << "Fetch root STATE node " << hash;
+		else
+			Log(lsTRACE) << "Fetch root SHAMap node " << hash;
+	}
 	root = fetchNodeExternal(SHAMapNode(), hash);
-	root->makeInner();
 	mTNByID[*root] = root;
+	assert(root->getNodeHash() == hash);
 }
 
 void SHAMap::armDirty()
@@ -735,8 +747,8 @@ int SHAMap::flushDirty(int maxNodes, HashedObjectType t, uint32 seq)
 		boost::unordered_map<SHAMapNode, SHAMapTreeNode::pointer>::iterator it = dirtyNodes.begin();
 		while (it != dirtyNodes.end())
 		{
-			tLog(mType == smtTRANSACTION, lsDEBUG) << "TX node write " << it->first;
+//			tLog(mType == smtTRANSACTION, lsDEBUG) << "TX node write " << it->first;
-			tLog(mType == smtSTATE, lsDEBUG) << "STATE node write " << it->first;
+//			tLog(mType == smtSTATE, lsDEBUG) << "STATE node write " << it->first;
 			s.erase();
 			it->second->addRaw(s, snfPREFIX);
 			theApp->getHashedObjectStore().store(t, seq, s.peekData(), s.getSHA512Half());

src/SHAMap.h

@@ -195,6 +195,7 @@ public:
 	bool isInnerNode() const { return !mItem; }
 	bool setChildHash(int m, const uint256& hash);
 	bool isEmptyBranch(int m) const { return !mHashes[m]; }
+	bool isEmpty() const;
 	int getBranchCount() const;
 	void makeInner();
 	const uint256& getChildHash(int m) const
@@ -400,6 +401,8 @@ public:
 	static std::vector<unsigned char> checkTrustedPath(const uint256& ledgerHash, const uint256& leafIndex,
 		const std::list<std::vector<unsigned char> >& path);
 
+	void walkMap(std::vector<SHAMapMissingNode>& missingNodes, int maxMissing);
+
 	bool deepCompare(SHAMap& other);
 	virtual void dump(bool withHashes = false);
 };

src/SHAMapDiff.cpp

@@ -186,3 +186,38 @@ bool SHAMap::compare(SHAMap::ref otherMap, SHAMapDiff& differences, int maxCount
 
 	return true;
 }
+
+void SHAMap::walkMap(std::vector<SHAMapMissingNode>& missingNodes, int maxMissing)
+{
+	std::stack<SHAMapTreeNode::pointer> nodeStack;
+
+	boost::recursive_mutex::scoped_lock sl(mLock);
+
+	if (!root->isInner())	// root is only node, and we have it
+		return;
+
+	nodeStack.push(root);
+
+	while (!nodeStack.empty())
+	{
+		SHAMapTreeNode::pointer node = nodeStack.top();
+		nodeStack.pop();
+
+		for (int i = 0; i < 16; ++i)
+			if (!node->isEmptyBranch(i))
+			{
+				try
+				{
+					SHAMapTreeNode::pointer d = getNode(node->getChildNodeID(i), node->getChildHash(i), false);
+					if (d->isInner())
+						nodeStack.push(d);
+				}
+				catch (SHAMapMissingNode& n)
+				{
+					missingNodes.push_back(n);
+					if (--maxMissing <= 0)
+						return;
+				}
+			}
+	}
+}

src/SHAMapNodes.cpp

@@ -278,6 +278,8 @@ SHAMapTreeNode::SHAMapTreeNode(const SHAMapNode& id, const std::vector<unsigned
 		}
 		else if (prefix == sHP_LeafNode)
 		{
+			if (s.getLength() < 32)
+				throw std::runtime_error("short PLN node");
 			uint256 u;
 			s.get256(u, s.getLength() - 32);
 			s.chop(32);
@@ -291,7 +293,7 @@ SHAMapTreeNode::SHAMapTreeNode(const SHAMapNode& id, const std::vector<unsigned
 		}
 		else if (prefix == sHP_InnerNode)
 		{
-			if (rawNode.size() != (512 + 4))
+			if (s.getLength() != 512)
 				throw std::runtime_error("invalid PIN node");
 			for (int i = 0; i < 16; ++i)
 				s.get256(mHashes[i] , i * 32);
@@ -347,9 +349,11 @@ bool SHAMapTreeNode::updateHash()
 	{
 		nh = Serializer::getPrefixHash(sHP_TransactionNode, mItem->peekData());
 	}
-	else assert(false);
+	else
+		assert(false);
 
-	if (nh == mHash) return false;
+	if (nh == mHash)
+		return false;
 	mHash = nh;
 	return true;
 }
@@ -357,10 +361,12 @@ bool SHAMapTreeNode::updateHash()
 void SHAMapTreeNode::addRaw(Serializer& s, SHANodeFormat format)
 {
 	assert((format == snfPREFIX) || (format == snfWIRE));
-	if (mType == tnERROR) throw std::runtime_error("invalid I node type");
+	if (mType == tnERROR)
+		throw std::runtime_error("invalid I node type");
 
 	if (mType == tnINNER)
 	{
+		assert(!isEmpty());
 		if (format == snfPREFIX)
 		{
 			s.add32(sHP_InnerNode);
@@ -449,6 +455,14 @@ SHAMapItem::pointer SHAMapTreeNode::getItem() const
 	return boost::make_shared<SHAMapItem>(*mItem);
 }
 
+bool SHAMapTreeNode::isEmpty() const
+{
+	assert(isInner());
+	for (int i = 0; i < 16; ++i)
+		if (mHashes[i].isNonZero()) return false;
+	return true;
+}
+
 int SHAMapTreeNode::getBranchCount() const
 {
 	assert(isInner());
@@ -513,9 +527,9 @@ bool SHAMapTreeNode::setChildHash(int m, const uint256 &hash)
 std::ostream& operator<<(std::ostream& out, const SHAMapMissingNode& mn)
 {
 	if (mn.getMapType() == smtTRANSACTION)
-		out << "Missing/TXN(" << mn.getNodeID() << ")";
+		out << "Missing/TXN(" << mn.getNodeID() << "/" << mn.getNodeHash() << ")";
 	else if (mn.getMapType() == smtSTATE)
-		out << "Missing/STA(" << mn.getNodeID() << ")";
+		out << "Missing/STA(" << mn.getNodeID() << "/" << mn.getNodeHash() << ")";
 	else
 		out << "Missing/" << mn.getNodeID();
 	return out;

src/SerializedLedger.cpp

@@ -98,10 +98,12 @@ bool SerializedLedgerEntry::thread(const uint256& txID, uint32 ledgerSeq, uint25
 	uint256 oldPrevTxID = getFieldH256(sfLastTxnID);
 	Log(lsTRACE) << "Thread Tx:" << txID << " prev:" << oldPrevTxID;
 	if (oldPrevTxID == txID)
+	{ // this transaction is already threaded
+		assert(getFieldU32(sfLastTxnSeq) == ledgerSeq);
 		return false;
+	}
 	prevTxID = oldPrevTxID;
 	prevLedgerID = getFieldU32(sfLastTxnSeq);
-	assert(prevTxID != txID);
 	setFieldH256(sfLastTxnID, txID);
 	setFieldU32(sfLastTxnSeq, ledgerSeq);
 	return true;

src/SerializedObject.cpp

@@ -13,6 +13,8 @@
 #include "TransactionFormats.h"
 #include "SerializedTransaction.h"
 
+SETUP_LOG();
+
 std::auto_ptr<SerializedType> STObject::makeDefaultObject(SerializedTypeID id, SField::ref name)
 {
 	assert((id == STI_NOTPRESENT) || (id == name.fieldType));
@@ -154,7 +156,7 @@ bool STObject::setType(const std::vector<SOElement::ptr> &type)
 		{
 			if (elem->flags != SOE_OPTIONAL)
 			{
-				Log(lsWARNING) << "setType !valid missing " << elem->e_field.fieldName;
+				cLog(lsWARNING) << "setType !valid missing " << elem->e_field.fieldName;
 				valid = false;
 			}
 			newData.push_back(makeNonPresentObject(elem->e_field));
@@ -168,7 +170,7 @@ bool STObject::setType(const std::vector<SOElement::ptr> &type)
 		{
 			if (!t.getFName().isDiscardable())
 			{
-				Log(lsWARNING) << "setType !valid leftover: " << t.getFName().getName();
+				cLog(lsWARNING) << "setType !valid leftover: " << t.getFName().getName();
 				valid = false;
 			}
 		}
@@ -214,7 +216,7 @@ bool STObject::set(SerializerIterator& sit, int depth)
 		SField::ref fn = SField::getField(type, field);
 		if (fn.isInvalid())
 		{
-			Log(lsWARNING) << "Unknown field: field_type=" << type << ", field_name=" << field;
+			cLog(lsWARNING) << "Unknown field: field_type=" << type << ", field_name=" << field;
 			throw std::runtime_error("Unknown field");
 		}
 		giveObject(makeDeserializedObject(fn.fieldType, fn, sit, depth + 1));
@@ -853,7 +855,7 @@ STArray* STArray::construct(SerializerIterator& sit, SField::ref field)
 		SField::ref fn = SField::getField(type, field);
 		if (fn.isInvalid())
 		{
-			Log(lsTRACE) << "Unknown field: " << type << "/" << field;
+			cLog(lsTRACE) << "Unknown field: " << type << "/" << field;
 			throw std::runtime_error("Unknown field");
 		}
 
@@ -1097,7 +1099,7 @@ std::auto_ptr<STObject> STObject::parseJson(const Json::Value& object, SField::r
 					NewcoinAddress a;
 					if (!a.setAccountID(strValue))
 					{
-						Log(lsINFO) << "Invalid acccount JSON: " << fieldName << ": " << strValue;
+						cLog(lsINFO) << "Invalid acccount JSON: " << fieldName << ": " << strValue;
 						throw std::runtime_error("Account invalid");
 					}
 					data.push_back(new STAccount(field, a.getAccountID()));
@@ -1163,8 +1165,8 @@ BOOST_AUTO_TEST_CASE( FieldManipulation_test )
 
 	if (object1.getSerializer() == object2.getSerializer())
 	{
-		Log(lsINFO) << "O1: " << object1.getJson(0);
+		cLog(lsINFO) << "O1: " << object1.getJson(0);
-		Log(lsINFO) << "O2: " << object2.getJson(0);
+		cLog(lsINFO) << "O2: " << object2.getJson(0);
 		BOOST_FAIL("STObject error 4");
 	}
 	object1.makeFieldAbsent(sfTestH256);

src/Serializer.cpp

@@ -9,6 +9,9 @@
 #include <boost/test/unit_test.hpp>
 
 #include "key.h"
+#include "Log.h"
+
+SETUP_LOG();
 
 int Serializer::addZeros(size_t uBytes)
 {
@@ -185,7 +188,10 @@ int Serializer::addFieldID(int type, int name)
 bool Serializer::getFieldID(int& type, int& name, int offset) const
 {
 	if (!get8(type, offset))
+	{
+		cLog(lsWARNING) << "gFID: unable to get type";
 		return false;
+	}
 	name = type & 15;
 	type >>= 4;
 	if (type == 0)
@@ -193,15 +199,21 @@ bool Serializer::getFieldID(int& type, int& name, int offset) const
 		if (!get8(type, ++offset))
 			return false;
 		if ((type == 0) || (type < 16))
+		{
+			cLog(lsWARNING) << "gFID: uncommon type out of range " << type;
 			return false;
 		}
+	}
 	if (name == 0)
 	{ // uncommon name
 		if (!get8(name, ++offset))
 			return false;
 		if ((name == 0) || (name < 16))
+		{
+			cLog(lsWARNING) << "gFID: uncommon name out of range " << name;
 			return false;
 		}
+	}
 	return true;
 }
 

src/TaggedCache.h

@@ -169,7 +169,7 @@ bool TaggedCache<c_Key, c_Data>::canonicalize(const key_type& key, boost::shared
 	{ // in map, but expired. Update in map, insert in cache
 		mit->second = data;
 		mCache.insert(std::make_pair(key, std::make_pair(time(NULL), data)));
-		return false;
+		return true;
 	}
 
 	// in map and cache, canonicalize

src/TransactionMeta.cpp

@@ -195,7 +195,11 @@ bool TransactionMetaNode::thread(const uint256& prevTx, uint32 prevLgr)
 {
 	BOOST_FOREACH(TransactionMetaNodeEntry& it, mEntries)
 		if (it.getType() == TMSThread)
+		{
+			TMNEThread* a = dynamic_cast<TMNEThread *>(&it);
+			assert(a && (a->getPrevTxID() == prevTx) && (a->getPrevLgr() == prevLgr));
 			return false;
+		}
 	addNode(new TMNEThread(prevTx, prevLgr));
 	return true;
 }
@@ -267,8 +271,7 @@ TransactionMetaSet::TransactionMetaSet(uint32 ledger, const std::vector<unsigned
 void TransactionMetaSet::addRaw(Serializer& s)
 {
 	s.add256(mTransactionID);
-	for (std::map<uint256, TransactionMetaNode>::iterator it = mNodes.begin(), end = mNodes.end();
+	for (std::map<uint256, TransactionMetaNode>::iterator it = mNodes.begin(), end = mNodes.end(); it != end; ++it)
-			it != end; ++it)
 		it->second.addRaw(s);
 	s.add8(TMNEndOfMetadata);
 }

src/TransactionMeta.h

@@ -81,6 +81,9 @@ public:
 	virtual void addRaw(Serializer&) const;
 	virtual Json::Value getJson(int) const;
 
+	const uint256& getPrevTxID() const		{ return mPrevTxID; }
+	uint32 getPrevLgr() const				{ return mPrevLgrSeq; }
+
 protected:
 	virtual TransactionMetaNodeEntry* duplicate(void) const { return new TMNEThread(*this); }
 	virtual int compare(const TransactionMetaNodeEntry&) const;
@@ -175,7 +178,7 @@ public:
 protected:
 	uint256 mTransactionID;
 	uint32 mLedger;
-	std::map<uint256, TransactionMetaNode> mNodes;
+	std::map<uint256, TransactionMetaNode> mNodes; // must be an ordered set
 
 public:
 	TransactionMetaSet() : mLedger(0) { ; }