From afe016415e894529b6e2c63198b87da6b85043f8 Mon Sep 17 00:00:00 2001
From: JoelKatz <DavidJoelSchwartz@GMail.com>
Date: Tue, 16 Oct 2012 06:53:06 -0700
Subject: [PATCH] Ensure all dirty SHA nodes are fully written before writing
 the ledger header. To avoid a performance penalty, write accountstate nodes
 while we're waiting.

---
 src/HashedObject.cpp | 12 ++++++++++--
 src/HashedObject.h   |  8 +++++++-
 src/Ledger.cpp       | 13 +++++++------
 3 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/src/HashedObject.cpp b/src/HashedObject.cpp
index 837a518a6..c1df0b3fa 100644
--- a/src/HashedObject.cpp
+++ b/src/HashedObject.cpp
@@ -36,7 +36,7 @@ bool HashedObjectStore::store(HashedObjectType type, uint32 index,
 	if (!mCache.canonicalize(hash, object))
 	{
 //		cLog(lsTRACE) << "Queuing write for " << hash;
-		boost::recursive_mutex::scoped_lock sl(mWriteMutex);
+		boost::mutex::scoped_lock sl(mWriteMutex);
 		mWriteSet.push_back(object);
 		if (!mWritePending)
 		{
@@ -50,6 +50,13 @@ bool HashedObjectStore::store(HashedObjectType type, uint32 index,
 	return true;
 }
 
+void HashedObjectStore::waitWrite()
+{
+	boost::unique_lock<boost::mutex> sl(mWriteMutex);
+	while (mWritePending)
+		mWriteCondition.wait(sl);
+}
+
 void HashedObjectStore::bulkWrite()
 {
 	std::vector< boost::shared_ptr<HashedObject> > set;
@@ -59,11 +66,12 @@ void HashedObjectStore::bulkWrite()
 		set.reserve(128);
 
 		{
-			boost::recursive_mutex::scoped_lock sl(mWriteMutex);
+			boost::unique_lock<boost::mutex> sl(mWriteMutex);
 			mWriteSet.swap(set);
 			if (set.empty())
 			{
 				mWritePending = false;
+				mWriteCondition.notify_all();
 				return;
 			}
 		}
diff --git a/src/HashedObject.h b/src/HashedObject.h
index f403d5042..5178af089 100644
--- a/src/HashedObject.h
+++ b/src/HashedObject.h
@@ -3,6 +3,9 @@
 
 #include <vector>
 
+#include <boost/thread/mutex.hpp>
+#include <boost/thread/condition_variable.hpp>
+
 #include "types.h"
 #include "uint256.h"
 #include "ScopedLock.h"
@@ -41,7 +44,9 @@ class HashedObjectStore
 protected:
 	TaggedCache<uint256, HashedObject> mCache;
 
-	boost::recursive_mutex mWriteMutex;
+	boost::mutex mWriteMutex;
+	boost::condition_variable mWriteCondition;
+
 	std::vector< boost::shared_ptr<HashedObject> > mWriteSet;
 	bool mWritePending;
 
@@ -55,6 +60,7 @@ public:
 	HashedObject::pointer retrieve(const uint256& hash);
 
 	void bulkWrite();
+	void waitWrite();
 };
 
 #endif
diff --git a/src/Ledger.cpp b/src/Ledger.cpp
index 7ee7e2689..ceabf6048 100644
--- a/src/Ledger.cpp
+++ b/src/Ledger.cpp
@@ -375,12 +375,6 @@ void Ledger::saveAcceptedLedger()
 		{ cLog(lsINFO) << "Flushed " << fc << " dirty state nodes"; }
 		disarmDirty();
 
-		theApp->getLedgerDB()->getDB()->executeSQL(boost::str(addLedger %
-			getHash().GetHex() % mLedgerSeq % mParentHash.GetHex() %
-			boost::lexical_cast<std::string>(mTotCoins) % mCloseTime % mParentCloseTime %
-			mCloseResolution % mCloseFlags %
-			mAccountHash.GetHex() % mTransHash.GetHex()));
-
 		SHAMap& txSet = *peekTransactionMap();
 		Database *db = theApp->getTxnDB()->getDB();
 		ScopedLock dbLock = theApp->getTxnDB()->getDBLock();
@@ -437,6 +431,13 @@ void Ledger::saveAcceptedLedger()
 			}
 		}
 		db->executeSQL("COMMIT TRANSACTION;");
+
+		theApp->getHashedObjectStore().waitWrite(); // wait until all nodes are written
+		theApp->getLedgerDB()->getDB()->executeSQL(boost::str(addLedger %
+			getHash().GetHex() % mLedgerSeq % mParentHash.GetHex() %
+			boost::lexical_cast<std::string>(mTotCoins) % mCloseTime % mParentCloseTime %
+			mCloseResolution % mCloseFlags %
+			mAccountHash.GetHex() % mTransHash.GetHex()));
 	}
 
 	theApp->getOPs().pubLedger(shared_from_this());