Ensure all dirty SHA nodes are fully written before writing the ledger header.

To avoid a performance penalty, write accountstate nodes while we're waiting.

src/HashedObject.cpp

@@ -36,7 +36,7 @@ bool HashedObjectStore::store(HashedObjectType type, uint32 index,
 	if (!mCache.canonicalize(hash, object))
 	{
 //		cLog(lsTRACE) << "Queuing write for " << hash;
-		boost::recursive_mutex::scoped_lock sl(mWriteMutex);
+		boost::mutex::scoped_lock sl(mWriteMutex);
 		mWriteSet.push_back(object);
 		if (!mWritePending)
 		{
@@ -50,6 +50,13 @@ bool HashedObjectStore::store(HashedObjectType type, uint32 index,
 	return true;
 }
 
+void HashedObjectStore::waitWrite()
+{
+	boost::unique_lock<boost::mutex> sl(mWriteMutex);
+	while (mWritePending)
+		mWriteCondition.wait(sl);
+}
+
 void HashedObjectStore::bulkWrite()
 {
 	std::vector< boost::shared_ptr<HashedObject> > set;
@@ -59,11 +66,12 @@ void HashedObjectStore::bulkWrite()
 		set.reserve(128);
 
 		{
-			boost::recursive_mutex::scoped_lock sl(mWriteMutex);
+			boost::unique_lock<boost::mutex> sl(mWriteMutex);
 			mWriteSet.swap(set);
 			if (set.empty())
 			{
 				mWritePending = false;
+				mWriteCondition.notify_all();
 				return;
 			}
 		}

src/HashedObject.h

@@ -3,6 +3,9 @@
 
 #include <vector>
 
+#include <boost/thread/mutex.hpp>
+#include <boost/thread/condition_variable.hpp>
+
 #include "types.h"
 #include "uint256.h"
 #include "ScopedLock.h"
@@ -41,7 +44,9 @@ class HashedObjectStore
 protected:
 	TaggedCache<uint256, HashedObject> mCache;
 
-	boost::recursive_mutex mWriteMutex;
+	boost::mutex mWriteMutex;
+	boost::condition_variable mWriteCondition;
+
 	std::vector< boost::shared_ptr<HashedObject> > mWriteSet;
 	bool mWritePending;
 
@@ -55,6 +60,7 @@ public:
 	HashedObject::pointer retrieve(const uint256& hash);
 
 	void bulkWrite();
+	void waitWrite();
 };
 
 #endif

src/Ledger.cpp

@@ -375,12 +375,6 @@ void Ledger::saveAcceptedLedger()
 		{ cLog(lsINFO) << "Flushed " << fc << " dirty state nodes"; }
 		disarmDirty();
 
-		theApp->getLedgerDB()->getDB()->executeSQL(boost::str(addLedger %
-			getHash().GetHex() % mLedgerSeq % mParentHash.GetHex() %
-			boost::lexical_cast<std::string>(mTotCoins) % mCloseTime % mParentCloseTime %
-			mCloseResolution % mCloseFlags %
-			mAccountHash.GetHex() % mTransHash.GetHex()));
-
 		SHAMap& txSet = *peekTransactionMap();
 		Database *db = theApp->getTxnDB()->getDB();
 		ScopedLock dbLock = theApp->getTxnDB()->getDBLock();
@@ -437,6 +431,13 @@ void Ledger::saveAcceptedLedger()
 			}
 		}
 		db->executeSQL("COMMIT TRANSACTION;");
+
+		theApp->getHashedObjectStore().waitWrite(); // wait until all nodes are written
+		theApp->getLedgerDB()->getDB()->executeSQL(boost::str(addLedger %
+			getHash().GetHex() % mLedgerSeq % mParentHash.GetHex() %
+			boost::lexical_cast<std::string>(mTotCoins) % mCloseTime % mParentCloseTime %
+			mCloseResolution % mCloseFlags %
+			mAccountHash.GetHex() % mTransHash.GetHex()));
 	}
 
 	theApp->getOPs().pubLedger(shared_from_this());