From 62b6ec172d269ac74277b8c4a7bff171d86cf0fb Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 22 Jun 2012 00:36:24 -0700 Subject: [PATCH 1/6] Cleanup. --- src/ScopedLock.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ScopedLock.h b/src/ScopedLock.h index e0e65ddfac..a82e18dcff 100644 --- a/src/ScopedLock.h +++ b/src/ScopedLock.h @@ -13,11 +13,11 @@ class ScopedLock { protected: - mutable boost::shared_ptr > mHolder; + mutable boost::shared_ptr mHolder; public: ScopedLock(boost::recursive_mutex& mutex) : - mHolder(boost::make_shared >(boost::ref(mutex))) + mHolder(boost::make_shared(boost::ref(mutex))) { ; } void lock() const { From d7fa4d17cd440d51dd9ab18060906332311bdd69 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 22 Jun 2012 00:41:24 -0700 Subject: [PATCH 2/6] Cleanup. --- src/ScopedLock.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ScopedLock.h b/src/ScopedLock.h index a82e18dcff..1fd1ee6082 100644 --- a/src/ScopedLock.h +++ b/src/ScopedLock.h @@ -2,7 +2,6 @@ #define __SCOPEDLOCKHOLDER__ #include -#include #include #include #include From 14a51421c997010b503f308e0848644c1b72e705 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 22 Jun 2012 00:41:28 -0700 Subject: [PATCH 3/6] Support bulk writes. --- src/HashedObject.cpp | 10 ++++++++++ src/HashedObject.h | 30 +++++++++++++++++++++++------- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/HashedObject.cpp b/src/HashedObject.cpp index ceb7a1b2a5..52f520f8ae 100644 --- a/src/HashedObject.cpp +++ b/src/HashedObject.cpp @@ -119,4 +119,14 @@ HashedObject::pointer HashedObject::retrieve(const uint256& hash) return obj; } +HashedObjectBulkWriter::HashedObjectBulkWriter() : sl(theApp->getHashNodeDB()->getDBLock()) +{ + theApp->getHashNodeDB()->getDB()->executeSQL("BEGIN TRANSACTION;"); +} + +HashedObjectBulkWriter::~HashedObjectBulkWriter() +{ + theApp->getHashNodeDB()->getDB()->executeSQL("END TRANSACTION;"); +} + // vim:ts=4 diff --git a/src/HashedObject.h b/src/HashedObject.h index 7234082c40..6a0fa28693 100644 --- a/src/HashedObject.h +++ b/src/HashedObject.h @@ -3,18 +3,17 @@ #include -#include - #include "types.h" #include "uint256.h" +#include "ScopedLock.h" enum HashedObjectType { - UNKNOWN=0, - LEDGER=1, - TRANSACTION=2, - ACCOUNT_NODE=3, - TRANSACTION_NODE=4 + UNKNOWN = 0, + LEDGER = 1, + TRANSACTION = 2, + ACCOUNT_NODE = 3, + TRANSACTION_NODE = 4 }; class HashedObject @@ -37,10 +36,27 @@ public: const std::vector& getData() { return mData; } bool store() const; + static bool store(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash); static HashedObject::pointer retrieve(const uint256& hash); }; +class HashedObjectBulkWriter +{ +protected: + ScopedLock sl; + +public: + + HashedObjectBulkWriter(); + ~HashedObjectBulkWriter(); + + bool store(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash) + { return HashedObject::store(type, index, data, hash); } + HashedObject::pointer retrieve(const uint256& hash) + { return HashedObject::retrieve(hash); } +}; + #endif From d81b1e6c8470995fd8738728f9ab43db55fd6cf8 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 22 Jun 2012 00:41:40 -0700 Subject: [PATCH 4/6] Cleanups. --- src/Ledger.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/Ledger.cpp b/src/Ledger.cpp index 5549c23433..e583647a6a 100644 --- a/src/Ledger.cpp +++ b/src/Ledger.cpp @@ -20,11 +20,8 @@ Ledger::Ledger(const NewcoinAddress& masterID, uint64 startAmount) : mTotCoins(startAmount), mCloseTime(0), mLedgerSeq(0), mLedgerInterval(LEDGER_INTERVAL), mClosed(false), mValidHash(false), - mAccepted(false), mImmutable(false) + mAccepted(false), mImmutable(false), mTransactionMap(new SHAMap()), mAccountStateMap(new SHAMap()) { - mTransactionMap = boost::make_shared(); - mAccountStateMap = boost::make_shared(); - // special case: put coins in root account AccountState::pointer startAccount = boost::make_shared(masterID); startAccount->peekSLE().setIFieldAmount(sfBalance, startAmount); @@ -263,10 +260,11 @@ void Ledger::saveAcceptedLedger(Ledger::pointer ledger) theApp->getLedgerDB()->getDB()->executeSQL(sql); // write out dirty nodes - while(ledger->mTransactionMap->flushDirty(64, TRANSACTION_NODE, ledger->mLedgerSeq)) + while(ledger->mTransactionMap->flushDirty(256, TRANSACTION_NODE, ledger->mLedgerSeq)) { ; } - while(ledger->mAccountStateMap->flushDirty(64, ACCOUNT_NODE, ledger->mLedgerSeq)) + while(ledger->mAccountStateMap->flushDirty(256, ACCOUNT_NODE, ledger->mLedgerSeq)) { ; } + ledger->disarmDirty(); SHAMap& txSet = *ledger->peekTransactionMap(); Database *db = theApp->getTxnDB()->getDB(); From 6182c4bb51fcb33244d81307b133ad5712daf4a2 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 22 Jun 2012 00:42:03 -0700 Subject: [PATCH 5/6] Make the dirty node save logic work during consensus ledger close. --- src/Ledger.h | 6 ++++-- src/LedgerConsensus.cpp | 1 + src/SHAMap.cpp | 44 +++++++++++++++++++++++++++-------------- src/SHAMap.h | 4 +++- 4 files changed, 37 insertions(+), 18 deletions(-) diff --git a/src/Ledger.h b/src/Ledger.h index 598c148b2d..482fb27ba3 100644 --- a/src/Ledger.h +++ b/src/Ledger.h @@ -60,6 +60,8 @@ public: private: + static uint64 sGenesisClose; + uint256 mHash, mParentHash, mTransHash, mAccountHash; uint64 mTotCoins; uint64 mCloseTime; // when this ledger closes @@ -67,8 +69,6 @@ private: uint16 mLedgerInterval; bool mClosed, mValidHash, mAccepted, mImmutable; - static uint64 sGenesisClose; - SHAMap::pointer mTransactionMap, mAccountStateMap; mutable boost::recursive_mutex mLock; @@ -102,6 +102,8 @@ public: bool isClosed() { return mClosed; } bool isAccepted() { return mAccepted; } bool isImmutable() { return mImmutable; } + void armDirty() { mTransactionMap->armDirty(); mAccountStateMap->armDirty(); } + void disarmDirty() { mTransactionMap->disarmDirty(); mAccountStateMap->disarmDirty(); } // This ledger has closed, will never be accepted, and is accepting // new transactions to be re-repocessed when do accept a new last-closed ledger diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index a129ca071d..4fc9ced246 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -752,6 +752,7 @@ void LedgerConsensus::accept(SHAMap::pointer set) Log(lsDEBUG) << "Previous LCL " << mPrevLedgerHash.GetHex(); Ledger::pointer newLCL = boost::make_shared(false, boost::ref(*mPreviousLedger)); + newLCL->armDirty(); #ifdef DEBUG Json::StyledStreamWriter ssw; diff --git a/src/SHAMap.cpp b/src/SHAMap.cpp index 8811dcce67..987a088d31 100644 --- a/src/SHAMap.cpp +++ b/src/SHAMap.cpp @@ -629,6 +629,12 @@ bool SHAMap::fetchNode(const uint256& hash, std::vector& data) return true; } +void SHAMap::armDirty() +{ // begin saving dirty nodes + ++mSeq; + mDirtyNodes = boost::make_shared< boost::unordered_map >(); +} + int SHAMap::flushDirty(int maxNodes, HashedObjectType t, uint32 seq) { int flushed = 0; @@ -636,37 +642,45 @@ int SHAMap::flushDirty(int maxNodes, HashedObjectType t, uint32 seq) if(mDirtyNodes) { - while (!mDirtyNodes->empty()) + HashedObjectBulkWriter bw; + boost::unordered_map& dirtyNodes = *mDirtyNodes; + boost::unordered_map::iterator it = dirtyNodes.begin(); + while (it != dirtyNodes.end()) { - SHAMapTreeNode::pointer& din = mDirtyNodes->begin()->second; s.erase(); - din->addRaw(s); - HashedObject::store(t, seq, s.peekData(), s.getSHA512Half()); - mDirtyNodes->erase(mDirtyNodes->begin()); - if(flushed++>=maxNodes) return flushed; + it->second->addRaw(s); + bw.store(t, seq, s.peekData(), s.getSHA512Half()); + if (flushed++ >= maxNodes) + return flushed; + it = dirtyNodes.erase(it); } } return flushed; } +void SHAMap::disarmDirty() +{ // stop saving dirty nodes + mDirtyNodes = boost::shared_ptr< boost::unordered_map >(); +} + SHAMapTreeNode::pointer SHAMap::getNode(const SHAMapNode& nodeID) { boost::recursive_mutex::scoped_lock sl(mLock); - SHAMapTreeNode::pointer node=checkCacheNode(nodeID); - if(node) return node; + SHAMapTreeNode::pointer node = checkCacheNode(nodeID); + if (node) return node; - node=root; - while(nodeID!=*node) + node = root; + while (nodeID != *node) { - int branch=node->selectBranch(nodeID.getNodeID()); - assert(branch>=0); - if( (branch<0) || (node->isEmptyBranch(branch)) ) + int branch = node->selectBranch(nodeID.getNodeID()); + assert(branch >= 0); + if ((branch < 0) || node->isEmptyBranch(branch)) return SHAMapTreeNode::pointer(); - node=getNode(node->getChildNodeID(branch), node->getChildHash(branch), false); - if(!node) throw SHAMapException(MissingNode); + node = getNode(node->getChildNodeID(branch), node->getChildHash(branch), false); + if (!node) throw SHAMapException(MissingNode); } return node; } diff --git a/src/SHAMap.h b/src/SHAMap.h index 8851abbf1f..8dda9db8d0 100644 --- a/src/SHAMap.h +++ b/src/SHAMap.h @@ -244,7 +244,7 @@ private: mutable boost::recursive_mutex mLock; boost::unordered_map mTNByID; - boost::shared_ptr > mDirtyNodes; + boost::shared_ptr< boost::unordered_map > mDirtyNodes; SHAMapTreeNode::pointer root; @@ -329,7 +329,9 @@ public: // return value: true=successfully completed, false=too different bool compare(SHAMap::pointer otherMap, SHAMapDiff& differences, int maxCount); + void armDirty(); int flushDirty(int maxNodes, HashedObjectType t, uint32 seq); + void disarmDirty(); void setSeq(uint32 seq) { mSeq = seq; } uint32 getSeq() { return mSeq; } From 0a6a2dcfc7f6bb9b6bf0c00ab7148a40f790b20e Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 22 Jun 2012 00:45:43 -0700 Subject: [PATCH 6/6] Cleanup. --- src/HashedObject.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/HashedObject.cpp b/src/HashedObject.cpp index 52f520f8ae..faaf870ae8 100644 --- a/src/HashedObject.cpp +++ b/src/HashedObject.cpp @@ -57,8 +57,13 @@ bool HashedObject::store(HashedObjectType type, uint32 index, const std::vector< sql.append(obj); sql.append(");"); + std::string exists = + boost::str(boost::format("SELECT ObjType FROM CommittedObject WHERE Hash = '%s';") % hash.GetHex()); + ScopedLock sl(theApp->getHashNodeDB()->getDBLock()); Database* db = theApp->getHashNodeDB()->getDB(); + if (SQL_EXISTS(db, exists)) + return false; return db->executeSQL(sql); } @@ -73,7 +78,7 @@ bool HashedObject::store() const HashedObject::pointer HashedObject::retrieve(const uint256& hash) { if (!theApp || !theApp->getHashNodeDB()) return HashedObject::pointer(); - std::string sql = "SELECT * from CommittedObjects WHERE Hash='"; + std::string sql = "SELECT * FROM CommittedObjects WHERE Hash='"; sql.append(hash.GetHex()); sql.append("';");