From 726b7dc5720cd3360de4110f25cb8c99bdf69967 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 23 Jun 2012 22:55:53 -0700 Subject: [PATCH 01/11] Speed up DB initialization --- src/Wallet.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/Wallet.cpp b/src/Wallet.cpp index 9f12140f0f..12dda320b6 100644 --- a/src/Wallet.cpp +++ b/src/Wallet.cpp @@ -76,7 +76,21 @@ bool Wallet::nodeIdentityCreate() { // Make new key. +#ifdef CREATE_NEW_DH_PARAMS std::string strDh512 = DH_der_gen(512); +#else + static char dh512Param[] = { + 0x30, 0x46, 0x02, 0x41, 0x00, 0x98, 0x15, 0xd2, 0xd0, 0x08, 0x32, 0xda, + 0xaa, 0xac, 0xc4, 0x71, 0xa3, 0x1b, 0x11, 0xf0, 0x6c, 0x62, 0xb2, 0x35, + 0x8a, 0x10, 0x92, 0xc6, 0x0a, 0xa3, 0x84, 0x7e, 0xaf, 0x17, 0x29, 0x0b, + 0x70, 0xef, 0x07, 0x4f, 0xfc, 0x9d, 0x6d, 0x87, 0x99, 0x19, 0x09, 0x5b, + 0x6e, 0xdb, 0x57, 0x72, 0x4a, 0x7e, 0xcd, 0xaf, 0xbd, 0x3a, 0x97, 0x55, + 0x51, 0x77, 0x5a, 0x34, 0x7c, 0xe8, 0xc5, 0x71, 0x63, 0x02, 0x01, 0x02 + }; + std::string strDh512(dh512Param, sizeof(dh512Param)); +#endif + + #if 1 std::string strDh1024 = strDh512; // For testing and most cases 512 is fine. #else From 4a15c2a105035ffcdbf804254a9caa1e8779dbad Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 04:52:01 -0700 Subject: [PATCH 02/11] Bugfix. --- src/TaggedCache.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/TaggedCache.h b/src/TaggedCache.h index a53b7d3dc5..8c84a34654 100644 --- a/src/TaggedCache.h +++ b/src/TaggedCache.h @@ -121,7 +121,7 @@ template bool TaggedCache::touch // Is the object in the map? typename boost::unordered_map::iterator mit = mMap.find(key); if (mit == mMap.end()) return false; - if (mit->second->expired()) + if (mit->second.expired()) { // in map, but expired mMap.erase(mit); return false; @@ -131,12 +131,12 @@ template bool TaggedCache::touch typename boost::unordered_map::iterator cit = mCache.find(key); if (cit != mCache.end()) { // in both map and cache - cit->second->first = time(NULL); + cit->second.first = time(NULL); return true; } // In map but not cache, put in cache - mCache.insert(std::make_pair(key, std::make_pair(time(NULL), weak_data_ptr(cit->second->second)))); + mCache.insert(std::make_pair(key, std::make_pair(time(NULL), weak_data_ptr(cit->second.second)))); return true; } From 748461ddc22df698076b60d48b37104357b58890 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 04:53:31 -0700 Subject: [PATCH 03/11] Make the hashed object write code not block on an SQL write. Make bulk updates efficient. --- src/Application.cpp | 1 + src/HashedObject.cpp | 107 +++++++++++++++++++++++++++---------------- src/HashedObject.h | 31 ++++--------- src/SHAMap.cpp | 3 +- 4 files changed, 80 insertions(+), 62 deletions(-) diff --git a/src/Application.cpp b/src/Application.cpp index f85992d93b..e500b39eeb 100644 --- a/src/Application.cpp +++ b/src/Application.cpp @@ -51,6 +51,7 @@ extern int TxnDBCount, LedgerDBCount, WalletDBCount, HashNodeDBCount, NetNodeDBC void Application::stop() { mIOService.stop(); + mHashedObjectStore.bulkWrite(); Log(lsINFO) << "Stopped: " << mIOService.stopped(); } diff --git a/src/HashedObject.cpp b/src/HashedObject.cpp index 63ff9f633f..03d885b71e 100644 --- a/src/HashedObject.cpp +++ b/src/HashedObject.cpp @@ -26,46 +26,79 @@ void HashedObject::setHash() mHash = Serializer::getSHA512Half(mData); } -// FIXME: Stores should be added to a queue that's serviced by an auxilliary thread or from an -// auxilliary thread pool. These should be tied into a cache, since you need one to handle -// an immedate read back (before the write completes) +HashedObjectStore::HashedObjectStore(int cacheSize, int cacheAge) : + mCache(cacheSize, cacheAge), mWritePending(false) +{ + mWriteSet.reserve(128); +} + bool HashedObjectStore::store(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash) -{ +{ // return: false=already in cache, true = added to cache if (!theApp->getHashNodeDB()) return true; + if (mCache.touch(hash)) return false; + HashedObject::pointer object = boost::make_shared(type, index, data); object->setHash(); if (object->getHash() != hash) throw std::runtime_error("Object added to store doesn't have valid hash"); - std::string sql = "INSERT INTO CommittedObjects (Hash,ObjType,LedgerIndex,Object) VALUES ('"; - sql.append(hash.GetHex()); - switch(type) + boost::recursive_mutex::scoped_lock sl(mWriteMutex); + mWriteSet.push_back(object); + if (mWriteSet.size() == 64) { - case LEDGER: sql.append("','L','"); break; - case TRANSACTION: sql.append("','T','"); break; - case ACCOUNT_NODE: sql.append("','A','"); break; - case TRANSACTION_NODE: sql.append("','N','"); break; - default: sql.append("','U','"); break; + boost::recursive_mutex::scoped_lock sl(mWriteMutex); + if (!mWritePending) + { + mWritePending = true; + boost::thread t(boost::bind(&HashedObjectStore::bulkWrite, this)); + t.detach(); + } } - sql.append(boost::lexical_cast(index)); - sql.append("',"); - std::string obj; - theApp->getHashNodeDB()->getDB()->escape(&(data.front()), data.size(), obj); - sql.append(obj); - sql.append(");"); + return true; +} - std::string exists = - boost::str(boost::format("SELECT ObjType FROM CommittedObjects WHERE Hash = '%s';") % hash.GetHex()); +void HashedObjectStore::bulkWrite() +{ + std::vector< boost::shared_ptr > set; + set.reserve(128); + + { + boost::recursive_mutex::scoped_lock sl(mWriteMutex); + mWriteSet.swap(set); + mWritePending = false; + } + + boost::format fExists("SELECT ObjType FROM CommittedObjects WHERE Hash = '%s';"); + boost::format fAdd("INSERT INTO ComittedObject (Hash,ObjType,LedgerIndex,Object) VALUES ('%s','%c','%u','%s');"); - ScopedLock sl(theApp->getHashNodeDB()->getDBLock()); - if (mCache.canonicalize(hash, object)) - return false; Database* db = theApp->getHashNodeDB()->getDB(); - if (SQL_EXISTS(db, exists)) - return false; - return db->executeSQL(sql); + ScopedLock sl = theApp->getHashNodeDB()->getDBLock(); + + db->executeSQL("BEGIN TRANSACTION;"); + + for (std::vector< boost::shared_ptr >::iterator it = set.begin(), end = set.end(); it != end; ++it) + { + HashedObject& obj = **it; + if (!SQL_EXISTS(db, boost::str(fExists % obj.getHash().GetHex()))) + { + char type; + switch(obj.getType()) + { + case LEDGER: type = 'L'; break; + case TRANSACTION: type = 'T'; break; + case ACCOUNT_NODE: type = 'A'; break; + case TRANSACTION_NODE: type = 'N'; break; + default: type = 'U'; + } + std::string rawData; + db->escape(&(obj.getData().front()), obj.getData().size(), rawData); + db->executeSQL(boost::str(fAdd % obj.getHash().GetHex() % type % obj.getIndex() % rawData )); + } + } + + db->executeSQL("END TRANSACTION;"); } HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash) @@ -74,7 +107,11 @@ HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash) { ScopedLock sl(theApp->getHashNodeDB()->getDBLock()); obj = mCache.fetch(hash); - if (obj) return obj; + if (obj) + { + Log(lsTRACE) << "HOS: " << hash.GetHex() << " fetch: incache"; + return obj; + } } if (!theApp || !theApp->getHashNodeDB()) return HashedObject::pointer(); @@ -90,7 +127,10 @@ HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash) Database* db = theApp->getHashNodeDB()->getDB(); if (!db->executeSQL(sql) || !db->startIterRows()) + { + Log(lsTRACE) << "HOS: " << hash.GetHex() << " fetch: not in db"; return HashedObject::pointer(); + } std::string type; db->getStr("ObjType", type); @@ -122,19 +162,8 @@ HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash) #ifdef DEBUG assert(obj->checkHash()); #endif + Log(lsTRACE) << "HOS: " << hash.GetHex() << " fetch: in db"; return obj; } -ScopedLock HashedObjectStore::beginBulk() -{ - ScopedLock sl(theApp->getHashNodeDB()->getDBLock()); - theApp->getHashNodeDB()->getDB()->executeSQL("BEGIN TRANSACTION;"); - return sl; -} - -void HashedObjectStore::endBulk() -{ - theApp->getHashNodeDB()->getDB()->executeSQL("END TRANSACTION;"); -} - // vim:ts=4 diff --git a/src/HashedObject.h b/src/HashedObject.h index 0f66c1509e..514a026670 100644 --- a/src/HashedObject.h +++ b/src/HashedObject.h @@ -34,8 +34,10 @@ public: bool checkFixHash(); void setHash(); - const std::vector& getData() { return mData; } - const uint256& getHash() { return mHash; } + const std::vector& getData() { return mData; } + const uint256& getHash() { return mHash; } + HashedObjectType getType() { return mType; } + uint32 getIndex() { return mLedgerIndex; } }; class HashedObjectStore @@ -43,33 +45,20 @@ class HashedObjectStore protected: TaggedCache mCache; + boost::recursive_mutex mWriteMutex; + std::vector< boost::shared_ptr > mWriteSet; + bool mWritePending; + public: - HashedObjectStore(int cacheSize, int cacheAge) : mCache(cacheSize, cacheAge) { ; } + HashedObjectStore(int cacheSize, int cacheAge); bool store(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash); HashedObject::pointer retrieve(const uint256& hash); - ScopedLock beginBulk(); - void endBulk(); -}; - -class HashedObjectBulkWriter -{ -protected: - HashedObjectStore& mStore; - ScopedLock sl; - -public: - HashedObjectBulkWriter(HashedObjectStore& ostore) : mStore(ostore), sl(mStore.beginBulk()) { ; } - ~HashedObjectBulkWriter() { mStore.endBulk(); } - - bool store(HashedObjectType type, uint32 index, const std::vector& data, - const uint256& hash) { return mStore.store(type, index, data, hash); } - - HashedObject::pointer retrieve(const uint256& hash) { return mStore.retrieve(hash); } + void bulkWrite(); }; #endif diff --git a/src/SHAMap.cpp b/src/SHAMap.cpp index e3961f5901..ea592d60a5 100644 --- a/src/SHAMap.cpp +++ b/src/SHAMap.cpp @@ -642,14 +642,13 @@ int SHAMap::flushDirty(int maxNodes, HashedObjectType t, uint32 seq) if(mDirtyNodes) { - HashedObjectBulkWriter bw(theApp->getHashedObjectStore()); boost::unordered_map& dirtyNodes = *mDirtyNodes; boost::unordered_map::iterator it = dirtyNodes.begin(); while (it != dirtyNodes.end()) { s.erase(); it->second->addRaw(s); - bw.store(t, seq, s.peekData(), s.getSHA512Half()); + theApp->getHashedObjectStore().store(t, seq, s.peekData(), s.getSHA512Half()); if (flushed++ >= maxNodes) return flushed; it = dirtyNodes.erase(it); From 955752c8bc9ef57d881af77d147caf49b7a40b1f Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 05:33:32 -0700 Subject: [PATCH 04/11] Bugfix. --- src/TaggedCache.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/TaggedCache.h b/src/TaggedCache.h index 8c84a34654..f07a80d138 100644 --- a/src/TaggedCache.h +++ b/src/TaggedCache.h @@ -213,9 +213,8 @@ boost::shared_ptr TaggedCache::fetch(const key_type& key) template bool TaggedCache::store(const key_type& key, const c_Data& data) { - if (!canonicalize(key, boost::shared_ptr(data))) - return false; - return true; + boost::shared_ptr d = boost::make_shared(boost::ref(data)); + return canonicalize(key, d); } template From c13c561295a2efdc54687f91ed9ce76f3f64ac92 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 05:33:53 -0700 Subject: [PATCH 05/11] Use the new accelerators to speed up ledger and transaction set synchronization --- src/LedgerAcquire.cpp | 16 ++++++--- src/LedgerAcquire.h | 26 -------------- src/LedgerConsensus.cpp | 13 +++---- src/LedgerConsensus.h | 1 - src/SHAMapSync.h | 75 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 93 insertions(+), 38 deletions(-) create mode 100644 src/SHAMapSync.h diff --git a/src/LedgerAcquire.cpp b/src/LedgerAcquire.cpp index 825045bf7e..9801719275 100644 --- a/src/LedgerAcquire.cpp +++ b/src/LedgerAcquire.cpp @@ -7,6 +7,7 @@ #include "Application.h" #include "Log.h" +#include "SHAMapSync.h" #define LA_DEBUG #define LEDGER_ACQUIRE_TIMEOUT 2 @@ -82,7 +83,7 @@ void PeerSet::TimerEntry(boost::weak_ptr wptr, const boost::system::err } LedgerAcquire::LedgerAcquire(const uint256& hash) : PeerSet(hash, LEDGER_ACQUIRE_TIMEOUT), - mFilter(&theApp->getNodeCache()), mHaveBase(false), mHaveState(false), mHaveTransactions(false) + mHaveBase(false), mHaveState(false), mHaveTransactions(false) { #ifdef LA_DEBUG Log(lsTRACE) << "Acquiring ledger " << mHash.GetHex(); @@ -168,7 +169,8 @@ void LedgerAcquire::trigger(Peer::pointer peer) { std::vector nodeIDs; std::vector nodeHashes; - mLedger->peekTransactionMap()->getMissingNodes(nodeIDs, nodeHashes, 128, &mFilter); + TransactionStateSF tFilter(mLedger->getHash(), mLedger->getSeq()); + mLedger->peekTransactionMap()->getMissingNodes(nodeIDs, nodeHashes, 128, &tFilter); if (nodeIDs.empty()) { if (!mLedger->peekTransactionMap()->isValid()) mFailed = true; @@ -220,7 +222,8 @@ void LedgerAcquire::trigger(Peer::pointer peer) { std::vector nodeIDs; std::vector nodeHashes; - mLedger->peekAccountStateMap()->getMissingNodes(nodeIDs, nodeHashes, 128, &mFilter); + AccountStateSF aFilter(mLedger->getHash(), mLedger->getSeq()); + mLedger->peekAccountStateMap()->getMissingNodes(nodeIDs, nodeHashes, 128, &aFilter); if (nodeIDs.empty()) { if (!mLedger->peekAccountStateMap()->isValid()) mFailed = true; @@ -297,6 +300,7 @@ bool LedgerAcquire::takeBase(const std::string& data, Peer::pointer peer) return false; } mHaveBase = true; + theApp->getHashedObjectStore().store(LEDGER, mLedger->getLedgerSeq(), data, mHash); progress(); if (!mLedger->getTransHash()) mHaveTransactions = true; if (!mLedger->getAccountHash()) mHaveState = true; @@ -311,6 +315,7 @@ bool LedgerAcquire::takeTxNode(const std::list& nodeIDs, if (!mHaveBase) return false; std::list::const_iterator nodeIDit = nodeIDs.begin(); std::list< std::vector >::const_iterator nodeDatait = data.begin(); + TransactionStateSF tFilter(mLedger->getHash(), mLedger->getSeq()); while (nodeIDit != nodeIDs.end()) { if (nodeIDit->isRoot()) @@ -318,7 +323,7 @@ bool LedgerAcquire::takeTxNode(const std::list& nodeIDs, if (!mLedger->peekTransactionMap()->addRootNode(mLedger->getTransHash(), *nodeDatait)) return false; } - else if (!mLedger->peekTransactionMap()->addKnownNode(*nodeIDit, *nodeDatait, &mFilter)) + else if (!mLedger->peekTransactionMap()->addKnownNode(*nodeIDit, *nodeDatait, &tFilter)) return false; ++nodeIDit; ++nodeDatait; @@ -342,6 +347,7 @@ bool LedgerAcquire::takeAsNode(const std::list& nodeIDs, if (!mHaveBase) return false; std::list::const_iterator nodeIDit = nodeIDs.begin(); std::list< std::vector >::const_iterator nodeDatait = data.begin(); + AccountStateSF tFilter(mLedger->getHash(), mLedger->getSeq()); while (nodeIDit != nodeIDs.end()) { if (nodeIDit->isRoot()) @@ -349,7 +355,7 @@ bool LedgerAcquire::takeAsNode(const std::list& nodeIDs, if (!mLedger->peekAccountStateMap()->addRootNode(mLedger->getAccountHash(), *nodeDatait)) return false; } - else if (!mLedger->peekAccountStateMap()->addKnownNode(*nodeIDit, *nodeDatait, &mFilter)) + else if (!mLedger->peekAccountStateMap()->addKnownNode(*nodeIDit, *nodeDatait, &tFilter)) return false; ++nodeIDit; ++nodeDatait; diff --git a/src/LedgerAcquire.h b/src/LedgerAcquire.h index 87e2c86b21..cc499fdb3a 100644 --- a/src/LedgerAcquire.h +++ b/src/LedgerAcquire.h @@ -58,31 +58,6 @@ private: static void TimerEntry(boost::weak_ptr, const boost::system::error_code& result); }; -typedef TaggedCache< uint256, std::vector > NodeCache; -typedef std::vector VUC; - -class THSyncFilter : public SHAMapSyncFilter -{ -protected: - NodeCache* mCache; // holds nodes we see during the consensus process - -public: - THSyncFilter(NodeCache* cache) : mCache(cache) { ; } - virtual void gotNode(const SHAMapNode& id, const uint256& nodeHash, - const std::vector& nodeData, bool) - { - boost::shared_ptr ptr = boost::make_shared(nodeData); - mCache->canonicalize(nodeHash, ptr); - } - virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) - { - boost::shared_ptr entry = mCache->fetch(nodeHash); - if (!entry) return false; - nodeData = *entry; - return true; - } -}; - class LedgerAcquire : public PeerSet, public boost::enable_shared_from_this { // A ledger we are trying to acquire public: @@ -90,7 +65,6 @@ public: protected: Ledger::pointer mLedger; - THSyncFilter mFilter; bool mHaveBase, mHaveState, mHaveTransactions; std::vector< boost::function > mOnComplete; diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index 5baeff0d6a..ce5a35f2b8 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -11,13 +11,13 @@ #include "LedgerTiming.h" #include "SerializedValidation.h" #include "Log.h" +#include "SHAMapSync.h" #define TRUST_NETWORK // #define LC_DEBUG -TransactionAcquire::TransactionAcquire(const uint256& hash) - : PeerSet(hash, 1), mFilter(&theApp->getNodeCache()), mHaveRoot(false) +TransactionAcquire::TransactionAcquire(const uint256& hash) : PeerSet(hash, 1), mHaveRoot(false) { mMap = boost::make_shared(); mMap->setSynching(); @@ -50,9 +50,9 @@ void TransactionAcquire::trigger(Peer::pointer peer) } if (mHaveRoot) { - std::vector nodeIDs; - std::vector nodeHashes; - mMap->getMissingNodes(nodeIDs, nodeHashes, 256, &mFilter); + std::vector nodeIDs; std::vector nodeHashes; + ConsensusTransSetSF sf; + mMap->getMissingNodes(nodeIDs, nodeHashes, 256, &sf); if (nodeIDs.empty()) { if (mMap->isValid()) @@ -91,6 +91,7 @@ bool TransactionAcquire::takeNodes(const std::list& nodeIDs, { std::list::const_iterator nodeIDit = nodeIDs.begin(); std::list< std::vector >::const_iterator nodeDatait = data.begin(); + ConsensusTransSetSF sf; while (nodeIDit != nodeIDs.end()) { if (nodeIDit->isRoot()) @@ -104,7 +105,7 @@ bool TransactionAcquire::takeNodes(const std::list& nodeIDs, return false; else mHaveRoot = true; } - else if (!mMap->addKnownNode(*nodeIDit, *nodeDatait, &mFilter)) + else if (!mMap->addKnownNode(*nodeIDit, *nodeDatait, &sf)) return false; ++nodeIDit; ++nodeDatait; diff --git a/src/LedgerConsensus.h b/src/LedgerConsensus.h index c9a5010cee..8db236487e 100644 --- a/src/LedgerConsensus.h +++ b/src/LedgerConsensus.h @@ -22,7 +22,6 @@ public: protected: SHAMap::pointer mMap; - THSyncFilter mFilter; // FIXME: Should use transaction master too bool mHaveRoot; void onTimer() { trigger(Peer::pointer()); } diff --git a/src/SHAMapSync.h b/src/SHAMapSync.h new file mode 100644 index 0000000000..d88b98fd51 --- /dev/null +++ b/src/SHAMapSync.h @@ -0,0 +1,75 @@ +#ifndef __SHAMAPYSNC__ +#define __SHAMAPSYNC__ + +#include "SHAMap.h" +#include "Application.h" + +// Sync filters allow low-level SHAMapSync code to interact correctly with +// higher-level structures such as caches and transaction stores + +class ConsensusTransSetSF : public SHAMapSyncFilter +{ // sync filter for transaction sets during consensus building +public: + ConsensusTransSetSF() { ; } + virtual void gotNode(const SHAMapNode& id, const uint256& nodeHash, + const std::vector& nodeData, bool isLeaf) + { + // WRITEME: If 'isLeaf' is true, this is a transaction + theApp->getNodeCache().store(nodeHash, nodeData); + } + virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) + { + // WRITEME: We could check our own map, we could check transaction tables + return theApp->getNodeCache().retrieve(nodeHash, nodeData); + } +}; + +class LedgerAccountStateSF : public SHAMapSyncFilter +{ // sync filter for account state nodes during ledger sync +protected: + uint256 mLedgerHash; + uint32 mLedgerSeq; + +public: + LedgerAccountStateSF(const uint256& ledgerHash, uint32 ledgerSeq) : mLedgerHash(ledgerHash), mLedgerSeq(ledgerSeq) + { ; } + + virtual void gotNode(const SHAMapNode& id, const uint256& nodeHash, + const std::vector& nodeData, bool isLeaf) + { + theApp->getHashedObjectStore().store(ACCOUNT_NODE, mLedgerSeq, nodeData, nodeHash); + } + virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) + { + HashedObject::pointer node = theApp->getHashedObjectStore().retrieve(nodeHash); + if (!node) return false; + nodeData = node->getData(); + return true; + } +}; + +class TransactionStateSF : public SHAMapSyncFilter +{ // sync filter for transactions tree during ledger sync +protected: + uint256 mLedgerHash; + uint32 mLedgerSeq; + +public: + TransactionStateSF(const uint256& ledgerHash, uint32 ledgerSeq) : mLedgerHash(ledgerHash), mLedgerSeq(ledgerSeq) + { ; } + + virtual void gotNode(const SHAMapNode& id, const uint256& nodeHash, + const std::vector& nodeData, bool isLeaf) + { + theApp->getHashedObjectStore().store(isLeaf ? TRANSACTION : TRANSACTION_NODE, mLedgerSeq, nodeData, nodeHash); + } + virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) + { + HashedObject::pointer node = theApp->getHashedObjectStore().retrieve(nodeHash); + if (!node) return false; + nodeData = node->getData(); + return true; + } +}; + +#endif From f9560e27c75230dab1147562bb1cb9c91a0faafc Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 05:42:44 -0700 Subject: [PATCH 06/11] Missing #include --- src/TaggedCache.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/TaggedCache.h b/src/TaggedCache.h index f07a80d138..bb5dc36667 100644 --- a/src/TaggedCache.h +++ b/src/TaggedCache.h @@ -6,6 +6,7 @@ #include #include #include +#include // This class implemented a cache and a map. The cache keeps objects alive // in the map. The map allows multiple code paths that reference objects From 340f17e471ed6b160b3c7ad857298685e5741038 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 05:48:09 -0700 Subject: [PATCH 07/11] Use the new filters. --- src/HashedObject.cpp | 14 +++++++++----- src/LedgerAcquire.cpp | 10 +++++----- src/SHAMapSync.h | 4 ++-- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/HashedObject.cpp b/src/HashedObject.cpp index 03d885b71e..6d59e8b293 100644 --- a/src/HashedObject.cpp +++ b/src/HashedObject.cpp @@ -37,25 +37,28 @@ bool HashedObjectStore::store(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash) { // return: false=already in cache, true = added to cache if (!theApp->getHashNodeDB()) return true; - if (mCache.touch(hash)) return false; + if (mCache.touch(hash)) + { + Log(lsTRACE) << "HOS: " << hash.GetHex() << " store: incache"; + return false; + } HashedObject::pointer object = boost::make_shared(type, index, data); object->setHash(); if (object->getHash() != hash) throw std::runtime_error("Object added to store doesn't have valid hash"); - boost::recursive_mutex::scoped_lock sl(mWriteMutex); - mWriteSet.push_back(object); - if (mWriteSet.size() == 64) { boost::recursive_mutex::scoped_lock sl(mWriteMutex); - if (!mWritePending) + mWriteSet.push_back(object); + if (!mWritePending && (mWriteSet.size() >= 64)) { mWritePending = true; boost::thread t(boost::bind(&HashedObjectStore::bulkWrite, this)); t.detach(); } } + Log(lsTRACE) << "HOS: " << hash.GetHex() << " store: deferred"; return true; } @@ -69,6 +72,7 @@ void HashedObjectStore::bulkWrite() mWriteSet.swap(set); mWritePending = false; } + Log(lsINFO) << "HOS: BulkWrite " << set.size(); boost::format fExists("SELECT ObjType FROM CommittedObjects WHERE Hash = '%s';"); boost::format fAdd("INSERT INTO ComittedObject (Hash,ObjType,LedgerIndex,Object) VALUES ('%s','%c','%u','%s');"); diff --git a/src/LedgerAcquire.cpp b/src/LedgerAcquire.cpp index 9801719275..c87d16ee3a 100644 --- a/src/LedgerAcquire.cpp +++ b/src/LedgerAcquire.cpp @@ -169,7 +169,7 @@ void LedgerAcquire::trigger(Peer::pointer peer) { std::vector nodeIDs; std::vector nodeHashes; - TransactionStateSF tFilter(mLedger->getHash(), mLedger->getSeq()); + TransactionStateSF tFilter(mLedger->getHash(), mLedger->getLedgerSeq()); mLedger->peekTransactionMap()->getMissingNodes(nodeIDs, nodeHashes, 128, &tFilter); if (nodeIDs.empty()) { @@ -222,7 +222,7 @@ void LedgerAcquire::trigger(Peer::pointer peer) { std::vector nodeIDs; std::vector nodeHashes; - AccountStateSF aFilter(mLedger->getHash(), mLedger->getSeq()); + AccountStateSF aFilter(mLedger->getHash(), mLedger->getLedgerSeq()); mLedger->peekAccountStateMap()->getMissingNodes(nodeIDs, nodeHashes, 128, &aFilter); if (nodeIDs.empty()) { @@ -300,7 +300,7 @@ bool LedgerAcquire::takeBase(const std::string& data, Peer::pointer peer) return false; } mHaveBase = true; - theApp->getHashedObjectStore().store(LEDGER, mLedger->getLedgerSeq(), data, mHash); + theApp->getHashedObjectStore().store(LEDGER, mLedger->getLedgerSeq(), strCopy(data), mHash); progress(); if (!mLedger->getTransHash()) mHaveTransactions = true; if (!mLedger->getAccountHash()) mHaveState = true; @@ -315,7 +315,7 @@ bool LedgerAcquire::takeTxNode(const std::list& nodeIDs, if (!mHaveBase) return false; std::list::const_iterator nodeIDit = nodeIDs.begin(); std::list< std::vector >::const_iterator nodeDatait = data.begin(); - TransactionStateSF tFilter(mLedger->getHash(), mLedger->getSeq()); + TransactionStateSF tFilter(mLedger->getHash(), mLedger->getLedgerSeq()); while (nodeIDit != nodeIDs.end()) { if (nodeIDit->isRoot()) @@ -347,7 +347,7 @@ bool LedgerAcquire::takeAsNode(const std::list& nodeIDs, if (!mHaveBase) return false; std::list::const_iterator nodeIDit = nodeIDs.begin(); std::list< std::vector >::const_iterator nodeDatait = data.begin(); - AccountStateSF tFilter(mLedger->getHash(), mLedger->getSeq()); + AccountStateSF tFilter(mLedger->getHash(), mLedger->getLedgerSeq()); while (nodeIDit != nodeIDs.end()) { if (nodeIDit->isRoot()) diff --git a/src/SHAMapSync.h b/src/SHAMapSync.h index d88b98fd51..26b6185520 100644 --- a/src/SHAMapSync.h +++ b/src/SHAMapSync.h @@ -24,14 +24,14 @@ public: } }; -class LedgerAccountStateSF : public SHAMapSyncFilter +class AccountStateSF : public SHAMapSyncFilter { // sync filter for account state nodes during ledger sync protected: uint256 mLedgerHash; uint32 mLedgerSeq; public: - LedgerAccountStateSF(const uint256& ledgerHash, uint32 ledgerSeq) : mLedgerHash(ledgerHash), mLedgerSeq(ledgerSeq) + AccountStateSF(const uint256& ledgerHash, uint32 ledgerSeq) : mLedgerHash(ledgerHash), mLedgerSeq(ledgerSeq) { ; } virtual void gotNode(const SHAMapNode& id, const uint256& nodeHash, From 08bfd8f1d04e66ca8aca48963fb39a4eb841670f Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 06:01:48 -0700 Subject: [PATCH 08/11] Fix the disconnect between hashes of nodes in transport format and hashes of nodes in internal format. --- src/HashedObject.cpp | 30 ++---------------------------- src/HashedObject.h | 8 ++------ 2 files changed, 4 insertions(+), 34 deletions(-) diff --git a/src/HashedObject.cpp b/src/HashedObject.cpp index 6d59e8b293..512a0ab025 100644 --- a/src/HashedObject.cpp +++ b/src/HashedObject.cpp @@ -7,25 +7,6 @@ #include "Application.h" #include "Log.h" -bool HashedObject::checkHash() const -{ - uint256 hash = Serializer::getSHA512Half(mData); - return hash == mHash; -} - -bool HashedObject::checkFixHash() -{ - uint256 hash = Serializer::getSHA512Half(mData); - if (hash == mHash) return true; - mHash = hash; - return false; -} - -void HashedObject::setHash() -{ - mHash = Serializer::getSHA512Half(mData); -} - HashedObjectStore::HashedObjectStore(int cacheSize, int cacheAge) : mCache(cacheSize, cacheAge), mWritePending(false) { @@ -43,10 +24,7 @@ bool HashedObjectStore::store(HashedObjectType type, uint32 index, return false; } - HashedObject::pointer object = boost::make_shared(type, index, data); - object->setHash(); - if (object->getHash() != hash) - throw std::runtime_error("Object added to store doesn't have valid hash"); + HashedObject::pointer object = boost::make_shared(type, index, data, hash); { boost::recursive_mutex::scoped_lock sl(mWriteMutex); @@ -159,13 +137,9 @@ HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash) return HashedObject::pointer(); } - obj = boost::make_shared(htype, index, data); - obj->mHash = hash; + obj = boost::make_shared(htype, index, data, hash); mCache.canonicalize(hash, obj); } -#ifdef DEBUG - assert(obj->checkHash()); -#endif Log(lsTRACE) << "HOS: " << hash.GetHex() << " fetch: in db"; return obj; } diff --git a/src/HashedObject.h b/src/HashedObject.h index 514a026670..0331cfa9b3 100644 --- a/src/HashedObject.h +++ b/src/HashedObject.h @@ -27,12 +27,8 @@ public: uint32 mLedgerIndex; std::vector mData; - HashedObject(HashedObjectType type, uint32 index, const std::vector& data) : - mType(type), mLedgerIndex(index), mData(data) { ; } - - bool checkHash() const; - bool checkFixHash(); - void setHash(); + HashedObject(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash) : + mType(type), mHash(hash), mLedgerIndex(index), mData(data) { ; } const std::vector& getData() { return mData; } const uint256& getHash() { return mHash; } From 9d5467feffd6780d71fe6ecbd807a47f417b78d2 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 06:15:19 -0700 Subject: [PATCH 09/11] Bugfix. --- src/HashedObject.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/HashedObject.cpp b/src/HashedObject.cpp index 512a0ab025..dd62f2594c 100644 --- a/src/HashedObject.cpp +++ b/src/HashedObject.cpp @@ -53,7 +53,7 @@ void HashedObjectStore::bulkWrite() Log(lsINFO) << "HOS: BulkWrite " << set.size(); boost::format fExists("SELECT ObjType FROM CommittedObjects WHERE Hash = '%s';"); - boost::format fAdd("INSERT INTO ComittedObject (Hash,ObjType,LedgerIndex,Object) VALUES ('%s','%c','%u','%s');"); + boost::format fAdd("INSERT INTO ComittedObjects (Hash,ObjType,LedgerIndex,Object) VALUES ('%s','%c','%u',%s);"); Database* db = theApp->getHashNodeDB()->getDB(); ScopedLock sl = theApp->getHashNodeDB()->getDBLock(); From 29586c8e7d5956c78b1f16845cdbd229bb42d689 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 06:16:32 -0700 Subject: [PATCH 10/11] Typo. --- src/HashedObject.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/HashedObject.cpp b/src/HashedObject.cpp index dd62f2594c..36f4273bc2 100644 --- a/src/HashedObject.cpp +++ b/src/HashedObject.cpp @@ -53,7 +53,7 @@ void HashedObjectStore::bulkWrite() Log(lsINFO) << "HOS: BulkWrite " << set.size(); boost::format fExists("SELECT ObjType FROM CommittedObjects WHERE Hash = '%s';"); - boost::format fAdd("INSERT INTO ComittedObjects (Hash,ObjType,LedgerIndex,Object) VALUES ('%s','%c','%u',%s);"); + boost::format fAdd("INSERT INTO CommittedObjects (Hash,ObjType,LedgerIndex,Object) VALUES ('%s','%c','%u',%s);"); Database* db = theApp->getHashNodeDB()->getDB(); ScopedLock sl = theApp->getHashNodeDB()->getDBLock(); From 324992b2ad99494655d4e9065b60720cb1c9ad85 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 06:31:03 -0700 Subject: [PATCH 11/11] Bugfixes. --- src/Application.cpp | 2 +- src/Application.h | 4 ++-- src/Ledger.cpp | 15 +++++++++++---- src/LedgerHistory.cpp | 8 +++++--- src/SHAMapSync.h | 18 ++++++------------ 5 files changed, 25 insertions(+), 22 deletions(-) diff --git a/src/Application.cpp b/src/Application.cpp index e500b39eeb..6b7ff795a2 100644 --- a/src/Application.cpp +++ b/src/Application.cpp @@ -37,7 +37,7 @@ DatabaseCon::~DatabaseCon() Application::Application() : mUNL(mIOService), - mNetOps(mIOService, &mMasterLedger), mNodeCache(16384, 600), mHashedObjectStore(16384, 300), + mNetOps(mIOService, &mMasterLedger), mTempNodeCache(16384, 90), mHashedObjectStore(16384, 300), mTxnDB(NULL), mLedgerDB(NULL), mWalletDB(NULL), mHashNodeDB(NULL), mNetNodeDB(NULL), mConnectionPool(mIOService), mPeerDoor(NULL), mRPCDoor(NULL) { diff --git a/src/Application.h b/src/Application.h index b63ff58d41..ff0024adc7 100644 --- a/src/Application.h +++ b/src/Application.h @@ -46,7 +46,7 @@ class Application LedgerAcquireMaster mMasterLedgerAcquire; TransactionMaster mMasterTransaction; NetworkOPs mNetOps; - NodeCache mNodeCache; + NodeCache mTempNodeCache; ValidationCollection mValidations; SuppressionTable mSuppressions; HashedObjectStore mHashedObjectStore; @@ -80,7 +80,7 @@ public: LedgerMaster& getMasterLedger() { return mMasterLedger; } LedgerAcquireMaster& getMasterLedgerAcquire() { return mMasterLedgerAcquire; } TransactionMaster& getMasterTransaction() { return mMasterTransaction; } - NodeCache& getNodeCache() { return mNodeCache; } + NodeCache& getTempNodeCache() { return mTempNodeCache; } HashedObjectStore& getHashedObjectStore() { return mHashedObjectStore; } ValidationCollection& getValidations() { return mValidations; } bool isNew(const uint256& s) { return mSuppressions.addSuppression(s); } diff --git a/src/Ledger.cpp b/src/Ledger.cpp index e583647a6a..1ab8fb3d5c 100644 --- a/src/Ledger.cpp +++ b/src/Ledger.cpp @@ -5,6 +5,8 @@ #include #include +#include "../json/writer.h" + #include "Application.h" #include "Ledger.h" #include "utils.h" @@ -350,6 +352,11 @@ Ledger::pointer Ledger::getSQL(const std::string& sql) boost::make_shared(prevHash, transHash, accountHash, totCoins, closingTime, ledgerSeq); if (ret->getHash() != ledgerHash) { + Json::StyledStreamWriter ssw; + Log(lsERROR) << "Failed on ledger"; + Json::Value p; + ret->addJson(p, LEDGER_JSON_FULL); + ssw.write(Log(lsERROR).ref(), p); assert(false); return Ledger::pointer(); } @@ -379,7 +386,8 @@ void Ledger::addJson(Json::Value& ret, int options) boost::recursive_mutex::scoped_lock sl(mLock); ledger["parentHash"] = mParentHash.GetHex(); - if(mClosed) + bool full = (options & LEDGER_JSON_FULL) != 0; + if(mClosed || full) { ledger["hash"] = mHash.GetHex(); ledger["transactionHash"] = mTransHash.GetHex(); @@ -391,8 +399,7 @@ void Ledger::addJson(Json::Value& ret, int options) else ledger["closed"] = false; if (mCloseTime != 0) ledger["closeTime"] = boost::posix_time::to_simple_string(ptFromSeconds(mCloseTime)); - bool full = (options & LEDGER_JSON_FULL) != 0; - if (full || ((options & LEDGER_JSON_DUMP_TXNS) != 0)) + if (mTransactionMap && (full || ((options & LEDGER_JSON_DUMP_TXNS) != 0))) { Json::Value txns(Json::arrayValue); for (SHAMapItem::pointer item = mTransactionMap->peekFirstItem(); !!item; @@ -408,7 +415,7 @@ void Ledger::addJson(Json::Value& ret, int options) } ledger["transactions"] = txns; } - if (full || ((options & LEDGER_JSON_DUMP_STATE) != 0)) + if (mAccountStateMap && (full || ((options & LEDGER_JSON_DUMP_STATE) != 0))) { Json::Value state(Json::arrayValue); for (SHAMapItem::pointer item = mAccountStateMap->peekFirstItem(); !!item; diff --git a/src/LedgerHistory.cpp b/src/LedgerHistory.cpp index f20a7ae53a..e42e13020e 100644 --- a/src/LedgerHistory.cpp +++ b/src/LedgerHistory.cpp @@ -60,9 +60,11 @@ Ledger::pointer LedgerHistory::getLedgerByHash(const uint256& hash) Ledger::pointer ret = mLedgersByHash.fetch(hash); if (ret) return ret; - ret = Ledger::loadByHash(hash); - if (!ret) return ret; - assert(ret->getHash() == hash); +// FIXME: A ledger without SHA maps isn't very useful +// This code will need to build them +// ret = Ledger::loadByHash(hash); +// if (!ret) return ret; +// assert(ret->getHash() == hash); boost::recursive_mutex::scoped_lock sl(mLedgersByHash.peekMutex()); mLedgersByHash.canonicalize(hash, ret); diff --git a/src/SHAMapSync.h b/src/SHAMapSync.h index 26b6185520..5eaf25acc5 100644 --- a/src/SHAMapSync.h +++ b/src/SHAMapSync.h @@ -15,12 +15,12 @@ public: const std::vector& nodeData, bool isLeaf) { // WRITEME: If 'isLeaf' is true, this is a transaction - theApp->getNodeCache().store(nodeHash, nodeData); + theApp->getTempNodeCache().store(nodeHash, nodeData); } virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) { // WRITEME: We could check our own map, we could check transaction tables - return theApp->getNodeCache().retrieve(nodeHash, nodeData); + return theApp->getTempNodeCache().retrieve(nodeHash, nodeData); } }; @@ -40,11 +40,8 @@ public: theApp->getHashedObjectStore().store(ACCOUNT_NODE, mLedgerSeq, nodeData, nodeHash); } virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) - { - HashedObject::pointer node = theApp->getHashedObjectStore().retrieve(nodeHash); - if (!node) return false; - nodeData = node->getData(); - return true; + { // fetchNode already tried + return false; } }; @@ -64,11 +61,8 @@ public: theApp->getHashedObjectStore().store(isLeaf ? TRANSACTION : TRANSACTION_NODE, mLedgerSeq, nodeData, nodeHash); } virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) - { - HashedObject::pointer node = theApp->getHashedObjectStore().retrieve(nodeHash); - if (!node) return false; - nodeData = node->getData(); - return true; + { // fetchNode already tried + return false; } };