From 98190a4284fb4f76dd778de3c0f6ffeb01c96a70 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 4 Jun 2012 06:00:27 -0700 Subject: [PATCH 1/9] Add an assert --- src/TransactionEngine.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/TransactionEngine.cpp b/src/TransactionEngine.cpp index ccfc88be41..c31f3c8f27 100644 --- a/src/TransactionEngine.cpp +++ b/src/TransactionEngine.cpp @@ -237,6 +237,7 @@ TransactionEngineResult TransactionEngine::applyTransaction(const SerializedTran std::cerr << "applyTransaction>" << std::endl; mLedger = mDefaultLedger; + assert(mLedger); if (mAlternateLedger && (targetLedger != 0) && (targetLedger != mLedger->getLedgerSeq()) && (targetLedger == mAlternateLedger->getLedgerSeq())) { From 2206ff3be6882bc08ce9acc97fc2f1c14a4c7009 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 4 Jun 2012 06:01:11 -0700 Subject: [PATCH 2/9] Get rid of hash_SMN and instead extend boost::hash. This makes the TaggedCache code cleaner. --- src/LedgerConsensus.h | 12 ++++++------ src/NetworkOPs.cpp | 2 +- src/Peer.cpp | 4 +++- src/SHAMap.cpp | 6 +++--- src/SHAMap.h | 14 ++++---------- src/TaggedCache.h | 23 ++++++++++++----------- 6 files changed, 29 insertions(+), 32 deletions(-) diff --git a/src/LedgerConsensus.h b/src/LedgerConsensus.h index e2b69e8fd2..9541f926c3 100644 --- a/src/LedgerConsensus.h +++ b/src/LedgerConsensus.h @@ -47,7 +47,7 @@ protected: int mYays, mNays; bool mOurPosition; std::vector transaction; - boost::unordered_map mVotes; + boost::unordered_map mVotes; public: typedef boost::shared_ptr pointer; @@ -85,17 +85,17 @@ protected: LedgerProposal::pointer mOurPosition; // Convergence tracking, trusted peers indexed by hash of public key - boost::unordered_map mPeerPositions; + boost::unordered_map mPeerPositions; // Transaction Sets, indexed by hash of transaction tree - boost::unordered_map mComplete; - boost::unordered_map mAcquiring; + boost::unordered_map mComplete; + boost::unordered_map mAcquiring; // Peer sets - boost::unordered_map >, hash_SMN> mPeerData; + boost::unordered_map > > mPeerData; // Disputed transactions - boost::unordered_map mDisputes; + boost::unordered_map mDisputes; // final accept logic static void Saccept(boost::shared_ptr This, SHAMap::pointer txSet); diff --git a/src/NetworkOPs.cpp b/src/NetworkOPs.cpp index 64cdc2ca7d..33838890bd 100644 --- a/src/NetworkOPs.cpp +++ b/src/NetworkOPs.cpp @@ -285,7 +285,7 @@ void NetworkOPs::checkState(const boost::system::error_code& result) // Do we have sufficient validations for our last closed ledger? Or do sufficient nodes // agree? And do we have no better ledger available? // If so, we are either tracking or full. - boost::unordered_map ledgers; + boost::unordered_map ledgers; for (std::vector::iterator it = peerList.begin(), end = peerList.end(); it != end; ++it) { diff --git a/src/Peer.cpp b/src/Peer.cpp index 02b975cae5..893db52ab0 100644 --- a/src/Peer.cpp +++ b/src/Peer.cpp @@ -726,6 +726,7 @@ void Peer::recvGetLedger(newcoin::TMGetLedger& packet) { SHAMap::pointer map; newcoin::TMLedgerData reply; + bool fatLeaves = true; if (packet.itype() == newcoin::liTS_CANDIDATE) { // Request is for a transaction candidate set @@ -748,6 +749,7 @@ void Peer::recvGetLedger(newcoin::TMGetLedger& packet) reply.set_ledgerseq(0); reply.set_ledgerhash(txHash.begin(), txHash.size()); reply.set_type(newcoin::liTS_CANDIDATE); + fatLeaves = false; // We'll already have most transactions } else { // Figure out what ledger they want @@ -828,7 +830,7 @@ void Peer::recvGetLedger(newcoin::TMGetLedger& packet) } std::vector nodeIDs; std::list< std::vector > rawNodes; - if(map->getNodeFat(mn, nodeIDs, rawNodes)) + if(map->getNodeFat(mn, nodeIDs, rawNodes, fatLeaves)) { std::vector::iterator nodeIDIterator; std::list< std::vector >::iterator rawNodeIterator; diff --git a/src/SHAMap.cpp b/src/SHAMap.cpp index 31ef55c436..709a732b1c 100644 --- a/src/SHAMap.cpp +++ b/src/SHAMap.cpp @@ -15,14 +15,14 @@ #include "SHAMap.h" #include "Application.h" -std::size_t hash_SMN::operator() (const SHAMapNode& mn) const +std::size_t hash_value(const SHAMapNode& mn) { return mn.getDepth() ^ *reinterpret_cast(mn.getNodeID().begin()) ^ *reinterpret_cast(theApp->getNonce256().begin()); } -std::size_t hash_SMN::operator() (const uint256& u) const +std::size_t hash_value(const uint256& u) { return *reinterpret_cast(u.begin()) ^ *reinterpret_cast(theApp->getNonce256().begin()); @@ -671,7 +671,7 @@ void SHAMap::dump(bool hash) std::cerr << " MAP Contains" << std::endl; boost::recursive_mutex::scoped_lock sl(mLock); - for(boost::unordered_map::iterator it = mTNByID.begin(); + for(boost::unordered_map::iterator it = mTNByID.begin(); it != mTNByID.end(); ++it) { std::cerr << it->second->getString() << std::endl; diff --git a/src/SHAMap.h b/src/SHAMap.h index d5c79586d6..329ce65f6e 100644 --- a/src/SHAMap.h +++ b/src/SHAMap.h @@ -76,14 +76,8 @@ public: SHAMapNode(const void *ptr, int len); }; -class hash_SMN -{ - -public: - std::size_t operator() (const SHAMapNode& mn) const; - - std::size_t operator() (const uint256& u) const; -}; +extern std::size_t hash_value(const SHAMapNode& mn); +extern std::size_t hash_value(const uint256& u); class SHAMapItem { // an item stored in a SHAMap @@ -234,7 +228,7 @@ public: private: uint32 mSeq; mutable boost::recursive_mutex mLock; - boost::unordered_map mTNByID; + boost::unordered_map mTNByID; boost::shared_ptr > mDirtyNodes; @@ -301,7 +295,7 @@ public: // comparison/sync functions void getMissingNodes(std::vector& nodeIDs, std::vector& hashes, int max); bool getNodeFat(const SHAMapNode& node, std::vector& nodeIDs, - std::list >& rawNode); + std::list >& rawNode, bool fatLeaves); bool addRootNode(const uint256& hash, const std::vector& rootNode); bool addRootNode(const std::vector& rootNode); bool addKnownNode(const SHAMapNode& nodeID, const std::vector& rawNode); diff --git a/src/TaggedCache.h b/src/TaggedCache.h index f5c501d0a9..6f79e172c5 100644 --- a/src/TaggedCache.h +++ b/src/TaggedCache.h @@ -4,6 +4,7 @@ #include #include +#include #include // This class implemented a cache and a map. The cache keeps objects alive @@ -25,10 +26,10 @@ protected: mutable boost::recursive_mutex mLock; int mTargetSize, mTargetAge; - std::map mCache; // Hold strong reference to recent objects + boost::unordered_map mCache; // Hold strong reference to recent objects time_t mLastSweep; - std::map mMap; // Track stored objects + boost::unordered_map mMap; // Track stored objects public: TaggedCache(int size, int age) : mTargetSize(size), mTargetAge(age), mLastSweep(time(NULL)) { ; } @@ -81,7 +82,7 @@ template void TaggedCache::sweep time_t target = now - mTargetAge; // Pass 1, remove old objects from cache - typename std::map::iterator cit = mCache.begin(); + typename boost::unordered_map::iterator cit = mCache.begin(); while (cit != mCache.end()) { if (cit->second->second.first < target) @@ -90,7 +91,7 @@ template void TaggedCache::sweep } // Pass 2, remove dead objects from map - typename std::map::iterator mit = mMap.begin(); + typename boost::unordered_map::iterator mit = mMap.begin(); while (mit != mMap.end()) { if (mit->second->expired()) @@ -104,7 +105,7 @@ template bool TaggedCache::touch boost::recursive_mutex::scoped_lock sl(mLock); // Is the object in the map? - typename std::map::iterator mit = mMap.find(key); + typename boost::unordered_map::iterator mit = mMap.find(key); if (mit == mMap.end()) return false; if (mit->second->expired()) { // in map, but expired @@ -113,7 +114,7 @@ template bool TaggedCache::touch } // Is the object in the cache? - typename std::map::iterator cit = mCache.find(key); + typename boost::unordered_map::iterator cit = mCache.find(key); if (cit != mCache.end()) { // in both map and cache cit->second->first = time(NULL); @@ -129,7 +130,7 @@ template bool TaggedCache::del(c { // Remove from cache, map unaffected boost::recursive_mutex::scoped_lock sl(mLock); - typename std::map::iterator cit = mCache.find(key); + typename boost::unordered_map::iterator cit = mCache.find(key); if (cit == mCache.end()) return false; mCache.erase(cit); return true; @@ -141,7 +142,7 @@ bool TaggedCache::canonicalize(const key_type& key, boost::shared // Return values: true=we had the data already boost::recursive_mutex::scoped_lock sl(mLock); - typename std::map::iterator mit = mMap.find(key); + typename boost::unordered_map::iterator mit = mMap.find(key); if (mit == mMap.end()) { // not in map mCache.insert(std::make_pair(key, std::make_pair(time(NULL), data))); @@ -159,7 +160,7 @@ bool TaggedCache::canonicalize(const key_type& key, boost::shared assert(!!data); // Valid in map, is it in cache? - typename std::map::iterator cit = mCache.find(key); + typename boost::unordered_map::iterator cit = mCache.find(key); if (cit != mCache.end()) cit->second.first = time(NULL); // Yes, refesh else // no, add to cache @@ -174,7 +175,7 @@ boost::shared_ptr TaggedCache::fetch(const key_type& key) boost::recursive_mutex::scoped_lock sl(mLock); // Is it in the map? - typename std::map::iterator mit = mMap.find(key); + typename boost::unordered_map::iterator mit = mMap.find(key); if (mit == mMap.end()) return data_ptr(); // No, we're done if (mit->second.expired()) { // in map, but expired @@ -185,7 +186,7 @@ boost::shared_ptr TaggedCache::fetch(const key_type& key) boost::shared_ptr data = mit->second.lock(); // Valid in map, is it in the cache? - typename std::map::iterator cit = mCache.find(key); + typename boost::unordered_map::iterator cit = mCache.find(key); if (cit != mCache.end()) cit->second.first = time(NULL); // Yes, refresh else // No, add to cache From 7dcb10b178915996aa5ead20c4136104633cfe88 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 4 Jun 2012 06:01:33 -0700 Subject: [PATCH 3/9] Make an option whether fat node gets should include leaves. This will save bandwidth and improve performance in ledger consensus. (Well, it will once the rest of the code is tied in.) --- src/SHAMapSync.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/SHAMapSync.cpp b/src/SHAMapSync.cpp index edc59f15c5..aca6891d54 100644 --- a/src/SHAMapSync.cpp +++ b/src/SHAMapSync.cpp @@ -58,7 +58,7 @@ void SHAMap::getMissingNodes(std::vector& nodeIDs, std::vector& nodeIDs, - std::list >& rawNodes) + std::list >& rawNodes, bool fatLeaves) { // Gets a node and some of its children boost::recursive_mutex::scoped_lock sl(mLock); @@ -82,7 +82,7 @@ bool SHAMap::getNodeFat(const SHAMapNode& wanted, std::vector& nodeI { SHAMapTreeNode::pointer nextNode = getNode(node->getChildNodeID(i), node->getChildHash(i), false); assert(nextNode); - if(nextNode) + if (nextNode && (fatLeaves || !nextNode->isLeaf())) { nodeIDs.push_back(*nextNode); Serializer s; @@ -387,7 +387,7 @@ BOOST_AUTO_TEST_CASE( SHAMapSync_test ) destination.setSynching(); - if (!source.getNodeFat(SHAMapNode(), nodeIDs, gotNodes)) + if (!source.getNodeFat(SHAMapNode(), nodeIDs, gotNodes, (rand() % 2) == 0)) { Log(lsFATAL) << "GetNodeFat(root) fails"; BOOST_FAIL("GetNodeFat"); @@ -423,7 +423,7 @@ BOOST_AUTO_TEST_CASE( SHAMapSync_test ) // get as many nodes as possible based on this information for (nodeIDIterator = nodeIDs.begin(); nodeIDIterator != nodeIDs.end(); ++nodeIDIterator) - if (!source.getNodeFat(*nodeIDIterator, gotNodeIDs, gotNodes)) + if (!source.getNodeFat(*nodeIDIterator, gotNodeIDs, gotNodes, (rand() % 2) == 0)) { Log(lsFATAL) << "GetNodeFat fails"; BOOST_FAIL("GetNodeFat"); From bef64eaad5c8f4494f56c3895acf18c7384ed0a4 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 4 Jun 2012 06:14:23 -0700 Subject: [PATCH 4/9] Nicer nonce/hash code. --- src/Application.cpp | 1 + src/Application.h | 14 ++++++++------ src/SHAMap.cpp | 10 +++++----- src/uint256.h | 9 +++++++++ 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/Application.cpp b/src/Application.cpp index 4a44115d94..92c009069b 100644 --- a/src/Application.cpp +++ b/src/Application.cpp @@ -37,6 +37,7 @@ Application::Application() : mConnectionPool(mIOService), mPeerDoor(NULL), mRPCDoor(NULL) { RAND_bytes(mNonce256.begin(), mNonce256.size()); + RAND_bytes(reinterpret_cast(&mNonceST), sizeof(mNonceST)); } extern const char *TxnDBInit[], *LedgerDBInit[], *WalletDBInit[], *HashNodeDBInit[], *NetNodeDBInit[]; diff --git a/src/Application.h b/src/Application.h index 96e58bdeef..bf763f3998 100644 --- a/src/Application.h +++ b/src/Application.h @@ -48,6 +48,7 @@ class Application RPCDoor* mRPCDoor; uint256 mNonce256; + std::size_t mNonceST; std::map mPeerMap; boost::recursive_mutex mPeerMapLock; @@ -69,13 +70,14 @@ public: LedgerAcquireMaster& getMasterLedgerAcquire() { return mMasterLedgerAcquire; } TransactionMaster& getMasterTransaction() { return mMasterTransaction; } - DatabaseCon* getTxnDB() { return mTxnDB; } - DatabaseCon* getLedgerDB() { return mLedgerDB; } - DatabaseCon* getWalletDB() { return mWalletDB; } - DatabaseCon* getHashNodeDB() { return mHashNodeDB; } - DatabaseCon* getNetNodeDB() { return mNetNodeDB; } + DatabaseCon* getTxnDB() { return mTxnDB; } + DatabaseCon* getLedgerDB() { return mLedgerDB; } + DatabaseCon* getWalletDB() { return mWalletDB; } + DatabaseCon* getHashNodeDB() { return mHashNodeDB; } + DatabaseCon* getNetNodeDB() { return mNetNodeDB; } - uint256 getNonce256() { return mNonce256; } + uint256 getNonce256() { return mNonce256; } + std::size_t getNonceST() { return mNonceST; } void run(); void stop(); diff --git a/src/SHAMap.cpp b/src/SHAMap.cpp index 709a732b1c..c84eec80e0 100644 --- a/src/SHAMap.cpp +++ b/src/SHAMap.cpp @@ -17,15 +17,15 @@ std::size_t hash_value(const SHAMapNode& mn) { - return mn.getDepth() - ^ *reinterpret_cast(mn.getNodeID().begin()) - ^ *reinterpret_cast(theApp->getNonce256().begin()); + std::size_t seed = theApp->getNonceST(); + boost::hash_combine(seed, mn.getDepth()); + return mn.getNodeID().hash_combine(seed); } std::size_t hash_value(const uint256& u) { - return *reinterpret_cast(u.begin()) - ^ *reinterpret_cast(theApp->getNonce256().begin()); + std::size_t seed = theApp->getNonceST(); + return u.hash_combine(seed); } SHAMap::SHAMap(uint32 seq) : mSeq(seq), mState(Modifying) diff --git a/src/uint256.h b/src/uint256.h index 78b9b169c4..356433e619 100644 --- a/src/uint256.h +++ b/src/uint256.h @@ -13,6 +13,8 @@ #include #include +#include + #include "types.h" #include "utils.h" @@ -136,6 +138,13 @@ public: return ret; } + std::size_t hash_combine(std::size_t& seed) const + { + for (int i = 0; i < WIDTH; ++i) + boost::hash_combine(seed, pn[i]); + return seed; + } + friend inline int compare(const base_uint& a, const base_uint& b) { const unsigned char* pA = a.begin(); From ba163bc89fcd8f600e62337aefa5cc540cff6879 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 4 Jun 2012 07:17:41 -0700 Subject: [PATCH 5/9] Logging modernization. --- src/Transaction.cpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/Transaction.cpp b/src/Transaction.cpp index 4a60814ffb..947533ce21 100644 --- a/src/Transaction.cpp +++ b/src/Transaction.cpp @@ -11,6 +11,7 @@ #include "BitcoinUtil.h" #include "Serializer.h" #include "SerializedTransaction.h" +#include "Log.h" Transaction::Transaction(const SerializedTransaction::pointer sit, bool bValidate) : mInLedger(0), mStatus(INVALID), mTransaction(sit) @@ -43,6 +44,7 @@ Transaction::pointer Transaction::sharedTransaction(const std::vector(); } } @@ -66,8 +68,8 @@ Transaction::Transaction( mTransaction = boost::make_shared(ttKind); - std::cerr << str(boost::format("Transaction: account: %s") % naSourceAccount.humanAccountID()) << std::endl; - std::cerr << str(boost::format("Transaction: mAccountFrom: %s") % mAccountFrom.humanAccountID()) << std::endl; + Log(lsINFO) << str(boost::format("Transaction: account: %s") % naSourceAccount.humanAccountID()); + Log(lsINFO) << str(boost::format("Transaction: mAccountFrom: %s") % mAccountFrom.humanAccountID()); mTransaction->setSigningPubKey(mFromPubKey); mTransaction->setSourceAccount(mAccountFrom); mTransaction->setSequence(uSeq); @@ -86,16 +88,12 @@ bool Transaction::sign(const NewcoinAddress& naAccountPrivate) if (!naAccountPrivate.isValid()) { -#ifdef DEBUG - std::cerr << "No private key for signing" << std::endl; -#endif + Log(lsWARNING) << "No private key for signing"; bResult = false; } else if (!getSTransaction()->sign(naAccountPrivate)) { -#ifdef DEBUG - std::cerr << "Failed to make signature" << std::endl; -#endif + Log(lsWARNING) << "Failed to make signature"; assert(false); bResult = false; } From 27241a9a5ce30eb3a85bb00d0eccaee1ede26233 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 4 Jun 2012 07:17:54 -0700 Subject: [PATCH 6/9] Application-level tagged cache. --- src/Application.cpp | 6 ++++-- src/Application.h | 23 ++++++++++++++--------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/Application.cpp b/src/Application.cpp index 92c009069b..37ca4f3952 100644 --- a/src/Application.cpp +++ b/src/Application.cpp @@ -12,8 +12,10 @@ #include "BitcoinUtil.h" #include "key.h" #include "utils.h" +#include "TaggedCache.h" -Application* theApp=NULL; + +Application* theApp = NULL; DatabaseCon::DatabaseCon(const std::string& name, const char *initStrings[], int initCount) { @@ -32,7 +34,7 @@ DatabaseCon::~DatabaseCon() Application::Application() : mUNL(mIOService), - mNetOps(mIOService, &mMasterLedger), + mNetOps(mIOService, &mMasterLedger), mNodeCache(16384, 600), mTxnDB(NULL), mLedgerDB(NULL), mWalletDB(NULL), mHashNodeDB(NULL), mNetNodeDB(NULL), mConnectionPool(mIOService), mPeerDoor(NULL), mRPCDoor(NULL) { diff --git a/src/Application.h b/src/Application.h index bf763f3998..55d405c66d 100644 --- a/src/Application.h +++ b/src/Application.h @@ -1,6 +1,8 @@ #ifndef __APPLICATION__ #define __APPLICATION__ +#include + #include "LedgerMaster.h" #include "UniqueNodeList.h" #include "ConnectionPool.h" @@ -10,12 +12,13 @@ #include "Wallet.h" #include "Peer.h" #include "NetworkOPs.h" +#include "TaggedCache.h" #include "../database/database.h" -#include class RPCDoor; class PeerDoor; +typedef TaggedCache< uint256, std::vector > NodeCache; class DatabaseCon { @@ -40,6 +43,7 @@ class Application LedgerAcquireMaster mMasterLedgerAcquire; TransactionMaster mMasterTransaction; NetworkOPs mNetOps; + NodeCache mNodeCache; DatabaseCon* mTxnDB, *mLedgerDB, *mWalletDB, *mHashNodeDB, *mNetNodeDB; @@ -57,18 +61,19 @@ public: Application(); ~Application(); - ConnectionPool& getConnectionPool() { return mConnectionPool; } + ConnectionPool& getConnectionPool() { return mConnectionPool; } - UniqueNodeList& getUNL() { return mUNL; } + UniqueNodeList& getUNL() { return mUNL; } - Wallet& getWallet() { return mWallet ; } - NetworkOPs& getOPs() { return mNetOps; } + Wallet& getWallet() { return mWallet ; } + NetworkOPs& getOPs() { return mNetOps; } - boost::asio::io_service& getIOService() { return mIOService; } + boost::asio::io_service& getIOService() { return mIOService; } - LedgerMaster& getMasterLedger() { return mMasterLedger; } - LedgerAcquireMaster& getMasterLedgerAcquire() { return mMasterLedgerAcquire; } - TransactionMaster& getMasterTransaction() { return mMasterTransaction; } + LedgerMaster& getMasterLedger() { return mMasterLedger; } + LedgerAcquireMaster& getMasterLedgerAcquire() { return mMasterLedgerAcquire; } + TransactionMaster& getMasterTransaction() { return mMasterTransaction; } + NodeCache& getNodeCache() { return mNodeCache; } DatabaseCon* getTxnDB() { return mTxnDB; } DatabaseCon* getLedgerDB() { return mLedgerDB; } From 0e7307200ebe7f776a5f33d66c683ff6782e7a83 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 4 Jun 2012 07:18:57 -0700 Subject: [PATCH 7/9] First set of efficiency improvements. --- src/LedgerAcquire.cpp | 10 +++++----- src/LedgerAcquire.h | 27 +++++++++++++++++++++++++++ src/LedgerConsensus.cpp | 7 ++++--- src/LedgerConsensus.h | 1 + src/SHAMap.h | 17 +++++++++++++++-- src/SHAMapSync.cpp | 30 ++++++++++++++++++++++++++---- 6 files changed, 78 insertions(+), 14 deletions(-) diff --git a/src/LedgerAcquire.cpp b/src/LedgerAcquire.cpp index 34bd631e29..1a71d79bc1 100644 --- a/src/LedgerAcquire.cpp +++ b/src/LedgerAcquire.cpp @@ -67,7 +67,7 @@ void PeerSet::TimerEntry(boost::weak_ptr wptr, const boost::system::err } LedgerAcquire::LedgerAcquire(const uint256& hash) : PeerSet(hash, LEDGER_ACQUIRE_TIMEOUT), - mHaveBase(false), mHaveState(false), mHaveTransactions(false) + mFilter(&theApp->getNodeCache()), mHaveBase(false), mHaveState(false), mHaveTransactions(false) { #ifdef DEBUG std::cerr << "Acquiring ledger " << mHash.GetHex() << std::endl; @@ -152,7 +152,7 @@ void LedgerAcquire::trigger(Peer::pointer peer) { std::vector nodeIDs; std::vector nodeHashes; - mLedger->peekTransactionMap()->getMissingNodes(nodeIDs, nodeHashes, 128); + mLedger->peekTransactionMap()->getMissingNodes(nodeIDs, nodeHashes, 128, &mFilter); if (nodeIDs.empty()) { if (!mLedger->peekTransactionMap()->isValid()) mFailed = true; @@ -204,7 +204,7 @@ void LedgerAcquire::trigger(Peer::pointer peer) { std::vector nodeIDs; std::vector nodeHashes; - mLedger->peekAccountStateMap()->getMissingNodes(nodeIDs, nodeHashes, 128); + mLedger->peekAccountStateMap()->getMissingNodes(nodeIDs, nodeHashes, 128, &mFilter); if (nodeIDs.empty()) { if (!mLedger->peekAccountStateMap()->isValid()) mFailed = true; @@ -303,7 +303,7 @@ bool LedgerAcquire::takeTxNode(const std::list& nodeIDs, if (!mLedger->peekTransactionMap()->addRootNode(mLedger->getTransHash(), *nodeDatait)) return false; } - else if (!mLedger->peekTransactionMap()->addKnownNode(*nodeIDit, *nodeDatait)) + else if (!mLedger->peekTransactionMap()->addKnownNode(*nodeIDit, *nodeDatait, &mFilter)) return false; ++nodeIDit; ++nodeDatait; @@ -333,7 +333,7 @@ bool LedgerAcquire::takeAsNode(const std::list& nodeIDs, if (!mLedger->peekAccountStateMap()->addRootNode(mLedger->getAccountHash(), *nodeDatait)) return false; } - else if (!mLedger->peekAccountStateMap()->addKnownNode(*nodeIDit, *nodeDatait)) + else if (!mLedger->peekAccountStateMap()->addKnownNode(*nodeIDit, *nodeDatait, &mFilter)) return false; ++nodeIDit; ++nodeDatait; diff --git a/src/LedgerAcquire.h b/src/LedgerAcquire.h index 7e92d024f2..bffe386998 100644 --- a/src/LedgerAcquire.h +++ b/src/LedgerAcquire.h @@ -13,6 +13,7 @@ #include "Ledger.h" #include "Peer.h" +#include "TaggedCache.h" #include "../obj/src/newcoin.pb.h" class PeerSet @@ -53,6 +54,30 @@ private: static void TimerEntry(boost::weak_ptr, const boost::system::error_code& result); }; +typedef TaggedCache< uint256, std::vector > NodeCache; +typedef std::vector VUC; + +class THSyncFilter : public SHAMapSyncFilter +{ +protected: + NodeCache* mCache; + +public: + THSyncFilter(NodeCache* cache) : mCache(cache) { ; } + virtual void gotNode(const uint256& nodeHash, const std::vector& nodeData, bool) + { + boost::shared_ptr ptr = boost::make_shared(nodeData); + mCache->canonicalize(nodeHash, ptr); + } + virtual bool haveNode(const uint256& nodeHash, std::vector& nodeData) + { + boost::shared_ptr entry = mCache->fetch(nodeHash); + if (!entry) return false; + nodeData = *entry; + return true; + } +}; + class LedgerAcquire : public PeerSet, public boost::enable_shared_from_this { // A ledger we are trying to acquire public: @@ -60,6 +85,7 @@ public: protected: Ledger::pointer mLedger; + THSyncFilter mFilter; bool mHaveBase, mHaveState, mHaveTransactions; std::vector< boost::function > mOnComplete; @@ -106,4 +132,5 @@ public: }; #endif + // vim:ts=4 diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index e2c95fbe84..d1bae73334 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -11,7 +11,8 @@ #define TRUST_NETWORK -TransactionAcquire::TransactionAcquire(const uint256& hash) : PeerSet(hash, 1), mHaveRoot(false) +TransactionAcquire::TransactionAcquire(const uint256& hash) + : PeerSet(hash, 1), mFilter(&theApp->getNodeCache()), mHaveRoot(false) { mMap = boost::make_shared(); mMap->setSynching(); @@ -48,7 +49,7 @@ void TransactionAcquire::trigger(Peer::pointer peer) Log(lsTRACE) << "Have root"; std::vector nodeIDs; std::vector nodeHashes; - mMap->getMissingNodes(nodeIDs, nodeHashes, 256); + mMap->getMissingNodes(nodeIDs, nodeHashes, 256, &mFilter); if (nodeIDs.empty()) { if (mMap->isValid()) @@ -100,7 +101,7 @@ bool TransactionAcquire::takeNodes(const std::list& nodeIDs, return false; else mHaveRoot = true; } - else if (!mMap->addKnownNode(*nodeIDit, *nodeDatait)) + else if (!mMap->addKnownNode(*nodeIDit, *nodeDatait, &mFilter)) return false; ++nodeIDit; ++nodeDatait; diff --git a/src/LedgerConsensus.h b/src/LedgerConsensus.h index 9541f926c3..3a6aeaecde 100644 --- a/src/LedgerConsensus.h +++ b/src/LedgerConsensus.h @@ -21,6 +21,7 @@ public: protected: SHAMap::pointer mMap; + THSyncFilter mFilter; // FIXME: Should use transaction master too bool mHaveRoot; void onTimer() { trigger(Peer::pointer()); } diff --git a/src/SHAMap.h b/src/SHAMap.h index 329ce65f6e..175958ab49 100644 --- a/src/SHAMap.h +++ b/src/SHAMap.h @@ -219,6 +219,17 @@ enum SHAMapState Invalid = 4, // Map is known not to be valid (usually synching a corrupt ledger) }; +class SHAMapSyncFilter +{ +public: + SHAMapSyncFilter() { ; } + virtual ~SHAMapSyncFilter() { ; } + virtual void gotNode(const uint256& nodeHash, const std::vector& nodeData, bool isLeaf) + { ; } + virtual bool haveNode(const uint256&nodeHash, std::vector& nodeData) + { return false; } +}; + class SHAMap { public: @@ -293,12 +304,14 @@ public: SHAMapItem::pointer peekPrevItem(const uint256&); // comparison/sync functions - void getMissingNodes(std::vector& nodeIDs, std::vector& hashes, int max); + void getMissingNodes(std::vector& nodeIDs, std::vector& hashes, int max, + SHAMapSyncFilter* filter); bool getNodeFat(const SHAMapNode& node, std::vector& nodeIDs, std::list >& rawNode, bool fatLeaves); bool addRootNode(const uint256& hash, const std::vector& rootNode); bool addRootNode(const std::vector& rootNode); - bool addKnownNode(const SHAMapNode& nodeID, const std::vector& rawNode); + bool addKnownNode(const SHAMapNode& nodeID, const std::vector& rawNode, + SHAMapSyncFilter* filter); // status functions void setImmutable(void) { assert(mState != Invalid); mState = Immutable; } diff --git a/src/SHAMapSync.cpp b/src/SHAMapSync.cpp index aca6891d54..54c7a0d20d 100644 --- a/src/SHAMapSync.cpp +++ b/src/SHAMapSync.cpp @@ -11,7 +11,8 @@ #include "Log.h" -void SHAMap::getMissingNodes(std::vector& nodeIDs, std::vector& hashes, int max) +void SHAMap::getMissingNodes(std::vector& nodeIDs, std::vector& hashes, int max, + SHAMapSyncFilter* filter) { boost::recursive_mutex::scoped_lock sl(mLock); @@ -44,6 +45,24 @@ void SHAMap::getMissingNodes(std::vector& nodeIDs, std::vectorisEmptyBranch(branch)) { SHAMapTreeNode::pointer d = getNode(node->getChildNodeID(branch), node->getChildHash(branch), false); + if ((!d) && (filter != NULL)) + { + std::vector nodeData; + if (filter->haveNode(node->getChildHash(branch), nodeData)) + { + d = boost::make_shared(node->getChildNodeID(branch), nodeData, mSeq); + if (node->getChildHash(branch) != d->getNodeHash()) + { + Log(lsERROR) << "Wrong hash from cached object"; + d = SHAMapTreeNode::pointer(); + } + else + { + Log(lsTRACE) << "Got sync node from cache"; + mTNByID[*d] = d; + } + } + } if (!d) { nodeIDs.push_back(node->getChildNodeID(branch)); @@ -153,7 +172,8 @@ bool SHAMap::addRootNode(const uint256& hash, const std::vector& return true; } -bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vector& rawNode) +bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vector& rawNode, + SHAMapSyncFilter* filter) { // return value: true=okay, false=error assert(!node.isRoot()); if (!isSynching()) return false; @@ -199,6 +219,8 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vectorgetNodeHash()) // these aren't the droids we're looking for return false; + if (filter) filter->gotNode(hash, rawNode, newNode->isLeaf()); + mTNByID[*newNode] = newNode; if (!newNode->isLeaf()) return true; // only a leaf can fill a branch @@ -416,7 +438,7 @@ BOOST_AUTO_TEST_CASE( SHAMapSync_test ) hashes.clear(); // get the list of nodes we know we need - destination.getMissingNodes(nodeIDs, hashes, 2048); + destination.getMissingNodes(nodeIDs, hashes, 2048, NULL); if(nodeIDs.empty()) break; Log(lsINFO) << nodeIDs.size() << " needed nodes"; @@ -446,7 +468,7 @@ BOOST_AUTO_TEST_CASE( SHAMapSync_test ) #ifdef SMS_DEBUG bytes += rawNodeIterator->size(); #endif - if (!destination.addKnownNode(*nodeIDIterator, *rawNodeIterator)) + if (!destination.addKnownNode(*nodeIDIterator, *rawNodeIterator, NULL)) { Log(lsTRACE) << "AddKnownNode fails"; BOOST_FAIL("AddKnownNode"); From 0648779ccb2cb4ad4f5036b7c05aaceac5b656a9 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 4 Jun 2012 07:23:56 -0700 Subject: [PATCH 8/9] Pass node IDs to the sync filter. --- src/LedgerAcquire.h | 5 +++-- src/SHAMap.h | 5 +++-- src/SHAMapSync.cpp | 12 +++++++----- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/LedgerAcquire.h b/src/LedgerAcquire.h index bffe386998..f06c4dbabf 100644 --- a/src/LedgerAcquire.h +++ b/src/LedgerAcquire.h @@ -64,12 +64,13 @@ protected: public: THSyncFilter(NodeCache* cache) : mCache(cache) { ; } - virtual void gotNode(const uint256& nodeHash, const std::vector& nodeData, bool) + virtual void gotNode(const SHAMapNode& id, const uint256& nodeHash, + const std::vector& nodeData, bool) { boost::shared_ptr ptr = boost::make_shared(nodeData); mCache->canonicalize(nodeHash, ptr); } - virtual bool haveNode(const uint256& nodeHash, std::vector& nodeData) + virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) { boost::shared_ptr entry = mCache->fetch(nodeHash); if (!entry) return false; diff --git a/src/SHAMap.h b/src/SHAMap.h index 175958ab49..b9090c963d 100644 --- a/src/SHAMap.h +++ b/src/SHAMap.h @@ -224,9 +224,10 @@ class SHAMapSyncFilter public: SHAMapSyncFilter() { ; } virtual ~SHAMapSyncFilter() { ; } - virtual void gotNode(const uint256& nodeHash, const std::vector& nodeData, bool isLeaf) + virtual void gotNode(const SHAMapNode& id, const uint256& nodeHash, + const std::vector& nodeData, bool isLeaf) { ; } - virtual bool haveNode(const uint256&nodeHash, std::vector& nodeData) + virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) { return false; } }; diff --git a/src/SHAMapSync.cpp b/src/SHAMapSync.cpp index 54c7a0d20d..dcc3b74d4f 100644 --- a/src/SHAMapSync.cpp +++ b/src/SHAMapSync.cpp @@ -44,14 +44,16 @@ void SHAMap::getMissingNodes(std::vector& nodeIDs, std::vectorisEmptyBranch(branch)) { - SHAMapTreeNode::pointer d = getNode(node->getChildNodeID(branch), node->getChildHash(branch), false); + SHAMapNode childID = node->getChildNodeID(branch); + const uint256& childHash = node->getChildHash(branch); + SHAMapTreeNode::pointer d = getNode(childID, childHash, false); if ((!d) && (filter != NULL)) { std::vector nodeData; - if (filter->haveNode(node->getChildHash(branch), nodeData)) + if (filter->haveNode(childID, childHash, nodeData)) { - d = boost::make_shared(node->getChildNodeID(branch), nodeData, mSeq); - if (node->getChildHash(branch) != d->getNodeHash()) + d = boost::make_shared(childID, nodeData, mSeq); + if (childHash != d->getNodeHash()) { Log(lsERROR) << "Wrong hash from cached object"; d = SHAMapTreeNode::pointer(); @@ -219,7 +221,7 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vectorgetNodeHash()) // these aren't the droids we're looking for return false; - if (filter) filter->gotNode(hash, rawNode, newNode->isLeaf()); + if (filter) filter->gotNode(node, hash, rawNode, newNode->isLeaf()); mTNByID[*newNode] = newNode; if (!newNode->isLeaf()) From dcf185057a4c3d1b927c6bdba10d9add6e334e66 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 4 Jun 2012 07:28:16 -0700 Subject: [PATCH 9/9] Whitespace. --- src/SHAMapSync.cpp | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/src/SHAMapSync.cpp b/src/SHAMapSync.cpp index dcc3b74d4f..d0af0c44d4 100644 --- a/src/SHAMapSync.cpp +++ b/src/SHAMapSync.cpp @@ -18,13 +18,13 @@ void SHAMap::getMissingNodes(std::vector& nodeIDs, std::vectorisValid()); - if(root->isFullBelow()) + if (root->isFullBelow()) { clearSynching(); return; } - if(!root->isInner()) + if (!root->isInner()) { Log(lsWARNING) << "synching empty tree"; return; @@ -86,7 +86,7 @@ bool SHAMap::getNodeFat(const SHAMapNode& wanted, std::vector& nodeI SHAMapTreeNode::pointer node = getNode(wanted); if (!node) { - assert(false); // Remove for release, this can happen if we get a bogus request + assert(false); // FIXME Remove for release, this can happen if we get a bogus request return false; } @@ -233,7 +233,7 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vectorisInner()); - for(int i = 0; i < 16; ++i) + for (int i = 0; i < 16; ++i) if (!iNode->isEmptyBranch(i)) { SHAMapTreeNode::pointer nextNode = getNode(iNode->getChildNodeID(i), iNode->getChildHash(i), false); @@ -258,7 +258,7 @@ bool SHAMap::deepCompare(SHAMap& other) stack.pop(); SHAMapTreeNode::pointer otherNode; - if(node->isRoot()) otherNode = other.root; + if (node->isRoot()) otherNode = other.root; else otherNode = other.getNode(*node, node->getNodeHash(), false); if (!otherNode) @@ -286,11 +286,11 @@ bool SHAMap::deepCompare(SHAMap& other) { if (!otherNode->isInner()) return false; - for(int i=0; i<16; i++) + for (int i = 0; i < 16; ++i) { - if(node->isEmptyBranch(i)) + if (node->isEmptyBranch(i)) { - if(!otherNode->isEmptyBranch(i)) return false; + if (!otherNode->isEmptyBranch(i)) return false; } else { @@ -314,9 +314,9 @@ bool SHAMap::deepCompare(SHAMap& other) static SHAMapItem::pointer makeRandomAS() { - Serializer s; - for(int d = 0; d < 3; ++d) s.add32(rand()); - return boost::make_shared(s.getRIPEMD160().to256(), s.peekData()); + Serializer s; + for (int d = 0; d < 3; ++d) s.add32(rand()); + return boost::make_shared(s.getRIPEMD160().to256(), s.peekData()); } static bool confuseMap(SHAMap &map, int count) @@ -386,25 +386,24 @@ BOOST_AUTO_TEST_CASE( SHAMapSync_test ) SHAMap source, destination; - // add random data to the source map int items = 10000; for (int i = 0; i < items; ++i) source.addItem(*makeRandomAS(), false); Log(lsTRACE) << "Adding items, then removing them"; - if(!confuseMap(source, 500)) BOOST_FAIL("ConfuseMap"); + if (!confuseMap(source, 500)) BOOST_FAIL("ConfuseMap"); source.setImmutable(); Log(lsTRACE) << "SOURCE COMPLETE, SYNCHING"; std::vector nodeIDs, gotNodeIDs; - std::list > gotNodes; + std::list< std::vector > gotNodes; std::vector hashes; std::vector::iterator nodeIDIterator; - std::list >::iterator rawNodeIterator; + std::list< std::vector >::iterator rawNodeIterator; int passes = 0; int nodes = 0; @@ -441,7 +440,7 @@ BOOST_AUTO_TEST_CASE( SHAMapSync_test ) // get the list of nodes we know we need destination.getMissingNodes(nodeIDs, hashes, 2048, NULL); - if(nodeIDs.empty()) break; + if (nodeIDs.empty()) break; Log(lsINFO) << nodeIDs.size() << " needed nodes";