From c13c561295a2efdc54687f91ed9ce76f3f64ac92 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 05:33:53 -0700 Subject: [PATCH] Use the new accelerators to speed up ledger and transaction set synchronization --- src/LedgerAcquire.cpp | 16 ++++++--- src/LedgerAcquire.h | 26 -------------- src/LedgerConsensus.cpp | 13 +++---- src/LedgerConsensus.h | 1 - src/SHAMapSync.h | 75 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 93 insertions(+), 38 deletions(-) create mode 100644 src/SHAMapSync.h diff --git a/src/LedgerAcquire.cpp b/src/LedgerAcquire.cpp index 825045bf7e..9801719275 100644 --- a/src/LedgerAcquire.cpp +++ b/src/LedgerAcquire.cpp @@ -7,6 +7,7 @@ #include "Application.h" #include "Log.h" +#include "SHAMapSync.h" #define LA_DEBUG #define LEDGER_ACQUIRE_TIMEOUT 2 @@ -82,7 +83,7 @@ void PeerSet::TimerEntry(boost::weak_ptr wptr, const boost::system::err } LedgerAcquire::LedgerAcquire(const uint256& hash) : PeerSet(hash, LEDGER_ACQUIRE_TIMEOUT), - mFilter(&theApp->getNodeCache()), mHaveBase(false), mHaveState(false), mHaveTransactions(false) + mHaveBase(false), mHaveState(false), mHaveTransactions(false) { #ifdef LA_DEBUG Log(lsTRACE) << "Acquiring ledger " << mHash.GetHex(); @@ -168,7 +169,8 @@ void LedgerAcquire::trigger(Peer::pointer peer) { std::vector nodeIDs; std::vector nodeHashes; - mLedger->peekTransactionMap()->getMissingNodes(nodeIDs, nodeHashes, 128, &mFilter); + TransactionStateSF tFilter(mLedger->getHash(), mLedger->getSeq()); + mLedger->peekTransactionMap()->getMissingNodes(nodeIDs, nodeHashes, 128, &tFilter); if (nodeIDs.empty()) { if (!mLedger->peekTransactionMap()->isValid()) mFailed = true; @@ -220,7 +222,8 @@ void LedgerAcquire::trigger(Peer::pointer peer) { std::vector nodeIDs; std::vector nodeHashes; - mLedger->peekAccountStateMap()->getMissingNodes(nodeIDs, nodeHashes, 128, &mFilter); + AccountStateSF aFilter(mLedger->getHash(), mLedger->getSeq()); + mLedger->peekAccountStateMap()->getMissingNodes(nodeIDs, nodeHashes, 128, &aFilter); if (nodeIDs.empty()) { if (!mLedger->peekAccountStateMap()->isValid()) mFailed = true; @@ -297,6 +300,7 @@ bool LedgerAcquire::takeBase(const std::string& data, Peer::pointer peer) return false; } mHaveBase = true; + theApp->getHashedObjectStore().store(LEDGER, mLedger->getLedgerSeq(), data, mHash); progress(); if (!mLedger->getTransHash()) mHaveTransactions = true; if (!mLedger->getAccountHash()) mHaveState = true; @@ -311,6 +315,7 @@ bool LedgerAcquire::takeTxNode(const std::list& nodeIDs, if (!mHaveBase) return false; std::list::const_iterator nodeIDit = nodeIDs.begin(); std::list< std::vector >::const_iterator nodeDatait = data.begin(); + TransactionStateSF tFilter(mLedger->getHash(), mLedger->getSeq()); while (nodeIDit != nodeIDs.end()) { if (nodeIDit->isRoot()) @@ -318,7 +323,7 @@ bool LedgerAcquire::takeTxNode(const std::list& nodeIDs, if (!mLedger->peekTransactionMap()->addRootNode(mLedger->getTransHash(), *nodeDatait)) return false; } - else if (!mLedger->peekTransactionMap()->addKnownNode(*nodeIDit, *nodeDatait, &mFilter)) + else if (!mLedger->peekTransactionMap()->addKnownNode(*nodeIDit, *nodeDatait, &tFilter)) return false; ++nodeIDit; ++nodeDatait; @@ -342,6 +347,7 @@ bool LedgerAcquire::takeAsNode(const std::list& nodeIDs, if (!mHaveBase) return false; std::list::const_iterator nodeIDit = nodeIDs.begin(); std::list< std::vector >::const_iterator nodeDatait = data.begin(); + AccountStateSF tFilter(mLedger->getHash(), mLedger->getSeq()); while (nodeIDit != nodeIDs.end()) { if (nodeIDit->isRoot()) @@ -349,7 +355,7 @@ bool LedgerAcquire::takeAsNode(const std::list& nodeIDs, if (!mLedger->peekAccountStateMap()->addRootNode(mLedger->getAccountHash(), *nodeDatait)) return false; } - else if (!mLedger->peekAccountStateMap()->addKnownNode(*nodeIDit, *nodeDatait, &mFilter)) + else if (!mLedger->peekAccountStateMap()->addKnownNode(*nodeIDit, *nodeDatait, &tFilter)) return false; ++nodeIDit; ++nodeDatait; diff --git a/src/LedgerAcquire.h b/src/LedgerAcquire.h index 87e2c86b21..cc499fdb3a 100644 --- a/src/LedgerAcquire.h +++ b/src/LedgerAcquire.h @@ -58,31 +58,6 @@ private: static void TimerEntry(boost::weak_ptr, const boost::system::error_code& result); }; -typedef TaggedCache< uint256, std::vector > NodeCache; -typedef std::vector VUC; - -class THSyncFilter : public SHAMapSyncFilter -{ -protected: - NodeCache* mCache; // holds nodes we see during the consensus process - -public: - THSyncFilter(NodeCache* cache) : mCache(cache) { ; } - virtual void gotNode(const SHAMapNode& id, const uint256& nodeHash, - const std::vector& nodeData, bool) - { - boost::shared_ptr ptr = boost::make_shared(nodeData); - mCache->canonicalize(nodeHash, ptr); - } - virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) - { - boost::shared_ptr entry = mCache->fetch(nodeHash); - if (!entry) return false; - nodeData = *entry; - return true; - } -}; - class LedgerAcquire : public PeerSet, public boost::enable_shared_from_this { // A ledger we are trying to acquire public: @@ -90,7 +65,6 @@ public: protected: Ledger::pointer mLedger; - THSyncFilter mFilter; bool mHaveBase, mHaveState, mHaveTransactions; std::vector< boost::function > mOnComplete; diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index 5baeff0d6a..ce5a35f2b8 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -11,13 +11,13 @@ #include "LedgerTiming.h" #include "SerializedValidation.h" #include "Log.h" +#include "SHAMapSync.h" #define TRUST_NETWORK // #define LC_DEBUG -TransactionAcquire::TransactionAcquire(const uint256& hash) - : PeerSet(hash, 1), mFilter(&theApp->getNodeCache()), mHaveRoot(false) +TransactionAcquire::TransactionAcquire(const uint256& hash) : PeerSet(hash, 1), mHaveRoot(false) { mMap = boost::make_shared(); mMap->setSynching(); @@ -50,9 +50,9 @@ void TransactionAcquire::trigger(Peer::pointer peer) } if (mHaveRoot) { - std::vector nodeIDs; - std::vector nodeHashes; - mMap->getMissingNodes(nodeIDs, nodeHashes, 256, &mFilter); + std::vector nodeIDs; std::vector nodeHashes; + ConsensusTransSetSF sf; + mMap->getMissingNodes(nodeIDs, nodeHashes, 256, &sf); if (nodeIDs.empty()) { if (mMap->isValid()) @@ -91,6 +91,7 @@ bool TransactionAcquire::takeNodes(const std::list& nodeIDs, { std::list::const_iterator nodeIDit = nodeIDs.begin(); std::list< std::vector >::const_iterator nodeDatait = data.begin(); + ConsensusTransSetSF sf; while (nodeIDit != nodeIDs.end()) { if (nodeIDit->isRoot()) @@ -104,7 +105,7 @@ bool TransactionAcquire::takeNodes(const std::list& nodeIDs, return false; else mHaveRoot = true; } - else if (!mMap->addKnownNode(*nodeIDit, *nodeDatait, &mFilter)) + else if (!mMap->addKnownNode(*nodeIDit, *nodeDatait, &sf)) return false; ++nodeIDit; ++nodeDatait; diff --git a/src/LedgerConsensus.h b/src/LedgerConsensus.h index c9a5010cee..8db236487e 100644 --- a/src/LedgerConsensus.h +++ b/src/LedgerConsensus.h @@ -22,7 +22,6 @@ public: protected: SHAMap::pointer mMap; - THSyncFilter mFilter; // FIXME: Should use transaction master too bool mHaveRoot; void onTimer() { trigger(Peer::pointer()); } diff --git a/src/SHAMapSync.h b/src/SHAMapSync.h new file mode 100644 index 0000000000..d88b98fd51 --- /dev/null +++ b/src/SHAMapSync.h @@ -0,0 +1,75 @@ +#ifndef __SHAMAPYSNC__ +#define __SHAMAPSYNC__ + +#include "SHAMap.h" +#include "Application.h" + +// Sync filters allow low-level SHAMapSync code to interact correctly with +// higher-level structures such as caches and transaction stores + +class ConsensusTransSetSF : public SHAMapSyncFilter +{ // sync filter for transaction sets during consensus building +public: + ConsensusTransSetSF() { ; } + virtual void gotNode(const SHAMapNode& id, const uint256& nodeHash, + const std::vector& nodeData, bool isLeaf) + { + // WRITEME: If 'isLeaf' is true, this is a transaction + theApp->getNodeCache().store(nodeHash, nodeData); + } + virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) + { + // WRITEME: We could check our own map, we could check transaction tables + return theApp->getNodeCache().retrieve(nodeHash, nodeData); + } +}; + +class LedgerAccountStateSF : public SHAMapSyncFilter +{ // sync filter for account state nodes during ledger sync +protected: + uint256 mLedgerHash; + uint32 mLedgerSeq; + +public: + LedgerAccountStateSF(const uint256& ledgerHash, uint32 ledgerSeq) : mLedgerHash(ledgerHash), mLedgerSeq(ledgerSeq) + { ; } + + virtual void gotNode(const SHAMapNode& id, const uint256& nodeHash, + const std::vector& nodeData, bool isLeaf) + { + theApp->getHashedObjectStore().store(ACCOUNT_NODE, mLedgerSeq, nodeData, nodeHash); + } + virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) + { + HashedObject::pointer node = theApp->getHashedObjectStore().retrieve(nodeHash); + if (!node) return false; + nodeData = node->getData(); + return true; + } +}; + +class TransactionStateSF : public SHAMapSyncFilter +{ // sync filter for transactions tree during ledger sync +protected: + uint256 mLedgerHash; + uint32 mLedgerSeq; + +public: + TransactionStateSF(const uint256& ledgerHash, uint32 ledgerSeq) : mLedgerHash(ledgerHash), mLedgerSeq(ledgerSeq) + { ; } + + virtual void gotNode(const SHAMapNode& id, const uint256& nodeHash, + const std::vector& nodeData, bool isLeaf) + { + theApp->getHashedObjectStore().store(isLeaf ? TRANSACTION : TRANSACTION_NODE, mLedgerSeq, nodeData, nodeHash); + } + virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) + { + HashedObject::pointer node = theApp->getHashedObjectStore().retrieve(nodeHash); + if (!node) return false; + nodeData = node->getData(); + return true; + } +}; + +#endif