From fa48a1fb0990e422669f86ac49378c2021e37ce7 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 26 Jun 2012 02:01:13 -0700 Subject: [PATCH] Fix the retrieve ledger hash problem for real. Make partial ledger operations work. Set base code for how thin servers will operate on partial ledgers. --- src/SHAMap.cpp | 102 ++++++++++++++++++++++++--------------------- src/SHAMap.h | 20 ++++++++- src/SHAMapSync.cpp | 74 ++++++++++++++++++++------------ src/SHAMapSync.h | 4 +- 4 files changed, 122 insertions(+), 78 deletions(-) diff --git a/src/SHAMap.cpp b/src/SHAMap.cpp index 0da1f50e85..cb48e8df5e 100644 --- a/src/SHAMap.cpp +++ b/src/SHAMap.cpp @@ -59,7 +59,7 @@ SHAMap::pointer SHAMap::snapShot(bool isMutable) return ret; } -std::stack SHAMap::getStack(const uint256& id, bool include_nonmatching_leaf) +std::stack SHAMap::getStack(const uint256& id, bool include_nonmatching_leaf, bool partialOk) { // Walk the tree as far as possible to the specified identifier // produce a stack of nodes along the way, with the terminal node at the top @@ -73,13 +73,18 @@ std::stack SHAMap::getStack(const uint256& id, bool inc assert(branch >= 0); uint256 hash = node->getChildHash(branch); - if (hash.isZero()) return stack; + if (hash.isZero()) + return stack; - node = getNode(node->getChildNodeID(branch), hash, false); - if (!node) + try { - if (isSynching()) return stack; - throw std::runtime_error("missing node"); + node = getNode(node->getChildNodeID(branch), hash, false); + } + catch (SHAMapMissingNode& mn) + { + if (partialOk) + return stack; + throw mn; } } @@ -135,15 +140,19 @@ SHAMapTreeNode::pointer SHAMap::walkTo(const uint256& id, bool modify) while (!inNode->isLeaf()) { int branch = inNode->selectBranch(id); - if (inNode->isEmptyBranch(branch)) return inNode; + if (inNode->isEmptyBranch(branch)) + return inNode; uint256 childHash = inNode->getChildHash(branch); SHAMapTreeNode::pointer nextNode = getNode(inNode->getChildNodeID(branch), childHash, false); - if (!nextNode) throw std::runtime_error("missing node"); + if (!nextNode) + throw SHAMapMissingNode(inNode->getChildNodeID(branch), childHash); inNode = nextNode; } - if (inNode->getTag() != id) return SHAMapTreeNode::pointer(); - if (modify) returnNode(inNode, true); + if (inNode->getTag() != id) + return SHAMapTreeNode::pointer(); + if (modify) + returnNode(inNode, true); return inNode; } @@ -156,7 +165,8 @@ SHAMapTreeNode* SHAMap::walkToPointer(const uint256& id) const uint256& nextHash = inNode->getChildHash(branch); if (!nextHash) return NULL; inNode = getNodePointer(inNode->getChildNodeID(branch), nextHash); - if (!inNode) throw std::runtime_error("missing node"); + if (!inNode) + throw SHAMapMissingNode(inNode->getChildNodeID(branch), nextHash); } return (inNode->getTag() == id) ? inNode : NULL; } @@ -166,27 +176,22 @@ SHAMapTreeNode::pointer SHAMap::getNode(const SHAMapNode& id, const uint256& has SHAMapTreeNode::pointer node = checkCacheNode(id); if (node) { +#ifdef DEBUG if (node->getNodeHash() != hash) { -#ifdef DEBUG std::cerr << "Attempt to get node, hash not in tree" << std::endl; std::cerr << "ID: " << id.getString() << std::endl; std::cerr << "TgtHash " << hash.GetHex() << std::endl; std::cerr << "NodHash " << node->getNodeHash().GetHex() << std::endl; dump(); -#endif throw std::runtime_error("invalid node"); } +#endif returnNode(node, modify); return node; } - node = fetchNode(id, hash); - if (!node) return node; - - if (node->getNodeHash() != hash) - throw std::runtime_error("invalid node hash"); - + node = fetchNodeExternal(id, hash); if (!mTNByID.insert(std::make_pair(id, node)).second) assert(false); return node; @@ -198,12 +203,7 @@ SHAMapTreeNode* SHAMap::getNodePointer(const SHAMapNode& id, const uint256& hash if (it != mTNByID.end()) return &*it->second; - SHAMapTreeNode::pointer node = fetchNode(id, hash); - if (!node) return NULL; - - if (node->getNodeHash() != hash) - throw std::runtime_error("invalid node fetched"); - + SHAMapTreeNode::pointer node = fetchNodeExternal(id, hash); if (!mTNByID.insert(std::make_pair(id, node)).second) assert(false); return &*node; @@ -311,26 +311,26 @@ SHAMapItem::pointer SHAMap::onlyBelow(SHAMapTreeNode* node) void SHAMap::eraseChildren(SHAMapTreeNode::pointer node) { // this node has only one item below it, erase its children - bool erase=false; - while(node->isInner()) + bool erase = false; + while (node->isInner()) { - for(int i=0; i<16; i++) - if(!node->isEmptyBranch(i)) + for (int i = 0; i < 16; ++i) + if (!node->isEmptyBranch(i)) { SHAMapTreeNode::pointer nextNode = getNode(node->getChildNodeID(i), node->getChildHash(i), false); - if(erase) + if (erase) { returnNode(node, true); - if(mTNByID.erase(*node)) - assert(false); + if (mTNByID.erase(*node)) + assert(false); } - erase=true; - node=nextNode; + erase = true; + node = nextNode; break; } } returnNode(node, true); - if(mTNByID.erase(*node) == 0) + if (mTNByID.erase(*node) == 0) assert(false); return; } @@ -351,7 +351,7 @@ SHAMapItem::pointer SHAMap::peekNextItem(const uint256& id) { // Get a pointer to the next item in the tree after a given item - item must be in tree boost::recursive_mutex::scoped_lock sl(mLock); - std::stack stack = getStack(id, true); + std::stack stack = getStack(id, true, false); while(!stack.empty()) { SHAMapTreeNode::pointer node = stack.top(); @@ -366,7 +366,6 @@ SHAMapItem::pointer SHAMap::peekNextItem(const uint256& id) if(!node->isEmptyBranch(i)) { node = getNode(node->getChildNodeID(i), node->getChildHash(i), false); - if (!node) throw std::runtime_error("missing node"); SHAMapItem::pointer item = firstBelow(&*node); if (!item) throw std::runtime_error("missing node"); return item; @@ -380,7 +379,7 @@ SHAMapItem::pointer SHAMap::peekPrevItem(const uint256& id) { // Get a pointer to the previous item in the tree after a given item - item must be in tree boost::recursive_mutex::scoped_lock sl(mLock); - std::stack stack = getStack(id, true); + std::stack stack = getStack(id, true, false); while (!stack.empty()) { SHAMapTreeNode::pointer node = stack.top(); @@ -395,7 +394,6 @@ SHAMapItem::pointer SHAMap::peekPrevItem(const uint256& id) if(!node->isEmptyBranch(i)) { node = getNode(node->getChildNodeID(i), node->getChildHash(i), false); - if(!node) throw std::runtime_error("missing node"); SHAMapItem::pointer item = firstBelow(&*node); if (!item) throw std::runtime_error("missing node"); return item; @@ -426,8 +424,9 @@ bool SHAMap::delItem(const uint256& id) boost::recursive_mutex::scoped_lock sl(mLock); assert(mState != Immutable); - std::stack stack=getStack(id, true); - if(stack.empty()) throw std::runtime_error("missing node"); + std::stack stack = getStack(id, true, false); + if(stack.empty()) + throw std::runtime_error("missing node"); SHAMapTreeNode::pointer leaf=stack.top(); stack.pop(); @@ -501,8 +500,9 @@ bool SHAMap::addGiveItem(SHAMapItem::pointer item, bool isTransaction) boost::recursive_mutex::scoped_lock sl(mLock); assert(mState != Immutable); - std::stack stack = getStack(tag, true); - if (stack.empty()) throw std::runtime_error("missing node"); + std::stack stack = getStack(tag, true, false); + if (stack.empty()) + throw std::runtime_error("missing node"); SHAMapTreeNode::pointer node = stack.top(); stack.pop(); @@ -592,7 +592,7 @@ bool SHAMap::updateGiveItem(SHAMapItem::pointer item, bool isTransaction) boost::recursive_mutex::scoped_lock sl(mLock); assert(mState != Immutable); - std::stack stack = getStack(tag, true); + std::stack stack = getStack(tag, true, false); if (stack.empty()) throw std::runtime_error("missing node"); SHAMapTreeNode::pointer node = stack.top(); @@ -620,15 +620,21 @@ void SHAMapItem::dump() std::cerr << "SHAMapItem(" << mTag.GetHex() << ") " << mData.size() << "bytes" << std::endl; } -SHAMapTreeNode::pointer SHAMap::fetchNode(const SHAMapNode& id, const uint256& hash) +SHAMapTreeNode::pointer SHAMap::fetchNodeExternal(const SHAMapNode& id, const uint256& hash) { - if (!theApp->running()) return SHAMapTreeNode::pointer(); + if (!theApp->running()) + throw SHAMapMissingNode(id, hash); HashedObject::pointer obj(theApp->getHashedObjectStore().retrieve(hash)); - if(!obj) return SHAMapTreeNode::pointer(); + if(!obj) + throw SHAMapMissingNode(id, hash); assert(Serializer::getSHA512Half(obj->getData()) == hash); - return boost::make_shared(id, obj->getData(), mSeq, STN_ARF_PREFIXED); + SHAMapTreeNode::pointer ret = boost::make_shared(id, obj->getData(), mSeq, STN_ARF_PREFIXED); +#ifdef DEBUG + assert((ret->getNodeHash() == hash) && (id == *ret)); +#endif + return ret; } void SHAMap::armDirty() diff --git a/src/SHAMap.h b/src/SHAMap.h index 6c6770a122..2e1af1dcaf 100644 --- a/src/SHAMap.h +++ b/src/SHAMap.h @@ -229,6 +229,22 @@ public: { return false; } }; +class SHAMapMissingNode : public std::runtime_error +{ +protected: + SHAMapNode mNodeID; + uint256 mNodeHash; + +public: + SHAMapMissingNode(const SHAMapNode& nodeID, const uint256& nodeHash) : + std::runtime_error(nodeID.getString()), mNodeID(nodeID), mNodeHash(nodeHash) + { ; } + virtual ~SHAMapMissingNode() throw() + { ; } + const SHAMapNode& getNodeID() const { return mNodeID; } + const uint256& getNodeHash() const { return mNodeHash; } +}; + class SHAMap { public: @@ -249,7 +265,7 @@ private: protected: void dirtyUp(std::stack& stack, const uint256& target, uint256 prevHash); - std::stack getStack(const uint256& id, bool include_nonmatching_leaf); + std::stack getStack(const uint256& id, bool include_nonmatching_leaf, bool partialOk); SHAMapTreeNode::pointer walkTo(const uint256& id, bool modify); SHAMapTreeNode* walkToPointer(const uint256& id); SHAMapTreeNode::pointer checkCacheNode(const SHAMapNode&); @@ -333,7 +349,7 @@ public: uint32 getSeq() { return mSeq; } // overloads for backed maps - boost::shared_ptr fetchNode(const SHAMapNode& id, const uint256& hash); + boost::shared_ptr fetchNodeExternal(const SHAMapNode& id, const uint256& hash); bool operator==(const SHAMap& s) { return getHash() == s.getHash(); } diff --git a/src/SHAMapSync.cpp b/src/SHAMapSync.cpp index 722c179b24..6ebc64344b 100644 --- a/src/SHAMapSync.cpp +++ b/src/SHAMapSync.cpp @@ -46,32 +46,39 @@ void SHAMap::getMissingNodes(std::vector& nodeIDs, std::vectorgetChildNodeID(branch); const uint256& childHash = node->getChildHash(branch); - SHAMapTreeNode::pointer d = getNode(childID, childHash, false); - if ((!d) && (filter != NULL)) + SHAMapTreeNode::pointer d; + try { - std::vector nodeData; - if (filter->haveNode(childID, childHash, nodeData)) + d = getNode(childID, childHash, false); + } + catch (SHAMapMissingNode& mn) + { // node is not in the map + if (filter != NULL) { - d = boost::make_shared(childID, nodeData, mSeq, STN_ARF_WIRE); - if (childHash != d->getNodeHash()) + std::vector nodeData; + if (filter->haveNode(childID, childHash, nodeData)) { - Log(lsERROR) << "Wrong hash from cached object"; - d = SHAMapTreeNode::pointer(); - } - else - { - Log(lsTRACE) << "Got sync node from cache: " << d->getString(); - mTNByID[*d] = d; + d = boost::make_shared(childID, nodeData, mSeq, STN_ARF_WIRE); + if (childHash != d->getNodeHash()) + { + Log(lsERROR) << "Wrong hash from cached object"; + d = SHAMapTreeNode::pointer(); + } + else + { + Log(lsTRACE) << "Got sync node from cache: " << d->getString(); + mTNByID[*d] = d; + } } } } if (!d) - { + { // we need this node nodeIDs.push_back(node->getChildNodeID(branch)); if (--max <= 0) return; } - else if (d->isInner() && !d->isFullBelow()) + else if (d->isInner() && !d->isFullBelow()) // we might need children of this node stack.push(d); } } @@ -159,8 +166,10 @@ bool SHAMap::addRootNode(const uint256& hash, const std::vector& } SHAMapTreeNode::pointer node = boost::make_shared(SHAMapNode(), rootNode, 0, STN_ARF_WIRE); - if (!node) return false; - if (node->getNodeHash() != hash) return false; + if (!node) + return false; + if (node->getNodeHash() != hash) + return false; returnNode(root, true); root = node; @@ -178,14 +187,17 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vector stack = getStack(node.getNodeID(), true); - if (stack.empty()) return false; + std::stack stack = getStack(node.getNodeID(), true, true); + if (stack.empty()) + return false; SHAMapTreeNode::pointer iNode = stack.top(); if (!iNode) @@ -221,7 +233,8 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vectorgetNodeHash()) // these aren't the droids we're looking for return false; - if (filter) filter->gotNode(node, hash, rawNode, newNode->isLeaf()); + if (filter) + filter->gotNode(node, hash, rawNode, newNode->isLeaf()); mTNByID[*newNode] = newNode; if (!newNode->isLeaf()) @@ -236,9 +249,16 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vectorisEmptyBranch(i)) { - SHAMapTreeNode::pointer nextNode = getNode(iNode->getChildNodeID(i), iNode->getChildHash(i), false); - if (!nextNode) return true; - if (nextNode->isInner() && !nextNode->isFullBelow()) return true; + try + { + SHAMapTreeNode::pointer nextNode = getNode(iNode->getChildNodeID(i), iNode->getChildHash(i), false); + if (nextNode->isInner() && !nextNode->isFullBelow()) + return true; + } + catch (SHAMapMissingNode) + { + return true; + } } iNode->setFullBelow(); } while (!stack.empty()); @@ -359,7 +379,7 @@ static bool confuseMap(SHAMap &map, int count) std::list > SHAMap::getTrustedPath(const uint256& index) { boost::recursive_mutex::scoped_lock sl(mLock); - std::stack stack = SHAMap::getStack(index, false); + std::stack stack = SHAMap::getStack(index, false, false); if (stack.empty() || !stack.top()->isLeaf()) throw std::runtime_error("requested leaf not present"); @@ -449,11 +469,13 @@ BOOST_AUTO_TEST_CASE( SHAMapSync_test ) // get as many nodes as possible based on this information for (nodeIDIterator = nodeIDs.begin(); nodeIDIterator != nodeIDs.end(); ++nodeIDIterator) + { if (!source.getNodeFat(*nodeIDIterator, gotNodeIDs, gotNodes, (rand() % 2) == 0)) { Log(lsFATAL) << "GetNodeFat fails"; BOOST_FAIL("GetNodeFat"); } + } assert(gotNodeIDs.size() == gotNodes.size()); nodeIDs.clear(); hashes.clear(); diff --git a/src/SHAMapSync.h b/src/SHAMapSync.h index 5eaf25acc5..2cd6ffc1e4 100644 --- a/src/SHAMapSync.h +++ b/src/SHAMapSync.h @@ -40,7 +40,7 @@ public: theApp->getHashedObjectStore().store(ACCOUNT_NODE, mLedgerSeq, nodeData, nodeHash); } virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) - { // fetchNode already tried + { // fetchNodeExternal already tried return false; } }; @@ -61,7 +61,7 @@ public: theApp->getHashedObjectStore().store(isLeaf ? TRANSACTION : TRANSACTION_NODE, mLedgerSeq, nodeData, nodeHash); } virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) - { // fetchNode already tried + { // fetchNodeExternal already tried return false; } };