From 3e9a3caa355195a1e84f8e4b28c14dafdf4172f3 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 30 Apr 2012 05:53:54 -0700 Subject: [PATCH] Sync stuff. --- src/SHAMapDiff.cpp | 102 ++++++++++++----------- src/SHAMapSync.cpp | 200 ++++++++++++++++++++++----------------------- 2 files changed, 147 insertions(+), 155 deletions(-) diff --git a/src/SHAMapDiff.cpp b/src/SHAMapDiff.cpp index 3f501d3c9f..f65ee1967c 100644 --- a/src/SHAMapDiff.cpp +++ b/src/SHAMapDiff.cpp @@ -27,71 +27,69 @@ bool SHAMap::walkBranch(SHAMapTreeNode::pointer node, SHAMapItem::pointer otherM std::stack nodeStack; nodeStack.push(node); - while(!nodeStack.empty()) + while (!nodeStack.empty()) { - SHAMapTreeNode::pointer node=nodeStack.top(); + SHAMapTreeNode::pointer node = nodeStack.top(); nodeStack.pop(); if(node->isInner()) { // This is an inner node, add all non-empty branches - for(int i=0; i<16; i++) - if(!node->isEmptyBranch(i)) + for(int i = 0; i < 16; ++i) + if (!node->isEmptyBranch(i)) { - SHAMapTreeNode::pointer newNode=getNode(node->getChildNodeID(i), node->getChildHash(i), false); - if(!newNode) throw SHAMapException(MissingNode); + SHAMapTreeNode::pointer newNode = getNode(node->getChildNodeID(i), node->getChildHash(i), false); + if (!newNode) throw SHAMapException(MissingNode); nodeStack.push(newNode); } } else { // This is a leaf node, process its item - SHAMapItem::pointer item=node->getItem(); + SHAMapItem::pointer item = node->getItem(); - if(otherMapItem && otherMapItem->getTag()getTag()) + if (otherMapItem && otherMapItem->getTag()getTag()) { // this item comes after the item from the other map, so add the other item - if(isFirstMap) // this is first map, so other item is from second + if (isFirstMap) // this is first map, so other item is from second differences.insert(std::make_pair(otherMapItem->getTag(), std::make_pair(SHAMapItem::pointer(), otherMapItem))); else differences.insert(std::make_pair(otherMapItem->getTag(), std::make_pair(otherMapItem, SHAMapItem::pointer()))); - if((--maxCount)<=0) return false; - otherMapItem=SHAMapItem::pointer(); + if (--maxCount <= 0) return false; + otherMapItem = SHAMapItem::pointer(); } - if( (!otherMapItem) || (item->getTag()getTag()) ) + if ((!otherMapItem) || (item->getTag() < otherMapItem->getTag())) { // unmatched - if(isFirstMap) + if (isFirstMap) differences.insert(std::make_pair(item->getTag(), std::make_pair(item, SHAMapItem::pointer()))); else differences.insert(std::make_pair(item->getTag(), std::make_pair(SHAMapItem::pointer(), item))); - if((--maxCount)<=0) return false; + if (--maxCount <= 0) return false; } - else if(item->getTag()==otherMapItem->getTag()) + else if (item->getTag() == otherMapItem->getTag()) { - if(item->peekData()!=otherMapItem->peekData()) + if (item->peekData() != otherMapItem->peekData()) { // non-matching items - if(isFirstMap) - differences.insert(std::make_pair(otherMapItem->getTag(), - std::make_pair(item, otherMapItem))); - else - differences.insert(std::make_pair(otherMapItem->getTag(), - std::make_pair(otherMapItem, item))); - if((--maxCount)<=0) return false; - item=SHAMapItem::pointer(); + if (isFirstMap) differences.insert(std::make_pair(otherMapItem->getTag(), + std::make_pair(item, otherMapItem))); + else differences.insert(std::make_pair(otherMapItem->getTag(), + std::make_pair(otherMapItem, item))); + if(--maxCount <= 0) return false; + item = SHAMapItem::pointer(); } } else assert(false); } } - if(otherMapItem) + if (otherMapItem) { // otherMapItem was unmatched, must add - if(isFirstMap) // this is first map, so other item is from second + if (isFirstMap) // this is first map, so other item is from second differences.insert(std::make_pair(otherMapItem->getTag(), std::make_pair(SHAMapItem::pointer(), otherMapItem))); else differences.insert(std::make_pair(otherMapItem->getTag(), std::make_pair(otherMapItem, SHAMapItem::pointer()))); - if((--maxCount)<=0) return false; + if (--maxCount <= 0) return false; } return true; @@ -105,68 +103,68 @@ bool SHAMap::compare(SHAMap::pointer otherMap, SHAMapDiff& differences, int maxC std::stack nodeStack; // track nodes we've pushed ScopedLock sl(Lock()); - if(getHash() == otherMap->getHash()) return true; + if (getHash() == otherMap->getHash()) return true; nodeStack.push(SHAMapDiffNode(SHAMapNode(), getHash(), otherMap->getHash())); - while(!nodeStack.empty()) + while (!nodeStack.empty()) { SHAMapDiffNode dNode(nodeStack.top()); nodeStack.pop(); - SHAMapTreeNode::pointer ourNode=getNode(dNode.mNodeID, dNode.mOurHash, false); - SHAMapTreeNode::pointer otherNode=otherMap->getNode(dNode.mNodeID, dNode.mOtherHash, false); - if(!ourNode || !otherNode) throw SHAMapException(MissingNode); + SHAMapTreeNode::pointer ourNode = getNode(dNode.mNodeID, dNode.mOurHash, false); + SHAMapTreeNode::pointer otherNode = otherMap->getNode(dNode.mNodeID, dNode.mOtherHash, false); + if (!ourNode || !otherNode) throw SHAMapException(MissingNode); - if(ourNode->isLeaf() && otherNode->isLeaf()) + if (ourNode->isLeaf() && otherNode->isLeaf()) { // two leaves - if(ourNode->getTag() == otherNode->getTag()) + if (ourNode->getTag() == otherNode->getTag()) { - if(ourNode->peekData()!=otherNode->peekData()) + if (ourNode->peekData() != otherNode->peekData()) { differences.insert(std::make_pair(ourNode->getTag(), std::make_pair(ourNode->getItem(), otherNode->getItem()))); - if((--maxCount)<=0) return false; + if (--maxCount <= 0) return false; } } else { differences.insert(std::make_pair(ourNode->getTag(), std::make_pair(ourNode->getItem(), SHAMapItem::pointer()))); - if((--maxCount)<=0) return false; + if (--maxCount <= 0) return false; differences.insert(std::make_pair(otherNode->getTag(), std::make_pair(SHAMapItem::pointer(), otherNode->getItem()))); - if((--maxCount)<=0) return false; + if (--maxCount <= 0) return false; } } - else if(ourNode->isInner() && otherNode->isLeaf()) + else if (ourNode->isInner() && otherNode->isLeaf()) { - if(!walkBranch(ourNode, otherNode->getItem(), true, differences, maxCount)) + if (!walkBranch(ourNode, otherNode->getItem(), true, differences, maxCount)) return false; } - else if(ourNode->isLeaf() && otherNode->isInner()) + else if (ourNode->isLeaf() && otherNode->isInner()) { - if(!otherMap->walkBranch(otherNode, ourNode->getItem(), false, differences, maxCount)) + if (!otherMap->walkBranch(otherNode, ourNode->getItem(), false, differences, maxCount)) return false; } - else if(ourNode->isInner() && otherNode->isInner()) + else if (ourNode->isInner() && otherNode->isInner()) { - for(int i=0; i<16; i++) - if(ourNode->getChildHash(i) != otherNode->getChildHash(i) ) + for (int i = 0; i < 16; ++i) + if (ourNode->getChildHash(i) != otherNode->getChildHash(i) ) { - if(!otherNode->getChildHash(i)) + if (!otherNode->getChildHash(i)) { // We have a branch, the other tree does not SHAMapTreeNode::pointer iNode=getNode(ourNode->getChildNodeID(i), ourNode->getChildHash(i), false); - if(!iNode) throw SHAMapException(MissingNode); - if(!walkBranch(iNode, SHAMapItem::pointer(), true, differences, maxCount)) + if (!iNode) throw SHAMapException(MissingNode); + if (!walkBranch(iNode, SHAMapItem::pointer(), true, differences, maxCount)) return false; } - else if(!ourNode->getChildHash(i)) + else if (!ourNode->getChildHash(i)) { // The other tree has a branch, we do not - SHAMapTreeNode::pointer iNode=otherMap->getNode(otherNode->getChildNodeID(i), + SHAMapTreeNode::pointer iNode = otherMap->getNode(otherNode->getChildNodeID(i), otherNode->getChildHash(i), false); - if(!iNode) throw SHAMapException(MissingNode); - if(!otherMap->walkBranch(iNode, SHAMapItem::pointer(), false, differences, maxCount)) + if (!iNode) throw SHAMapException(MissingNode); + if (!otherMap->walkBranch(iNode, SHAMapItem::pointer(), false, differences, maxCount)) return false; } else // The two trees have different non-empty branches diff --git a/src/SHAMapSync.cpp b/src/SHAMapSync.cpp index 6fd19361b4..3cd0a8013d 100644 --- a/src/SHAMapSync.cpp +++ b/src/SHAMapSync.cpp @@ -31,28 +31,28 @@ void SHAMap::getMissingNodes(std::vector& nodeIDs, std::vector stack; stack.push(root); - while( (max>0) && (!stack.empty()) ) + while ((max > 0) && (!stack.empty())) { - SHAMapTreeNode::pointer node=stack.top(); + SHAMapTreeNode::pointer node = stack.top(); stack.pop(); #ifdef GMN_DEBUG std::cerr << "gMN: popped " << node->getString() << std::endl; #endif - for(int i=0; i<16; i++) - if(!node->isEmptyBranch(i)) + for (int i = 0; i < 16; ++i) + if( !node->isEmptyBranch(i)) { #ifdef GMN_DEBUG std::cerr << "gMN: " << node->getString() << " has non-empty branch " << i << std::endl; #endif - SHAMapTreeNode::pointer desc=getNode(node->getChildNodeID(i), node->getChildHash(i), false); + SHAMapTreeNode::pointer desc = getNode(node->getChildNodeID(i), node->getChildHash(i), false); if(desc) { - if(desc->isInner() && !desc->isFullBelow()) + if (desc->isInner() && !desc->isFullBelow()) stack.push(desc); } - else if(max-- > 0) + else if (max-- > 0) { #ifdef GMN_DEBUG std::cerr << "gMN: need " << node->getChildNodeID(i).getString() << std::endl; @@ -65,11 +65,11 @@ void SHAMap::getMissingNodes(std::vector& nodeIDs, std::vector& nodeIDs, std::list >& rawNodes) -{ +{ // Gets a node and some of its children boost::recursive_mutex::scoped_lock sl(mLock); - SHAMapTreeNode::pointer node=getNode(wanted); - if(!node) + SHAMapTreeNode::pointer node = getNode(wanted); + if (!node) { assert(false); // Remove for release, this can happen if we get a bogus request return false; @@ -80,13 +80,13 @@ bool SHAMap::getNodeFat(const SHAMapNode& wanted, std::vector& nodeI node->addRaw(s); rawNodes.push_back(s.peekData()); - if(node->isRoot() || node->isLeaf()) // don't get a fat root, can't get a fat leaf + if (node->isRoot() || node->isLeaf()) // don't get a fat root, can't get a fat leaf return true; - for(int i=0; i<16; i++) - if(!node->isEmptyBranch(i)) + for (int i = 0; i < 16; ++i) + if (!node->isEmptyBranch(i)) { - SHAMapTreeNode::pointer nextNode=getNode(node->getChildNodeID(i), node->getChildHash(i), false); + SHAMapTreeNode::pointer nextNode = getNode(node->getChildNodeID(i), node->getChildHash(i), false); assert(nextNode); if(nextNode) { @@ -105,7 +105,7 @@ bool SHAMap::addRootNode(const std::vector& rootNode) boost::recursive_mutex::scoped_lock sl(mLock); // we already have a root node - if(root->getNodeHash().isNonZero()) + if (root->getNodeHash().isNonZero()) { #ifdef DEBUG std::cerr << "got root node, already have one" << std::endl; @@ -113,8 +113,8 @@ bool SHAMap::addRootNode(const std::vector& rootNode) return true; } - SHAMapTreeNode::pointer node=boost::make_shared(SHAMapNode(), rootNode, 0); - if(!node) return false; + SHAMapTreeNode::pointer node = boost::make_shared(SHAMapNode(), rootNode, 0); + if (!node) return false; #ifdef DEBUG node->dump(); @@ -122,9 +122,9 @@ bool SHAMap::addRootNode(const std::vector& rootNode) returnNode(root, true); - root=node; - mTNByID[*root]=root; - if(!root->getNodeHash()) + root = node; + mTNByID[*root] = root; + if (!root->getNodeHash()) { root->setFullBelow(); clearSynching(); @@ -138,23 +138,23 @@ bool SHAMap::addRootNode(const uint256& hash, const std::vector& boost::recursive_mutex::scoped_lock sl(mLock); // we already have a root node - if(root->getNodeHash().isNonZero()) + if (root->getNodeHash().isNonZero()) { #ifdef DEBUG std::cerr << "got root node, already have one" << std::endl; #endif - assert(root->getNodeHash()==hash); + assert(root->getNodeHash() == hash); return true; } - SHAMapTreeNode::pointer node=boost::make_shared(SHAMapNode(), rootNode, 0); - if(!node) return false; - if(node->getNodeHash()!=hash) return false; + SHAMapTreeNode::pointer node = boost::make_shared(SHAMapNode(), rootNode, 0); + if (!node) return false; + if (node->getNodeHash() != hash) return false; returnNode(root, true); - root=node; - mTNByID[*root]=root; - if(!root->getNodeHash()) + root = node; + mTNByID[*root] = root; + if (!root->getNodeHash()) { root->setFullBelow(); clearSynching(); @@ -166,23 +166,23 @@ bool SHAMap::addRootNode(const uint256& hash, const std::vector& bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vector& rawNode) { // return value: true=okay, false=error assert(!node.isRoot()); - if(!isSynching()) return false; + if (!isSynching()) return false; boost::recursive_mutex::scoped_lock sl(mLock); - if(checkCacheNode(node)) return true; + if (checkCacheNode(node)) return true; - std::stack stack=getStack(node.getNodeID(), true); - if(stack.empty()) return false; + std::stack stack = getStack(node.getNodeID(), true); + if (stack.empty()) return false; SHAMapTreeNode::pointer iNode=stack.top(); - if(!iNode) + if (!iNode) { // we should always have a root assert(false); return true; } - if(iNode->isLeaf() || (iNode->getDepth()==node.getDepth())) + if (iNode->isLeaf() || (iNode->getDepth() == node.getDepth())) { #ifdef DEBUG std::cerr << "got inner node, already had it (late)" << std::endl; @@ -190,8 +190,8 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vectorgetDepth()!=(node.getDepth()-1)) - { // Either this node is broken or we didn't request it + if (iNode->getDepth() != (node.getDepth() - 1)) + { // Either this node is broken or we didn't request it (yet) #ifdef DEBUG std::cerr << "unable to hook node " << node.getString() << std::endl; std::cerr << " stuck at " << iNode->getString() << std::endl; @@ -200,39 +200,39 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vectorselectBranch(node.getNodeID()); - if(branch<0) + int branch = iNode->selectBranch(node.getNodeID()); + if (branch < 0) { assert(false); return false; } - uint256 hash=iNode->getChildHash(branch); - if(!hash) return false; + uint256 hash = iNode->getChildHash(branch); + if (!hash) return false; - SHAMapTreeNode::pointer newNode=boost::make_shared(node, rawNode, mSeq); - if(hash!=newNode->getNodeHash()) // these aren't the droids we're looking for + SHAMapTreeNode::pointer newNode = boost::make_shared(node, rawNode, mSeq); + if (hash != newNode->getNodeHash()) // these aren't the droids we're looking for return false; - mTNByID[*newNode]=newNode; - if(!newNode->isLeaf()) + mTNByID[*newNode] = newNode; + if (!newNode->isLeaf()) return true; // only a leaf can fill a branch // did this new leaf cause its parents to fill up do { - iNode=stack.top(); + iNode = stack.top(); stack.pop(); assert(iNode->isInner()); - for(int i=0; i<16; i++) - if(!iNode->isEmptyBranch(i)) + for(int i = 0; i < 16; ++i) + if (!iNode->isEmptyBranch(i)) { - SHAMapTreeNode::pointer nextNode=getNode(iNode->getChildNodeID(i), iNode->getChildHash(i), false); - if(!nextNode) return true; - if(nextNode->isInner() && !nextNode->isFullBelow()) return true; + SHAMapTreeNode::pointer nextNode = getNode(iNode->getChildNodeID(i), iNode->getChildHash(i), false); + if (!nextNode) return true; + if (nextNode->isInner() && !nextNode->isFullBelow()) return true; } iNode->setFullBelow(); - } while(!stack.empty()); - if(root->isFullBelow()) clearSynching(); + } while (!stack.empty()); + if (root->isFullBelow()) clearSynching(); return true; } @@ -242,21 +242,21 @@ bool SHAMap::deepCompare(SHAMap& other) boost::recursive_mutex::scoped_lock sl(mLock); stack.push(root); - while(!stack.empty()) + while (!stack.empty()) { - SHAMapTreeNode::pointer node=stack.top(); + SHAMapTreeNode::pointer node = stack.top(); stack.pop(); SHAMapTreeNode::pointer otherNode; - if(node->isRoot()) otherNode=other.root; - else otherNode=other.getNode(*node, node->getNodeHash(), false); + if(node->isRoot()) otherNode = other.root; + else otherNode = other.getNode(*node, node->getNodeHash(), false); - if(!otherNode) + if (!otherNode) { std::cerr << "unable to fetch node" << std::endl; return false; } - else if(otherNode->getNodeHash()!=node->getNodeHash()) + else if (otherNode->getNodeHash() != node->getNodeHash()) { std::cerr << "node hash mismatch" << std::endl; return false; @@ -266,32 +266,28 @@ bool SHAMap::deepCompare(SHAMap& other) std::cerr << "Comparing inner nodes " << node->getString() << std::endl; #endif - if(node->getNodeHash() != otherNode->getNodeHash()) + if (node->getNodeHash() != otherNode->getNodeHash()) return false; - if(node->isLeaf()) + if (node->isLeaf()) { - if(!otherNode->isLeaf()) - return false; - if(node->peekItem()->getTag()!=otherNode->peekItem()->getTag()) - return false; - if(node->peekItem()->getData()!=otherNode->peekItem()->getData()) - return false; + if (!otherNode->isLeaf()) return false; + if (node->peekItem()->getTag() != otherNode->peekItem()->getTag()) return false; + if (node->peekItem()->getData() != otherNode->peekItem()->getData()) return false; } - else if(node->isInner()) + else if (node->isInner()) { - if(!otherNode->isInner()) + if (!otherNode->isInner()) return false; for(int i=0; i<16; i++) { if(node->isEmptyBranch(i)) { - if(!otherNode->isEmptyBranch(i)) - return false; + if(!otherNode->isEmptyBranch(i)) return false; } else { - SHAMapTreeNode::pointer next=getNode(node->getChildNodeID(i), node->getChildHash(i), false); - if(!next) + SHAMapTreeNode::pointer next = getNode(node->getChildNodeID(i), node->getChildHash(i), false); + if (!next) { std::cerr << "unable to fetch inner node" << std::endl; return false; @@ -311,8 +307,7 @@ bool SHAMap::deepCompare(SHAMap& other) static SHAMapItem::pointer makeRandomAS() { Serializer s; - for(int d=0; d<3; d++) - s.add32(rand()); + for(int d = 0; d < 3; ++d) s.add32(rand()); return boost::make_shared(s.getRIPEMD160().to256(), s.peekData()); } @@ -320,31 +315,31 @@ static bool confuseMap(SHAMap &map, int count) { // add a bunch of random states to a map, then remove them // map should be the same - uint256 beforeHash=map.getHash(); + uint256 beforeHash = map.getHash(); std::list items; - for(int i=0; igetTag()); - if(!map.addItem(*item, false)) + if (!map.addItem(*item, false)) { std::cerr << "Unable to add item to map" << std::endl; return false; } } - for(std::list::iterator it=items.begin(); it!=items.end(); ++it) + for (std::list::iterator it = items.begin(); it != items.end(); ++it) { - if(!map.delItem(*it)) + if (!map.delItem(*it)) { std::cerr << "Unable to remove item from map" << std::endl; return false; } } - if(beforeHash!=map.getHash()) + if (beforeHash != map.getHash()) { std::cerr << "Hashes do not match" << std::endl; return false; @@ -363,15 +358,14 @@ bool SHAMap::syncTest() // add random data to the source map - int items=10000; - for(int i=0; i::iterator nodeIDIterator; std::list >::iterator rawNodeIterator; - int passes=0; - int nodes=0; + int passes = 0; + int nodes = 0; destination.setSynching(); - if(!source.getNodeFat(SHAMapNode(), nodeIDs, gotNodes)) + if (!source.getNodeFat(SHAMapNode(), nodeIDs, gotNodes)) { std::cerr << "GetNodeFat(root) fails" << std::endl; assert(false); return false; } - if(gotNodes.size()!=1) + if (gotNodes.size() != 1) { std::cerr << "Didn't get root node " << gotNodes.size() << std::endl; assert(false); return false; } - if(!destination.addRootNode(*gotNodes.begin())) + if (!destination.addRootNode(*gotNodes.begin())) { std::cerr << "AddRootNode fails" << std::endl; assert(false); @@ -416,12 +410,12 @@ bool SHAMap::syncTest() std::cerr << "ROOT COMPLETE, INNER SYNCHING" << std::endl; #endif #ifdef SMS_DEBUG - int bytes=0; + int bytes = 0; #endif do { - passes++; + ++passes; hashes.clear(); // get the list of nodes we know we need @@ -433,8 +427,8 @@ bool SHAMap::syncTest() #endif // get as many nodes as possible based on this information - for(nodeIDIterator=nodeIDs.begin(); nodeIDIterator!=nodeIDs.end(); ++nodeIDIterator) - if(!source.getNodeFat(*nodeIDIterator, gotNodeIDs, gotNodes)) + for (nodeIDIterator = nodeIDs.begin(); nodeIDIterator != nodeIDs.end(); ++nodeIDIterator) + if (!source.getNodeFat(*nodeIDIterator, gotNodeIDs, gotNodes)) { std::cerr << "GetNodeFat fails" << std::endl; assert(false); @@ -444,7 +438,7 @@ bool SHAMap::syncTest() nodeIDs.clear(); hashes.clear(); - if(gotNodeIDs.empty()) + if (gotNodeIDs.empty()) { std::cerr << "No nodes gotten" << std::endl; assert(false); @@ -454,14 +448,14 @@ bool SHAMap::syncTest() #ifdef SMS_DEBUG std::cerr << gotNodeIDs.size() << " found nodes" << std::endl; #endif - for(nodeIDIterator=gotNodeIDs.begin(), rawNodeIterator=gotNodes.begin(); - nodeIDIterator!=gotNodeIDs.end(); ++nodeIDIterator, ++rawNodeIterator) + for (nodeIDIterator = gotNodeIDs.begin(), rawNodeIterator = gotNodes.begin(); + nodeIDIterator != gotNodeIDs.end(); ++nodeIDIterator, ++rawNodeIterator) { - nodes++; + ++nodes; #ifdef SMS_DEBUG - bytes+=rawNodeIterator->size(); + bytes += rawNodeIterator->size(); #endif - if(!destination.addKnownNode(*nodeIDIterator, *rawNodeIterator)) + if (!destination.addKnownNode(*nodeIDIterator, *rawNodeIterator)) { std::cerr << "AddKnownNode fails" << std::endl; assert(false); @@ -472,15 +466,15 @@ bool SHAMap::syncTest() gotNodes.clear(); - } while(1); + } while (1); destination.clearSynching(); #ifdef SMS_DEBUG std::cerr << "SYNCHING COMPLETE " << items << " items, " << nodes << " nodes, " << - bytes/1024 << " KB" << std::endl; + bytes / 1024 << " KB" << std::endl; #endif - if(!source.deepCompare(destination)) + if (!source.deepCompare(destination)) { std::cerr << "DeepCompare fails" << std::endl; assert(false);