From 08d6b55fed8f3a3961007a6e11eb6626a8ec60ee Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 1 Feb 2012 20:51:58 -0800 Subject: [PATCH] Major optimization in the way we track SHAMaps. --- SHAMap.cpp | 16 ++++----- SHAMap.h | 24 +++++++++---- SHAMapNodes.cpp | 19 ++++++++--- SHAMapSync.cpp | 89 +++++++++++++++++++++++++++++++------------------ Serializer.cpp | 6 ++-- uint256.h | 5 ++- 6 files changed, 103 insertions(+), 56 deletions(-) diff --git a/SHAMap.cpp b/SHAMap.cpp index 8c2d8a132a..b4015ca13c 100644 --- a/SHAMap.cpp +++ b/SHAMap.cpp @@ -67,7 +67,7 @@ SHAMapLeafNode::pointer SHAMap::checkCacheLeaf(const SHAMapNode& iNode) SHAMapInnerNode::pointer SHAMap::checkCacheNode(const SHAMapNode& iNode) { assert(!iNode.isLeaf()); - std::map::iterator it=mInnerNodeByID.find(iNode); + boost::unordered_map::iterator it=mInnerNodeByID.find(iNode); if(it==mInnerNodeByID.end()) return SHAMapInnerNode::pointer(); return it->second; } @@ -89,10 +89,6 @@ SHAMapLeafNode::pointer SHAMap::walkToLeaf(const uint256& id, bool create, bool throw SHAMapException(InvalidNode); if(inNode->isEmptyBranch(branch)) { // no nodes below this one -#ifdef DEBUG - std::cerr << "No nodes below level " << i << ", branch " << branch << std::endl; - std::cerr << " terminal node is " << inNode->getString() << std::endl; -#endif if(!create) return SHAMapLeafNode::pointer(); return createLeaf(*inNode, id); } @@ -140,7 +136,7 @@ SHAMapInnerNode::pointer SHAMap::walkTo(const SHAMapNode& id) SHAMapInnerNode::pointer next=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), false); if(!next) // we don't have the next node { -#ifdef DEBUG +#ifdef ST_DEBUG std::cerr << "Unable to find node " << inNode->getChildNodeID(branch).getString() << std::endl; #endif return inNode; @@ -166,7 +162,7 @@ SHAMapLeafNode::pointer SHAMap::getLeaf(const SHAMapNode& id, const uint256& has if(leaf) return returnLeaf(leaf, modify); std::vector leafData; - if(!fetchNode(hash, leafData)) throw SHAMapException(MissingNode); + if(!fetchNode(hash, leafData)) return SHAMapLeafNode::pointer(); leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(id, leafData, mSeq)); if(leaf->getNodeHash()!=hash) throw SHAMapException(InvalidNode); mLeafByID.insert(std::make_pair(id, leaf)); @@ -179,7 +175,7 @@ SHAMapInnerNode::pointer SHAMap::getInner(const SHAMapNode& id, const uint256& h if(node) return returnNode(node, modify); std::vector rawNode; - if(!fetchNode(hash, rawNode)) throw SHAMapException(MissingNode); + if(!fetchNode(hash, rawNode)) return SHAMapInnerNode::pointer(); node=SHAMapInnerNode::pointer(new SHAMapInnerNode(id, rawNode, mSeq)); if(node->getNodeHash()!=hash) throw SHAMapException(InvalidNode); @@ -421,7 +417,7 @@ SHAMapItem::pointer SHAMap::peekPrevItem(const uint256& id) SHAMapLeafNode::pointer SHAMap::createLeaf(const SHAMapInnerNode& lowestParent, const uint256& id) { // caller must call dirtyUp if they populate the leaf -#ifdef DEBUG +#ifdef ST_DEBUG std::cerr << "createLeaf(" << lowestParent.getString() << std::endl; std::cerr << " for " << id.GetHex() << ")" << std::endl; #endif @@ -433,7 +429,7 @@ SHAMapLeafNode::pointer SHAMap::createLeaf(const SHAMapInnerNode& lowestParent, } SHAMapLeafNode::pointer newLeaf(new SHAMapLeafNode(SHAMapNode(SHAMapNode::leafDepth, id), mSeq)); mLeafByID[*newLeaf]=newLeaf; -#ifdef DEBUG +#ifdef ST_DEBUG std::cerr << "made leaf " << newLeaf->getString() << std::endl; #endif return newLeaf; diff --git a/SHAMap.h b/SHAMap.h index 571136e604..d762c054a4 100644 --- a/SHAMap.h +++ b/SHAMap.h @@ -7,6 +7,7 @@ #include #include +#include #include "types.h" #include "uint256.h" @@ -49,9 +50,9 @@ public: bool isInner() const { return !isRoot() && !isLeaf(); } virtual bool isPopulated() const { return false; } - SHAMapNode getParentNodeID() { return SHAMapNode(mDepth-1, mNodeID); } - SHAMapNode getChildNodeID(int m); - int selectBranch(const uint256& hash); + SHAMapNode getParentNodeID() const { return SHAMapNode(mDepth-1, mNodeID); } + SHAMapNode getChildNodeID(int m) const; + int selectBranch(const uint256& hash) const; bool operator<(const SHAMapNode&) const; bool operator>(const SHAMapNode&) const; @@ -62,8 +63,10 @@ public: bool operator<=(const SHAMapNode&) const; bool operator>=(const SHAMapNode&) const; + std::size_t getHash() const; + virtual std::string getString() const; - void dump(); + void dump() const; static void ClassInit(); static uint256 getNodeID(int depth, const uint256& hash); @@ -164,7 +167,6 @@ class SHAMapInnerNode : public SHAMapNode public: typedef boost::shared_ptr pointer; - private: uint256 mHash; uint256 mHashes[32]; @@ -208,6 +210,14 @@ enum SHAMapException InvalidNode=2 }; +class hash_SMN : std::unary_function +{ +public: + std::size_t operator() (const SHAMapNode& mn) const + { + return mn.getHash(); + } +}; class SHAMap { @@ -219,7 +229,9 @@ private: uint32 mSeq; mutable boost::recursive_mutex mLock; std::map mLeafByID; - std::map mInnerNodeByID; + + boost::unordered_map mInnerNodeByID; + boost::shared_ptr > mDirtyLeafNodes; boost::shared_ptr > mDirtyInnerNodes; diff --git a/SHAMapNodes.cpp b/SHAMapNodes.cpp index 1963d0aa1d..14add7ed7a 100644 --- a/SHAMapNodes.cpp +++ b/SHAMapNodes.cpp @@ -81,13 +81,24 @@ uint256 SHAMapNode::getNodeID(int depth, const uint256& hash) return hash & smMasks[depth]; } +std::size_t SHAMapNode::getHash() const +{ + std::size_t ret=mDepth; + for(int i=0; i<5; i++) + { + ret*=2654435761U; + ret^=mNodeID.PeekAt(i); + } + return ret; +} + SHAMapNode::SHAMapNode(int depth, const uint256 &hash) : mDepth(depth) { // canonicalize the hash to a node ID for this depth assert(depth>=0 && depth<=leafDepth); mNodeID = getNodeID(depth, hash); } -SHAMapNode SHAMapNode::getChildNodeID(int m) +SHAMapNode SHAMapNode::getChildNodeID(int m) const { assert(!isLeaf()); assert((m>=0) && (m<32)); @@ -99,7 +110,7 @@ SHAMapNode SHAMapNode::getChildNodeID(int m) return ret; } -int SHAMapNode::selectBranch(const uint256 &hash) +int SHAMapNode::selectBranch(const uint256 &hash) const { if(isLeaf()) // no nodes under this node { @@ -118,7 +129,7 @@ int SHAMapNode::selectBranch(const uint256 &hash) return branch; } -void SHAMapNode::dump() +void SHAMapNode::dump() const { std::cerr << getString() << std::endl; } @@ -172,7 +183,7 @@ bool SHAMapLeafNode::hasItem(const uint256& item) const bool SHAMapLeafNode::addUpdateItem(SHAMapItem::pointer item, bool doHash) { // The node will almost never have more than one item in it -#ifdef DEBUG +#ifdef ST_DEBUG std::cerr << "Leaf(" << getString() << ")" << std::endl; std::cerr << " addi(" << item->getTag().GetHex() << std::endl; #endif diff --git a/SHAMapSync.cpp b/SHAMapSync.cpp index eeda44ae20..2ff2dbe372 100644 --- a/SHAMapSync.cpp +++ b/SHAMapSync.cpp @@ -1,6 +1,8 @@ #include +#include + #include "SHAMap.h" void SHAMap::getMissingNodes(std::vector& nodeIDs, std::vector& hashes, int max) @@ -9,7 +11,7 @@ void SHAMap::getMissingNodes(std::vector& nodeIDs, std::vectorisFullBelow()) { -#ifdef DEBUG +#ifdef GMN_DEBUG std::cerr << "getMissingNodes: root is full below" << std::endl; #endif return; @@ -23,7 +25,7 @@ void SHAMap::getMissingNodes(std::vector& nodeIDs, std::vectorgetString() << std::endl; #endif @@ -31,35 +33,29 @@ void SHAMap::getMissingNodes(std::vector& nodeIDs, std::vectorisEmptyBranch(i)) { -#ifdef DEBUG +#ifdef GNMN_DEBUG std::cerr << "gMN: " << node->getString() << " has non-empty branch " << i << std::endl; #endif - try - { - if(node->isChildLeaf()) - { // do we have this leaf node? - SHAMapLeafNode::pointer leaf=getLeaf(node->getChildNodeID(i), node->getChildHash(i), false); - assert(leaf); - } - else - { - SHAMapInnerNode::pointer desc=getInner(node->getChildNodeID(i), node->getChildHash(i), false); - assert(desc); - if(!desc->isFullBelow()) - stack.push(desc); - } + bool missing=false; + SHAMapNode childNID=node->getChildNodeID(i); + if(node->isChildLeaf()) + { // do we have this leaf node? + if(!getLeaf(childNID, node->getChildHash(i), false)) missing=true; } - catch(const SHAMapException &x) + else { - if(x!=SHAMapException(MissingNode)) throw(x); - if(max-->0) - { -#ifdef DEBUG - std::cerr << "gMN: need leaf " << node->getChildNodeID(i).getString() << std::endl; + SHAMapInnerNode::pointer desc=getInner(childNID, node->getChildHash(i), false); + if(!desc) + missing=true; + else if(!desc->isFullBelow()) + stack.push(desc); + } + if(missing && max-->0) + { +#ifdef GMN_DEBUG + std::cerr << "gMN: need " << node->getChildNodeID(i).getString() << std::endl; #endif - nodeIDs.push_back(node->getChildNodeID(i)); - hashes.push_back(node->getChildHash(i)); - } + nodeIDs.push_back(childNID); } } } @@ -253,7 +249,32 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vectorgetChildNodeID(i))) + return true; + pNode->setFullBelow(); + + while(!pNode->isRoot()) + { + pNode=checkCacheNode(pNode->getParentNodeID()); + if(!pNode) + { + assert(false); + return false; + } + for(int i=0; i<32; i++) + if(!checkCacheNode(pNode->getChildNodeID(i))) + return true; + pNode->setFullBelow(); + } return true; } @@ -270,7 +291,7 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vectorgetString() << std::endl; #endif @@ -353,10 +374,14 @@ bool SHAMap::deepCompare(SHAMap& other) bool SHAMap::syncTest() { + unsigned int seed; + RAND_pseudo_bytes(reinterpret_cast(&seed), sizeof(seed)); + srand(seed); + SHAMap source, destination; // add random data to the source map - int items=3; // 10+rand()%400; + int items=10+rand()%4000; for(int i=0; i& data, int size) { - char buf[64]; + uint256 j[2]; if((size<0)||(size>data.size())) size=data.size(); - SHA512(&(data.front()), size, (unsigned char *) buf); - return * reinterpret_cast(&buf[0]); + SHA512(&(data.front()), size, (unsigned char *) j); + return j[0]; } bool Serializer::checkSignature(int pubkeyOffset, int signatureOffset) const diff --git a/uint256.h b/uint256.h index 4ddbb772f4..719c3355da 100644 --- a/uint256.h +++ b/uint256.h @@ -290,7 +290,10 @@ public: return (!(a == b)); } - + unsigned int PeekAt(int j) const + { + return pn[j]; + } std::string GetHex() const {