From 0df9ec91fa634a7fcc3d2f95cad704431879c776 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 25 Nov 2011 00:03:33 -0800 Subject: [PATCH] Debug and unit test code. Copy on write logic. Bugfixes. --- SHAMap.cpp | 172 +++++++++++++++++++++++++++++++++++------------- SHAMap.h | 40 ++++++++--- SHAMapNodes.cpp | 43 +++++++++--- 3 files changed, 192 insertions(+), 63 deletions(-) diff --git a/SHAMap.cpp b/SHAMap.cpp index 7e0d9ee808..b8c648c0af 100644 --- a/SHAMap.cpp +++ b/SHAMap.cpp @@ -4,16 +4,22 @@ #include -SHAMap::SHAMap() +SHAMap::SHAMap() : mSeq(0) { - ; + root=SHAMapInnerNode::pointer(new SHAMapInnerNode(SHAMapNode(SHAMapNode::rootDepth, uint256()), mSeq)); } void SHAMap::dirtyUp(const uint256& id) { // walk the tree up from through the inner nodes to the root // update linking hashes and add nodes to dirty list +#ifdef DEBUG + std::cerr << "dirtyUp(" << id.GetHex() << ")" << std::endl; +#endif SHAMapLeafNode::pointer leaf=mLeafByID[SHAMapNode(SHAMapNode::leafDepth, id)]; if(!leaf) throw SHAMapException(MissingNode); +#ifdef DEBUG + std::cerr << "found leaf " << leaf->getString() << std::endl; +#endif uint256 hVal=leaf->getNodeHash(); if(mDirtyLeafNodes) (*mDirtyLeafNodes)[*leaf]=leaf; @@ -21,6 +27,9 @@ void SHAMap::dirtyUp(const uint256& id) for(int depth=SHAMapNode::leafDepth-1; depth>=0; depth--) { // walk up the tree to the root updating nodes +#ifdef DEBUG + std::cerr << "walk up to " << depth << ", " << leaf->getNodeID().GetHex() << std::endl; +#endif SHAMapInnerNode::pointer node=mInnerNodeByID[SHAMapNode(depth, leaf->getNodeID())]; if(!node) throw SHAMapException(MissingNode); if(!node->setChildHash(node->selectBranch(id), hVal)) return; @@ -37,11 +46,17 @@ SHAMapLeafNode::pointer SHAMap::checkCacheLeaf(const SHAMapNode& iNode) } -SHAMapLeafNode::pointer SHAMap::walkToLeaf(const uint256& id, bool create) +SHAMapLeafNode::pointer SHAMap::walkToLeaf(const uint256& id, bool create, bool modify) { // walk down to the leaf that would contain this ID // is leaf node in cache +#ifdef DEBUG + std::cerr << "walkToLeaf(" << id.GetHex() << ")"; + if(create) std::cerr << " create"; + if(modify) std::cerr << " modify"; + std::cerr << std::endl; +#endif SHAMapLeafNode::pointer ln=checkCacheLeaf(SHAMapNode(SHAMapNode::leafDepth, id)); - if(ln) return ln; + if(ln) return returnLeaf(ln, modify); // walk tree to leaf SHAMapInnerNode::pointer inNode=root; @@ -53,66 +68,99 @@ SHAMapLeafNode::pointer SHAMap::walkToLeaf(const uint256& id, bool create) throw SHAMapException(InvalidNode); if(inNode->isEmptyBranch(branch)) { // no nodes below this one +#ifdef DEBUG + std::cerr << "No nodes below level " << i << std::endl; +#endif if(!create) return SHAMapLeafNode::pointer(); return createLeaf(*inNode, id); } if(i!=(SHAMapNode::leafDepth)-1) { // child is another inner node - inNode=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch)); + inNode=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), modify); if(inNode==NULL) throw SHAMapException(InvalidNode); } else // child is leaf node { - ln=getLeaf(inNode->getChildNodeID(branch), inNode->getChildHash(branch)); + ln=getLeaf(inNode->getChildNodeID(branch), inNode->getChildHash(branch), modify); if(ln==NULL) { - if(!create) return SHAMapLeafNode::pointer(); + if(!create) return SHAMapLeafNode::pointer(); return createLeaf(*inNode, id); } } } - return ln; + return returnLeaf(ln, modify); } -SHAMapLeafNode::pointer SHAMap::getLeaf(const SHAMapNode &id, const uint256& hash) +SHAMapLeafNode::pointer SHAMap::getLeaf(const SHAMapNode &id, const uint256& hash, bool modify) { // retrieve a leaf whose node hash is known +#ifdef DEBUG + std::cerr << "getLeaf(" << id.getString() << ", " << hash.GetHex() << ")"; + if(modify) std::cerr << " modify"; + std::cerr << std::endl; +#endif assert(!!hash); if(!id.isLeaf()) return SHAMapLeafNode::pointer(); SHAMapLeafNode::pointer leaf=mLeafByID[id]; // is the leaf in memory - if(leaf) return leaf; + if(leaf) return returnLeaf(leaf, modify); std::vector leafData; // is it in backing store if(!fetchLeafNode(hash, id, leafData)) - throw SHAMapException(MissingNode); + throw SHAMapException(MissingNode); - leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(id)); - + leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(id, mSeq)); BOOST_FOREACH(SHAMapItem::pointer& item, leafData) leaf->addUpdateItem(item); - leaf->updateHash(); if(leaf->getNodeHash()!=hash) throw SHAMapException(InvalidNode); mLeafByID[id]=leaf; return leaf; } -SHAMapInnerNode::pointer SHAMap::getInner(const SHAMapNode &id, const uint256& hash) +SHAMapInnerNode::pointer SHAMap::getInner(const SHAMapNode &id, const uint256& hash, bool modify) { // retrieve an inner node whose node hash is known +#ifdef DEBUG + std::cerr << "getInner(" << id.getString() << ", " << hash.GetHex() << ")"; + if(modify) std::cerr << " modify"; + std::cerr << std::endl; +#endif SHAMapInnerNode::pointer node=mInnerNodeByID[id]; - if(node) return node; + if(node) return returnNode(node, modify); std::vector rawNode; if(!fetchInnerNode(hash, id, rawNode)) throw SHAMapException(MissingNode); - node=SHAMapInnerNode::pointer(new SHAMapInnerNode(id, rawNode)); + node=SHAMapInnerNode::pointer(new SHAMapInnerNode(id, rawNode, mSeq)); if(node->getNodeHash()!=hash) throw SHAMapException(InvalidNode); mInnerNodeByID[id]=node; return node; } +SHAMapLeafNode::pointer SHAMap::returnLeaf(SHAMapLeafNode::pointer leaf, bool modify) +{ // make sure the leaf is suitable for the intended operation (copy on write) + if(leaf && modify && (leaf->getSeq()!=mSeq)) + { + leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(*leaf, mSeq)); + mLeafByID[*leaf]=leaf; + if(mDirtyLeafNodes) (*mDirtyLeafNodes)[*leaf]=leaf; + } + return leaf; +} + +SHAMapInnerNode::pointer SHAMap::returnNode(SHAMapInnerNode::pointer node, bool modify) +{ // make sure the node is suitable for the intended operation (copy on write) + if(node && modify && (node->getSeq()!=mSeq)) + { + node=SHAMapInnerNode::pointer(new SHAMapInnerNode(*node, mSeq)); + mInnerNodeByID[*node]=node; + if(mDirtyInnerNodes) (*mDirtyInnerNodes)[*node]=node; + } + return node; +} + SHAMapItem::SHAMapItem(const uint256& tag, const std::vector& data) : mTag(tag), mData(data) { ; } @@ -133,20 +181,24 @@ SHAMapItem::pointer SHAMap::peekLastItem() return lastBelow(root); } -SHAMapItem::pointer SHAMap::firstBelow(SHAMapInnerNode::pointer Node) +SHAMapItem::pointer SHAMap::firstBelow(SHAMapInnerNode::pointer node) { +#ifdef DEBUG + std::cerr << "firstBelow(" << node->getString() << ")" << std::endl; +#endif + const uint256 zero; int i; - while(Node->isChildLeaf()) + while(!node->isChildLeaf()) { for(i=0; i<32; i++) { - uint256 cHash(Node->getChildHash(i)); + uint256 cHash(node->getChildHash(i)); if(cHash!=zero) { - Node=getInner(Node->getChildNodeID(i), cHash); - if(!Node) return SHAMapItem::pointer(); + node=getInner(node->getChildNodeID(i), cHash, false); + if(!node) return SHAMapItem::pointer(); break; } } @@ -155,10 +207,10 @@ SHAMapItem::pointer SHAMap::firstBelow(SHAMapInnerNode::pointer Node) for(int i=0; i<32; i++) { - uint256 cHash=Node->getChildHash(i); + uint256 cHash=node->getChildHash(i); if(cHash!=zero) { - SHAMapLeafNode::pointer mLeaf=getLeaf(Node->getChildNodeID(i), cHash); + SHAMapLeafNode::pointer mLeaf=getLeaf(node->getChildNodeID(i), cHash, false); if(!mLeaf) return SHAMapItem::pointer(); return mLeaf->firstItem(); } @@ -166,22 +218,22 @@ SHAMapItem::pointer SHAMap::firstBelow(SHAMapInnerNode::pointer Node) return SHAMapItem::pointer(); } -SHAMapItem::pointer SHAMap::lastBelow(SHAMapInnerNode::pointer Node) +SHAMapItem::pointer SHAMap::lastBelow(SHAMapInnerNode::pointer node) { ScopedLock sl(mLock); const uint256 zero; int i; - while(Node->isChildLeaf()) + while(!node->isChildLeaf()) { for(i=31; i>=0; i--) { - uint256 cHash(Node->getChildHash(i)); + uint256 cHash(node->getChildHash(i)); if(cHash!=0) { - Node=getInner(Node->getChildNodeID(i), cHash); - if(!Node) return SHAMapItem::pointer(); + node=getInner(node->getChildNodeID(i), cHash, false); + if(!node) return SHAMapItem::pointer(); break; } } @@ -189,10 +241,10 @@ SHAMapItem::pointer SHAMap::lastBelow(SHAMapInnerNode::pointer Node) } for(int i=31; i>=0; i--) { - uint256 cHash=Node->getChildHash(i); + uint256 cHash=node->getChildHash(i); if(cHash!=zero) { - SHAMapLeafNode::pointer mLeaf=getLeaf(Node->getChildNodeID(i), cHash); + SHAMapLeafNode::pointer mLeaf=getLeaf(node->getChildNodeID(i), cHash, false); if(!mLeaf) return SHAMapItem::pointer(); return mLeaf->lastItem(); } @@ -203,7 +255,7 @@ SHAMapItem::pointer SHAMap::peekNextItem(const uint256& id) { // Get a pointer to the next item in the tree after a given item - item must be in tree ScopedLock sl(mLock); - SHAMapLeafNode::pointer leaf=walkToLeaf(id, false); + SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false); if(!leaf) return SHAMapItem::pointer(); // is there another item in this leaf? (there almost never will be) @@ -223,7 +275,7 @@ SHAMapItem::pointer SHAMap::peekNextItem(const uint256& id) if(nextNode.isLeaf()) { // this is a terminal inner node - leaf=getLeaf(nextNode, nextHash); + leaf=getLeaf(nextNode, nextHash, false); if(!leaf) throw SHAMapException(MissingNode); next=leaf->firstItem(); if(!next) throw SHAMapException(InvalidNode); @@ -231,7 +283,7 @@ SHAMapItem::pointer SHAMap::peekNextItem(const uint256& id) } // the next item is the first item below this node - SHAMapInnerNode::pointer inner=getInner(nextNode, nextHash); + SHAMapInnerNode::pointer inner=getInner(nextNode, nextHash, false); if(!inner) throw SHAMapException(MissingNode); next=firstBelow(inner); if(!next) throw SHAMapException(InvalidNode); @@ -250,22 +302,31 @@ SHAMapItem::pointer SHAMap::peekPrevItem(const uint256& id) SHAMapLeafNode::pointer SHAMap::createLeaf(const SHAMapInnerNode& lowestParent, const uint256& id) { - int depth=lowestParent.getDepth(); - for(int depth=lowestParent.getDepth(); depthgetString() << std::endl; +#endif mInnerNodeByID[*newNode]=newNode; } - - SHAMapLeafNode::pointer newLeaf(new SHAMapLeafNode(SHAMapNode(SHAMapNode::leafDepth, id))); + SHAMapLeafNode::pointer newLeaf(new SHAMapLeafNode(SHAMapNode(SHAMapNode::leafDepth, id), mSeq)); mLeafByID[*newLeaf]=newLeaf; +#ifdef DEBUG + std::cerr << "made leaf " << newLeaf->getString() << std::endl; +#endif return newLeaf; } SHAMapItem::pointer SHAMap::peekItem(const uint256& id) { ScopedLock sl(mLock); - SHAMapLeafNode::pointer leaf=walkToLeaf(id, false); + SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false); if(!leaf) return SHAMapItem::pointer(); return leaf->findItem(id); } @@ -273,7 +334,7 @@ SHAMapItem::pointer SHAMap::peekItem(const uint256& id) bool SHAMap::hasItem(const uint256& id) { // does the tree have an item with this ID ScopedLock sl(mLock); - SHAMapLeafNode::pointer leaf=walkToLeaf(id, false); + SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false); if(!leaf) return false; SHAMapItem::pointer item=leaf->findItem(id); return (bool) item; @@ -282,7 +343,7 @@ bool SHAMap::hasItem(const uint256& id) bool SHAMap::delItem(const uint256& id) { // delete the item with this ID ScopedLock sl(mLock); - SHAMapLeafNode::pointer leaf=walkToLeaf(id, false); + SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false); if(!leaf) return false; if(!leaf->delItem(id)) return false; dirtyUp(id); @@ -292,7 +353,7 @@ bool SHAMap::delItem(const uint256& id) bool SHAMap::addGiveItem(const SHAMapItem::pointer item) { // add the specified item ScopedLock sl(mLock); - SHAMapLeafNode::pointer leaf=walkToLeaf(item->getTag(), true); + SHAMapLeafNode::pointer leaf=walkToLeaf(item->getTag(), true, true); if(!leaf) return false; if(leaf->hasItem(item->getTag())) return false; if(!leaf->addUpdateItem(item)) return false; @@ -300,10 +361,15 @@ bool SHAMap::addGiveItem(const SHAMapItem::pointer item) return true; } +bool SHAMap::addItem(const SHAMapItem& i) +{ + return addGiveItem(SHAMapItem::pointer(new SHAMapItem(i))); +} + bool SHAMap::updateGiveItem(SHAMapItem::pointer item) { ScopedLock sl(mLock); - SHAMapLeafNode::pointer leaf=walkToLeaf(item->getTag(), true); + SHAMapLeafNode::pointer leaf=walkToLeaf(item->getTag(), true, true); if(!leaf) return false; if(!leaf->addUpdateItem(item)) return false; dirtyUp(item->getTag()); @@ -338,6 +404,14 @@ bool SHAMap::writeLeafNode(const uint256&, const SHAMapNode&, const std::vector< void SHAMap::dump() { + std::cerr << "SHAMap::dump" << std::endl; + SHAMapItem::pointer i=peekFirstItem(); + while(i) + { + std::cerr << "Item: id=" << i->getTag().GetHex() << std::endl; + i=peekNextItem(i->getTag()); + } + std::cerr << "SHAMap::dump done" << std::endl; } static std::vectorIntToVUC(int i) @@ -358,6 +432,16 @@ bool SHAMap::TestSHAMap() SHAMap sMap; SHAMapItem i1(h1, IntToVUC(1)), i2(h2, IntToVUC(2)), i3(h3, IntToVUC(3)), i4(h4, IntToVUC(4)), i5(h5, IntToVUC(5)); + +#ifdef DEBUG + std::cerr << "addItem1" << std::endl; +#endif + sMap.addItem(i1); +#ifdef DEBUG + std::cerr << "addItem2" << std::endl; +#endif + sMap.addItem(i2); + sMap.dump(); sMap.dump(); } diff --git a/SHAMap.h b/SHAMap.h index e2c5a90ae1..67ce5dd680 100644 --- a/SHAMap.h +++ b/SHAMap.h @@ -17,12 +17,11 @@ class SHAMap; // The trees are designed for rapid synchronization and compression of differences - class SHAMapNode +class SHAMapNode { // Identifies a node in a SHA256 hash public: typedef boost::shared_ptr pointer; - private: static uint256 smMasks[11]; // AND with hash to get node id @@ -56,7 +55,8 @@ public: bool operator<=(const SHAMapNode&) const; bool operator>=(const SHAMapNode&) const; - virtual void dump(void); + virtual std::string getString(void) const; + void dump(void); static void ClassInit(); static uint256 getNodeID(int depth, const uint256& hash); @@ -112,19 +112,27 @@ public: private: uint256 mHash; std::list mItems; + uint32 mSeq; bool updateHash(); + SHAMapLeafNode(const SHAMapLeafNode&); // no implementation + SHAMapLeafNode& operator=(const SHAMapLeafNode&); // no implementation + protected: bool addUpdateItem(SHAMapItem::pointer); bool delItem(const SHAMapItem::pointer i) { delItem(i->getTag()); } bool delItem(const uint256& tag); public: - SHAMapLeafNode(const SHAMapNode& nodeID); + SHAMapLeafNode(const SHAMapNode& nodeID, uint32 seq); + SHAMapLeafNode(const SHAMapLeafNode& node, uint32 seq); virtual bool isPopulated(void) const { return true; } + uint32 getSeq(void) const { return mSeq; } + void setSeq(uint32 s) { mSeq=s; } + const uint256& getNodeHash() const { return mHash; } bool isEmpty() const { return mItems.empty(); } int getItemCount() const { return mItems.size(); } @@ -150,15 +158,23 @@ public: private: uint256 mHash; uint256 mHashes[32]; + uint32 mSeq; bool updateHash(); + SHAMapInnerNode(const SHAMapInnerNode&); // no implementation + SHAMapInnerNode& operator=(const SHAMapInnerNode&); // no implementation + protected: bool setChildHash(int m, const uint256& hash); public: - SHAMapInnerNode(const SHAMapNode& id); - SHAMapInnerNode(const SHAMapNode& id, const std::vector& contents); + SHAMapInnerNode(const SHAMapNode& id, uint32 seq); + SHAMapInnerNode(const SHAMapInnerNode& node, uint32 seq); + SHAMapInnerNode(const SHAMapNode& id, const std::vector& contents, uint32 seq); + + uint32 getSeq(void) const { return mSeq; } + void setSeq(uint32 s) { mSeq=s; } virtual bool isPopulated(void) const { return true; } @@ -183,7 +199,7 @@ public: typedef boost::shared_ptr pointer; private: - int mLeafDataSize, mLeafDataOffset, mSeq; + uint32 mSeq; mutable boost::recursive_mutex mLock; std::map mLeafByID; std::map mInnerNodeByID; @@ -197,15 +213,19 @@ protected: SHAMapLeafNode::pointer createLeaf(const SHAMapInnerNode& lowestParent, const uint256& id); SHAMapLeafNode::pointer checkCacheLeaf(const SHAMapNode &); - SHAMapLeafNode::pointer walkToLeaf(const uint256& id, bool create); + SHAMapLeafNode::pointer walkToLeaf(const uint256& id, bool create, bool modify); - SHAMapLeafNode::pointer getLeaf(const SHAMapNode& id, const uint256& hash); - SHAMapInnerNode::pointer getInner(const SHAMapNode& id, const uint256& hash); + SHAMapLeafNode::pointer getLeaf(const SHAMapNode& id, const uint256& hash, bool modify); + SHAMapLeafNode::pointer returnLeaf(SHAMapLeafNode::pointer leaf, bool modify); + SHAMapInnerNode::pointer getInner(const SHAMapNode& id, const uint256& hash, bool modify); + SHAMapInnerNode::pointer returnNode(SHAMapInnerNode::pointer node, bool modify); SHAMapItem::pointer firstBelow(SHAMapInnerNode::pointer); SHAMapItem::pointer lastBelow(SHAMapInnerNode::pointer); public: + + // build new map SHAMap(); // hold the map stable across operations diff --git a/SHAMapNodes.cpp b/SHAMapNodes.cpp index 86c228a177..1b01301837 100644 --- a/SHAMapNodes.cpp +++ b/SHAMapNodes.cpp @@ -3,6 +3,17 @@ #include "SHAMap.h" #include +#include + +std::string SHAMapNode::getString() const +{ + std::string ret="NodeID("; + ret+=boost::lexical_cast(mDepth); + ret+=","; + ret+=mNodeID.GetHex(); + ret+=")"; + return ret; +} uint256 SHAMapNode::smMasks[11]; @@ -51,7 +62,7 @@ void SHAMapNode::ClassInit() for(i=0; i<64; i++) HexBuf[i]='0'; HexBuf[64]=0; - for(i=0; i& contents) - : SHAMapNode(id) +SHAMapInnerNode::SHAMapInnerNode(const SHAMapNode& id, const std::vector& contents, uint32 seq) + : SHAMapNode(id), mSeq(seq) { + assert(!id.isLeaf()); Serializer s(contents); for(int i=0; i<32; i++) - mHashes[i]=s.get256(i*32); + mHashes[i]=s.get256(i*32); +} + +SHAMapInnerNode::SHAMapInnerNode(const SHAMapInnerNode& node, uint32 seq) : SHAMapNode(node), mHash(node.mHash), + mSeq(seq) +{ + assert(!node.isLeaf()); + memcpy(mHashes, node.mHashes, sizeof(mHashes)); } bool SHAMapInnerNode::setChildHash(int m, const uint256 &hash)