diff --git a/SHAMap.cpp b/SHAMap.cpp index 16c3e8e706..7ccd8e83cd 100644 --- a/SHAMap.cpp +++ b/SHAMap.cpp @@ -1,139 +1,250 @@ +#include "Serializer.h" #include "BitcoinUtil.h" #include "SHAMap.h" + #include -bool SHAMapNode::operator<(const SHAMapNode &s) const -{ - if(s.mDepthmDepth) return false; - return mNodeID(const SHAMapNode &s) const -{ - if(s.mDepthmDepth) return true; - return mNodeID>s.mNodeID; -} - -bool SHAMapNode::operator<=(const SHAMapNode &s) const -{ - if(s.mDepthmDepth) return false; - return mNodeID<=s.mNodeID; -} - -bool SHAMapNode::operator>=(const SHAMapNode &s) const -{ - if(s.mDepthmDepth) return true; - return mNodeID>=s.mNodeID; -} - -bool SHAMapNode::operator==(const SHAMapNode &s) const -{ - return (s.mDepth==mDepth) && (s.mNodeID==mNodeID); -} - -bool SHAMapNode::operator!=(const SHAMapNode &s) const -{ - return (s.mDepth!=mDepth) || (s.mNodeID!=mNodeID); -} - -void SHAMapNode::ClassInit() -{ - int i; - char HexBuf[65]; - - for(i=0; i<64; i++) HexBuf[i]='0'; - HexBuf[64]=0; - for(i=0; i=0 && depth>=(mDepth*8); - - return SHAMapNode(mDepth+1, mNodeID | branch); -} - -int SHAMapNode::selectBranch(const uint256 &hash) -{ - if(isLeaf()) // no nodes under this node - return -1; - if((hash&smMasks[mDepth])!=mNodeID) - return -1; // does not go under this node - - uint256 selector=hash&smMasks[mDepth+1]; - int branch=*(selector.begin()+mDepth); - - assert(branch>=0 && branch<32); - return branch; -} - -SHAMapLeafNode::SHAMapLeafNode(const SHAMapNode& nodeID) : SHAMapNode(nodeID), mHash(0) +SHAMap::SHAMap(int leafDataSize) : mLeafDataSize(leafDataSize) { ; } -bool SHAMapLeafNode::hasItem(const uint256& item) const -{ - BOOST_FOREACH(const SHAMapItem& nodeItem, mItems) - if(nodeItem==item) return true; - return false; +void SHAMap::dirtyUp(const uint256& id, const std::vector& path) +{ // walk the tree up from through the inner nodes to the root + // update linking hashes and add nodes to dirty list + std::vector::const_reverse_iterator it; + + it = path.rbegin(); + if(it == path.rend()) return; + + mDirtyInnerNodes[**it]=*it; + uint256 hVal=(*it)->getNodeHash(); + if(hVal==0) + mInnerNodeByID.erase(**it); + + for(++it; itsetChildHash((*it)->selectBranch(id), hVal)) + return; + mDirtyInnerNodes[**it]=*it; + hVal = (*it)->getNodeHash(); + if(hVal == 0) mInnerNodeByID.erase(**it); + } + assert(**it == *root); } -bool SHAMapLeafNode::addUpdateItem(const SHAMapItem& item) -{ // The node will almost never have more than one item in it - std::list::iterator it; - for(it=mItems.begin(); it!=mItems.end(); it++) - { - if(*it==item) - { - if(it->peekData()==item.peekData()) - return false; // no change - it->updateData(item.peekData()); - updateHash(); - return true; - } - if(*it>item) break; - } - mItems.insert(it, item); - updateHash(); - return true; -} - -bool SHAMapLeafNode::delItem(const uint256& tag) +SHAMapLeafNode::pointer SHAMap::checkCacheLeaf(const SHAMapNode& iNode) { - std::list::iterator it; - for(it=mItems.begin(); it!=mItems.end(); it++) + assert(iNode.isLeaf()); + return mLeafByID[iNode]; +} + + +SHAMapLeafNode::pointer SHAMap::walkToLeaf(const uint256& id, bool create, + std::vector& path) +{ // walk down to the leaf that would contain this ID + // is leaf node in cache + SHAMapLeafNode::pointer ln=checkCacheLeaf(SHAMapNode(SHAMapNode::leafDepth, id)); + if(ln != SHAMapLeafNode::pointer()) return ln; + + // walk tree to leaf + SHAMapInnerNode::pointer inNode=root; + path.push_back(inNode); + + for(int i=1; iselectBranch(id); + if(branch<0) + { // somehow we got on the wrong branch + assert(false); + return SHAMapLeafNode::pointer(); + } + if(inNode->isEmptyBranch(branch)) + { // no nodes below this one + if(!create) return SHAMapLeafNode::pointer(); + return createLeaf(*inNode, id); + } + if(i!=(SHAMapNode::leafDepth)-1) + { // child is another inner node + inNode=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch)); + if(inNode==NULL) return SHAMapLeafNode::pointer(); // we don't have the node + path.push_back(inNode); + } + else // child is leaf node { - mItems.erase(it); - updateHash(); - return true; + ln=getLeaf(inNode->getChildNodeID(branch), inNode->getChildHash(branch)); + if(ln==NULL) + { + if(!create) return SHAMapLeafNode::pointer(); + return createLeaf(*inNode, id); + } } } - return false; + + return ln; } -void SHAMapLeafNode::updateHash(void) +SHAMapLeafNode::pointer SHAMap::getLeaf(const SHAMapNode &id, const uint256& hash) +{ // retrieve a leaf whose node hash is known + SHAMapLeafNode::pointer leaf=mLeafByID[id]; + if(leaf != SHAMapLeafNode::pointer()) return leaf; + + std::vector rawNode; + if(!fetchNode(hash, id, rawNode)) return leaf; + + leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(id)); + // construct leaf WRITEME + return leaf; +} + +SHAMapInnerNode::pointer SHAMap::getInner(const SHAMapNode &id, const uint256& hash) +{ // retrieve an inner node whose node hash is known + SHAMapInnerNode::pointer node=mInnerNodeByID[id]; + if(node != SHAMapInnerNode::pointer()) return node; + + std::vector rawNode; + if(!fetchNode(hash, id, rawNode)) return node; + + node=SHAMapInnerNode::pointer(new SHAMapInnerNode(id, rawNode)); + if(node->getNodeHash()!=hash) + { + badNode(hash, id); + return SHAMapInnerNode::pointer(); + } + + mInnerNodeByID[id]=node; + return node; +} + +SHAMapItem::pointer SHAMap::firstItem() { + ScopedLock sl(mLock); + return firstBelow(root); } +SHAMapItem::pointer SHAMap::lastItem() +{ + ScopedLock sl(mLock); + return lastBelow(root); +} + +SHAMapItem::pointer SHAMap::firstBelow(SHAMapInnerNode::pointer Node) +{ + const uint256 zero; + int i; + + while(Node->isChildLeaf()) + { + for(i=0; i<32; i++) + { + uint256 cHash(Node->getChildHash(i)); + if(cHash!=zero) + { + Node=getInner(Node->getChildNodeID(i), cHash); + if(Node==SHAMapInnerNode::pointer()) return SHAMapItem::pointer(); + break; + } + } + if(i==32) return SHAMapItem::pointer(); + } + + for(int i=0; i<32; i++) + { + uint256 cHash=Node->getChildHash(i); + if(cHash!=zero) + { + SHAMapLeafNode::pointer mLeaf=getLeaf(Node->getChildNodeID(i), cHash); + if(mLeaf==SHAMapLeafNode::pointer()) return SHAMapItem::pointer(); + return mLeaf->firstItem(); + } + } + return SHAMapItem::pointer(); +} + +SHAMapItem::pointer SHAMap::lastBelow(SHAMapInnerNode::pointer Node) +{ + ScopedLock sl(mLock); + + const uint256 zero; + int i; + + while(Node->isChildLeaf()) + { + for(i=31; i>=0; i--) + { + uint256 cHash(Node->getChildHash(i)); + if(cHash!=0) + { + Node=getInner(Node->getChildNodeID(i), cHash); + if(Node==SHAMapInnerNode::pointer()) return SHAMapItem::pointer(); + break; + } + } + if(i<0) return SHAMapItem::pointer(); + } + for(int i=31; i>=0; i--) + { + uint256 cHash=Node->getChildHash(i); + if(cHash!=zero) + { + SHAMapLeafNode::pointer mLeaf=getLeaf(Node->getChildNodeID(i), cHash); + if(mLeaf==SHAMapLeafNode::pointer()) return SHAMapItem::pointer(); + return mLeaf->lastItem(); + } + } +} + +SHAMapItem::pointer SHAMap::nextItem(const SHAMapItem &) +{ + // WRITEME +} + +SHAMapItem::pointer SHAMap::prevItem(const SHAMapItem &) +{ + // WRITEME +} + +bool SHAMap::hasItem(const uint256& id) +{ // does the tree have an item with this ID + ScopedLock sl(mLock); + std::vector path; + SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, path); + if(leaf == SHAMapLeafNode::pointer()) return false; + SHAMapItem::pointer item=leaf->findItem(id); + return item == SHAMapItem::pointer(); +} + +bool SHAMap::delItem(const uint256& id) +{ // delete the item with this ID + ScopedLock sl(mLock); + std::vector path; + SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, path); + if(leaf == SHAMapLeafNode::pointer()) return false; + if(!leaf->delItem(id)) return false; + dirtyUp(id, path); +} + +bool SHAMap::addItem(const SHAMapItem& item) +{ // add the specified item + ScopedLock sl(mLock); + std::vector path; + SHAMapLeafNode::pointer leaf=walkToLeaf(item.getTag(), true, path); + if(leaf == SHAMapLeafNode::pointer()) return false; + if(!leaf->addUpdateItem(item)) return false; + dirtyUp(item.getTag(), path); +} + +void TestSHAMap() +{ + uint256 h1, h2, h3, h4, h5; + h1.SetHex("436ccbac3347baa1f1e53baeef1f43334da88f1f6d70d963b833afd6dfa289fe"); + h2.SetHex("b92891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7"); + h3.SetHex("b92891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca8"); + h4.SetHex("a92891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7"); + h5.SetHex("092891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7"); + + SHAMap sMap(32); + SHAMapItem i1(h1), i2(h2), i3(h3), i4(h4), i5(h5); + + sMap.dump(); +} diff --git a/SHAMap.h b/SHAMap.h index 27458b2db1..332a612cb3 100644 --- a/SHAMap.h +++ b/SHAMap.h @@ -2,6 +2,8 @@ #define __SHAMAP__ #include +#include +#include #include #include @@ -37,8 +39,9 @@ public: bool isRoot() const { return mDepth==0; } bool isLeaf() const { return mDepth==leafDepth; } + bool isChildLeaf() const { return mDepth<(leafDepth-1); } bool isInner() const { return !isRoot() && !isLeaf(); } - virtual bool IsPopulated(void) const { return false; } + virtual bool isPopulated(void) const { return false; } SHAMapNode getParentNodeID() { return SHAMapNode(mDepth-1, mNodeID); } SHAMapNode getChildNodeID(int m); @@ -51,6 +54,8 @@ public: bool operator<=(const SHAMapNode &) const; bool operator>=(const SHAMapNode &) const; + virtual void dump(void); + static void ClassInit(); static uint256 getNodeID(int depth, const uint256 &hash); }; @@ -66,6 +71,7 @@ private: std::vector mData; public: + SHAMapItem(const uint256 &tag); // tag is data SHAMapItem(const uint256 &tag, const std::vector& data); SHAMapItem(const std::vector& data); // tag by hash @@ -87,6 +93,7 @@ public: bool operator!=(const uint256& i) const { return mTag!=i; } bool operator<=(const uint256& i) const { return mTag<=i; } bool operator>=(const uint256& i) const { return mTag>=i; } + virtual void dump(void); }; @@ -101,7 +108,7 @@ private: uint256 mHash; std::list mItems; - void updateHash(); + bool updateHash(); protected: bool addUpdateItem(const SHAMapItem&); @@ -111,15 +118,19 @@ protected: public: SHAMapLeafNode(const SHAMapNode& nodeID); - virtual bool IsPopulated(void) const { return true; } + virtual bool isPopulated(void) const { return true; } const uint256& GetNodeHash() const { return mHash; } bool isEmpty() const { return mItems.empty(); } int getItemCount() const { return mItems.size(); } - const uint256& getHash(int m) const; bool hasItem(const uint256 &item) const; SHAMapItem::pointer findItem(const uint256 &tag); + SHAMapItem::pointer firstItem(); + SHAMapItem::pointer lastItem(); + SHAMapItem::pointer nextItem(SHAMapItem::pointer); + + virtual void dump(void); }; @@ -134,20 +145,24 @@ private: uint256 mHash; uint256 mHashes[32]; - void updateHash(); + bool updateHash(); protected: - void setChildHash(int m, const uint256 &hash); + bool setChildHash(int m, const uint256 &hash); public: SHAMapInnerNode(int Depth, const uint256 &NodeID); + SHAMapInnerNode(const SHAMapNode &id, const std::vector &contents); virtual bool isPopulated(void) const { return true; } + + bool isEmptyBranch(int m) const { return mHashes[m]==0; } const uint256& getNodeHash() const { return mHash; } const uint256& getChildHash(int m) const; bool isEmpty() const; -}; + virtual void dump(void); +}; class SHAMap @@ -156,13 +171,31 @@ public: typedef boost::shared_ptr pointer; private: + int mLeafDataSize; mutable boost::recursive_mutex mLock; - std::map mLeafByID; - std::map mInnerNodeByID; - boost::bimap NodeHash; + std::map mLeafByID; + std::map mInnerNodeByID; + std::map mDirtyLeafNodes; + std::map mDirtyInnerNodes; + SHAMapInnerNode::pointer root; + +protected: + void dirtyUp(const uint256& id, const std::vector& path); + + SHAMapLeafNode::pointer createLeaf(const SHAMapInnerNode& lowestParent, const uint256 &id); + SHAMapLeafNode::pointer checkCacheLeaf(const SHAMapNode &); + SHAMapLeafNode::pointer walkToLeaf(const uint256& id, bool create, + std::vector& path); + + SHAMapLeafNode::pointer getLeaf(const SHAMapNode &id, const uint256& hash); + SHAMapInnerNode::pointer getInner(const SHAMapNode &id, const uint256& hash); + + SHAMapItem::pointer firstBelow(SHAMapInnerNode::pointer); + SHAMapItem::pointer lastBelow(SHAMapInnerNode::pointer); + public: - SHAMap(); + SHAMap(int leafDataSize); // hold the map stable across operations ScopedLock Lock() const { return ScopedLock(mLock); } @@ -182,17 +215,16 @@ public: bool addRawNode(const SHAMapNode& nodeID, std::vector rawNode); // normal hash access functions - bool hasHash(const uint256 &hash); - bool addHash(const uint256 &hash); - bool delHash(const uint256 &hash); - bool firstHash(uint256 &hash); - bool lastHash(uint256 &hash); - bool nextHash(uint256 &hash); - bool prevHash(uint256 &hash); + bool hasItem(const uint256& id); + bool delItem(const uint256& id); + bool addItem(const SHAMapItem& item); + SHAMapItem::pointer getItem(const uint256 &id); - // direct mapping - bool nodeToHash(const SHAMapNode &node, uint256 &hash); - bool hashToNode(const uint256& hash, SHAMapNode &node); + // traverse functions + SHAMapItem::pointer firstItem(); + SHAMapItem::pointer lastItem(); + SHAMapItem::pointer nextItem(const SHAMapItem &); + SHAMapItem::pointer prevItem(const SHAMapItem &); // comparison/sync functions void getMissingNodes(std::vector &nodeHashes, int max); @@ -201,12 +233,15 @@ public: bool getNodeFat(const uint256 &hash, std::vector &nodeHashes, int max); bool addKnownNode(const std::vector& rawNode); + int flushDirty(int maxNodes); + // overloads for backed maps - virtual bool fetchNode(const uint256 &hash, std::vector& rawNode); - virtual bool writeNode(const uint256 &hash, const std::vector& rawNode); - virtual bool haveObject(const uint256 &hash); + virtual bool fetchNode(const uint256 &hash, const SHAMapNode &id, std::vector& rawNode); + virtual bool writeNode(const uint256 &hash, const SHAMapNode &id, const std::vector& rawNode); + virtual void badNode(const uint256 &hash, const SHAMapNode &id); static bool TestSHAMap(); + virtual void dump(void); }; #endif diff --git a/SHAMapNodes.cpp b/SHAMapNodes.cpp new file mode 100644 index 0000000000..02d762a372 --- /dev/null +++ b/SHAMapNodes.cpp @@ -0,0 +1,192 @@ +#include "Serializer.h" +#include "BitcoinUtil.h" +#include "SHAMap.h" + +#include + +uint256 SHAMapNode::smMasks[11]; + +bool SHAMapNode::operator<(const SHAMapNode &s) const +{ + if(s.mDepthmDepth) return false; + return mNodeID(const SHAMapNode &s) const +{ + if(s.mDepthmDepth) return true; + return mNodeID>s.mNodeID; +} + +bool SHAMapNode::operator<=(const SHAMapNode &s) const +{ + if(s.mDepthmDepth) return false; + return mNodeID<=s.mNodeID; +} + +bool SHAMapNode::operator>=(const SHAMapNode &s) const +{ + if(s.mDepthmDepth) return true; + return mNodeID>=s.mNodeID; +} + +bool SHAMapNode::operator==(const SHAMapNode &s) const +{ + return (s.mDepth==mDepth) && (s.mNodeID==mNodeID); +} + +bool SHAMapNode::operator!=(const SHAMapNode &s) const +{ + return (s.mDepth!=mDepth) || (s.mNodeID!=mNodeID); +} + +void SHAMapNode::ClassInit() +{ // set up the depth masks + int i; + char HexBuf[65]; + + for(i=0; i<64; i++) HexBuf[i]='0'; + HexBuf[64]=0; + for(i=0; i=0 && depth<=leafDepth); + return hash & smMasks[depth]; +} + +SHAMapNode::SHAMapNode(int depth, const uint256 &hash) +{ // canonicalize the hash to a node ID for this depth + assert(depth>=0 && depth>=(mDepth*8); + + return SHAMapNode(mDepth+1, mNodeID | branch); +} + +int SHAMapNode::selectBranch(const uint256 &hash) +{ + if(isLeaf()) // no nodes under this node + return -1; + if((hash&smMasks[mDepth])!=mNodeID) + return -1; // does not go under this node + + uint256 selector=hash&smMasks[mDepth+1]; + int branch=*(selector.begin()+mDepth); + + assert(branch>=0 && branch<32); + return branch; +} + +SHAMapLeafNode::SHAMapLeafNode(const SHAMapNode& nodeID) : SHAMapNode(nodeID), mHash(0) +{ + ; +} + +bool SHAMapLeafNode::hasItem(const uint256& item) const +{ + BOOST_FOREACH(const SHAMapItem& nodeItem, mItems) + if(nodeItem==item) return true; + return false; +} + +bool SHAMapLeafNode::addUpdateItem(const SHAMapItem& item) +{ // The node will almost never have more than one item in it + std::list::iterator it; + for(it=mItems.begin(); it!=mItems.end(); it++) + { + if(*it==item) + { + if(it->peekData()==item.peekData()) + return false; // no change + it->updateData(item.peekData()); + return updateHash(); + } + if((*it)>item) + { + mItems.insert(it, item); + } + } + mItems.push_back(item); + return updateHash(); +} + +bool SHAMapLeafNode::delItem(const uint256& tag) +{ + std::list::iterator it; + for(it=mItems.begin(); it!=mItems.end(); it++) + { + if(*it==tag) + { + mItems.erase(it); + return updateHash(); + } + } + return false; +} + +bool SHAMapLeafNode::updateHash(void) +{ + uint256 nh; + if(mItems.size()!=0) nh=0; + { + Serializer s; + BOOST_FOREACH(const SHAMapItem &mi, mItems) + s.addRaw(mi.peekData()); + nh=s.getSHA512Half(); + } + if(nh==mHash) return false; + mHash=nh; + return true; +} + +bool SHAMapInnerNode::setChildHash(int m, const uint256 &hash) +{ + assert( (m>=0) && (m<32) ); + if(mHashes[m]==hash) + return false; + mHashes[m]=hash; + return updateHash(); +} + +const uint256& SHAMapInnerNode::getChildHash(int m) const +{ + assert( (m>=0) && (m<32) ); + return mHashes[m]; +} + +bool SHAMapInnerNode::updateHash() +{ + int nc=0; + Serializer s; + for(int i=0; i<32; i++) + { + if(mHashes[i]!=0) nc++; + s.add256(mHashes[i]); + } + uint256 nh=0; + if(nc!=0) + nh=s.getSHA512Half(); + if(mHash==nh) return false; + mHash=nh; + return true; +} +