Debug and unit test code. Copy on write logic. Bugfixes.

This commit is contained in:
JoelKatz
2011-11-25 00:03:33 -08:00
parent 83b2f8b73a
commit 0df9ec91fa
3 changed files with 192 additions and 63 deletions

View File

@@ -4,16 +4,22 @@
#include <boost/foreach.hpp>
SHAMap::SHAMap()
SHAMap::SHAMap() : mSeq(0)
{
;
root=SHAMapInnerNode::pointer(new SHAMapInnerNode(SHAMapNode(SHAMapNode::rootDepth, uint256()), mSeq));
}
void SHAMap::dirtyUp(const uint256& id)
{ // walk the tree up from through the inner nodes to the root
// update linking hashes and add nodes to dirty list
#ifdef DEBUG
std::cerr << "dirtyUp(" << id.GetHex() << ")" << std::endl;
#endif
SHAMapLeafNode::pointer leaf=mLeafByID[SHAMapNode(SHAMapNode::leafDepth, id)];
if(!leaf) throw SHAMapException(MissingNode);
#ifdef DEBUG
std::cerr << "found leaf " << leaf->getString() << std::endl;
#endif
uint256 hVal=leaf->getNodeHash();
if(mDirtyLeafNodes) (*mDirtyLeafNodes)[*leaf]=leaf;
@@ -21,6 +27,9 @@ void SHAMap::dirtyUp(const uint256& id)
for(int depth=SHAMapNode::leafDepth-1; depth>=0; depth--)
{ // walk up the tree to the root updating nodes
#ifdef DEBUG
std::cerr << "walk up to " << depth << ", " << leaf->getNodeID().GetHex() << std::endl;
#endif
SHAMapInnerNode::pointer node=mInnerNodeByID[SHAMapNode(depth, leaf->getNodeID())];
if(!node) throw SHAMapException(MissingNode);
if(!node->setChildHash(node->selectBranch(id), hVal)) return;
@@ -37,11 +46,17 @@ SHAMapLeafNode::pointer SHAMap::checkCacheLeaf(const SHAMapNode& iNode)
}
SHAMapLeafNode::pointer SHAMap::walkToLeaf(const uint256& id, bool create)
SHAMapLeafNode::pointer SHAMap::walkToLeaf(const uint256& id, bool create, bool modify)
{ // walk down to the leaf that would contain this ID
// is leaf node in cache
#ifdef DEBUG
std::cerr << "walkToLeaf(" << id.GetHex() << ")";
if(create) std::cerr << " create";
if(modify) std::cerr << " modify";
std::cerr << std::endl;
#endif
SHAMapLeafNode::pointer ln=checkCacheLeaf(SHAMapNode(SHAMapNode::leafDepth, id));
if(ln) return ln;
if(ln) return returnLeaf(ln, modify);
// walk tree to leaf
SHAMapInnerNode::pointer inNode=root;
@@ -53,66 +68,99 @@ SHAMapLeafNode::pointer SHAMap::walkToLeaf(const uint256& id, bool create)
throw SHAMapException(InvalidNode);
if(inNode->isEmptyBranch(branch))
{ // no nodes below this one
#ifdef DEBUG
std::cerr << "No nodes below level " << i << std::endl;
#endif
if(!create) return SHAMapLeafNode::pointer();
return createLeaf(*inNode, id);
}
if(i!=(SHAMapNode::leafDepth)-1)
{ // child is another inner node
inNode=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch));
inNode=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), modify);
if(inNode==NULL) throw SHAMapException(InvalidNode);
}
else // child is leaf node
{
ln=getLeaf(inNode->getChildNodeID(branch), inNode->getChildHash(branch));
ln=getLeaf(inNode->getChildNodeID(branch), inNode->getChildHash(branch), modify);
if(ln==NULL)
{
if(!create) return SHAMapLeafNode::pointer();
if(!create) return SHAMapLeafNode::pointer();
return createLeaf(*inNode, id);
}
}
}
return ln;
return returnLeaf(ln, modify);
}
SHAMapLeafNode::pointer SHAMap::getLeaf(const SHAMapNode &id, const uint256& hash)
SHAMapLeafNode::pointer SHAMap::getLeaf(const SHAMapNode &id, const uint256& hash, bool modify)
{ // retrieve a leaf whose node hash is known
#ifdef DEBUG
std::cerr << "getLeaf(" << id.getString() << ", " << hash.GetHex() << ")";
if(modify) std::cerr << " modify";
std::cerr << std::endl;
#endif
assert(!!hash);
if(!id.isLeaf()) return SHAMapLeafNode::pointer();
SHAMapLeafNode::pointer leaf=mLeafByID[id]; // is the leaf in memory
if(leaf) return leaf;
if(leaf) return returnLeaf(leaf, modify);
std::vector<SHAMapItem::pointer> leafData; // is it in backing store
if(!fetchLeafNode(hash, id, leafData))
throw SHAMapException(MissingNode);
throw SHAMapException(MissingNode);
leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(id));
leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(id, mSeq));
BOOST_FOREACH(SHAMapItem::pointer& item, leafData)
leaf->addUpdateItem(item);
leaf->updateHash();
if(leaf->getNodeHash()!=hash) throw SHAMapException(InvalidNode);
mLeafByID[id]=leaf;
return leaf;
}
SHAMapInnerNode::pointer SHAMap::getInner(const SHAMapNode &id, const uint256& hash)
SHAMapInnerNode::pointer SHAMap::getInner(const SHAMapNode &id, const uint256& hash, bool modify)
{ // retrieve an inner node whose node hash is known
#ifdef DEBUG
std::cerr << "getInner(" << id.getString() << ", " << hash.GetHex() << ")";
if(modify) std::cerr << " modify";
std::cerr << std::endl;
#endif
SHAMapInnerNode::pointer node=mInnerNodeByID[id];
if(node) return node;
if(node) return returnNode(node, modify);
std::vector<unsigned char> rawNode;
if(!fetchInnerNode(hash, id, rawNode)) throw SHAMapException(MissingNode);
node=SHAMapInnerNode::pointer(new SHAMapInnerNode(id, rawNode));
node=SHAMapInnerNode::pointer(new SHAMapInnerNode(id, rawNode, mSeq));
if(node->getNodeHash()!=hash) throw SHAMapException(InvalidNode);
mInnerNodeByID[id]=node;
return node;
}
SHAMapLeafNode::pointer SHAMap::returnLeaf(SHAMapLeafNode::pointer leaf, bool modify)
{ // make sure the leaf is suitable for the intended operation (copy on write)
if(leaf && modify && (leaf->getSeq()!=mSeq))
{
leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(*leaf, mSeq));
mLeafByID[*leaf]=leaf;
if(mDirtyLeafNodes) (*mDirtyLeafNodes)[*leaf]=leaf;
}
return leaf;
}
SHAMapInnerNode::pointer SHAMap::returnNode(SHAMapInnerNode::pointer node, bool modify)
{ // make sure the node is suitable for the intended operation (copy on write)
if(node && modify && (node->getSeq()!=mSeq))
{
node=SHAMapInnerNode::pointer(new SHAMapInnerNode(*node, mSeq));
mInnerNodeByID[*node]=node;
if(mDirtyInnerNodes) (*mDirtyInnerNodes)[*node]=node;
}
return node;
}
SHAMapItem::SHAMapItem(const uint256& tag, const std::vector<unsigned char>& data)
: mTag(tag), mData(data)
{ ; }
@@ -133,20 +181,24 @@ SHAMapItem::pointer SHAMap::peekLastItem()
return lastBelow(root);
}
SHAMapItem::pointer SHAMap::firstBelow(SHAMapInnerNode::pointer Node)
SHAMapItem::pointer SHAMap::firstBelow(SHAMapInnerNode::pointer node)
{
#ifdef DEBUG
std::cerr << "firstBelow(" << node->getString() << ")" << std::endl;
#endif
const uint256 zero;
int i;
while(Node->isChildLeaf())
while(!node->isChildLeaf())
{
for(i=0; i<32; i++)
{
uint256 cHash(Node->getChildHash(i));
uint256 cHash(node->getChildHash(i));
if(cHash!=zero)
{
Node=getInner(Node->getChildNodeID(i), cHash);
if(!Node) return SHAMapItem::pointer();
node=getInner(node->getChildNodeID(i), cHash, false);
if(!node) return SHAMapItem::pointer();
break;
}
}
@@ -155,10 +207,10 @@ SHAMapItem::pointer SHAMap::firstBelow(SHAMapInnerNode::pointer Node)
for(int i=0; i<32; i++)
{
uint256 cHash=Node->getChildHash(i);
uint256 cHash=node->getChildHash(i);
if(cHash!=zero)
{
SHAMapLeafNode::pointer mLeaf=getLeaf(Node->getChildNodeID(i), cHash);
SHAMapLeafNode::pointer mLeaf=getLeaf(node->getChildNodeID(i), cHash, false);
if(!mLeaf) return SHAMapItem::pointer();
return mLeaf->firstItem();
}
@@ -166,22 +218,22 @@ SHAMapItem::pointer SHAMap::firstBelow(SHAMapInnerNode::pointer Node)
return SHAMapItem::pointer();
}
SHAMapItem::pointer SHAMap::lastBelow(SHAMapInnerNode::pointer Node)
SHAMapItem::pointer SHAMap::lastBelow(SHAMapInnerNode::pointer node)
{
ScopedLock sl(mLock);
const uint256 zero;
int i;
while(Node->isChildLeaf())
while(!node->isChildLeaf())
{
for(i=31; i>=0; i--)
{
uint256 cHash(Node->getChildHash(i));
uint256 cHash(node->getChildHash(i));
if(cHash!=0)
{
Node=getInner(Node->getChildNodeID(i), cHash);
if(!Node) return SHAMapItem::pointer();
node=getInner(node->getChildNodeID(i), cHash, false);
if(!node) return SHAMapItem::pointer();
break;
}
}
@@ -189,10 +241,10 @@ SHAMapItem::pointer SHAMap::lastBelow(SHAMapInnerNode::pointer Node)
}
for(int i=31; i>=0; i--)
{
uint256 cHash=Node->getChildHash(i);
uint256 cHash=node->getChildHash(i);
if(cHash!=zero)
{
SHAMapLeafNode::pointer mLeaf=getLeaf(Node->getChildNodeID(i), cHash);
SHAMapLeafNode::pointer mLeaf=getLeaf(node->getChildNodeID(i), cHash, false);
if(!mLeaf) return SHAMapItem::pointer();
return mLeaf->lastItem();
}
@@ -203,7 +255,7 @@ SHAMapItem::pointer SHAMap::peekNextItem(const uint256& id)
{ // Get a pointer to the next item in the tree after a given item - item must be in tree
ScopedLock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(id, false);
SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false);
if(!leaf) return SHAMapItem::pointer();
// is there another item in this leaf? (there almost never will be)
@@ -223,7 +275,7 @@ SHAMapItem::pointer SHAMap::peekNextItem(const uint256& id)
if(nextNode.isLeaf())
{ // this is a terminal inner node
leaf=getLeaf(nextNode, nextHash);
leaf=getLeaf(nextNode, nextHash, false);
if(!leaf) throw SHAMapException(MissingNode);
next=leaf->firstItem();
if(!next) throw SHAMapException(InvalidNode);
@@ -231,7 +283,7 @@ SHAMapItem::pointer SHAMap::peekNextItem(const uint256& id)
}
// the next item is the first item below this node
SHAMapInnerNode::pointer inner=getInner(nextNode, nextHash);
SHAMapInnerNode::pointer inner=getInner(nextNode, nextHash, false);
if(!inner) throw SHAMapException(MissingNode);
next=firstBelow(inner);
if(!next) throw SHAMapException(InvalidNode);
@@ -250,22 +302,31 @@ SHAMapItem::pointer SHAMap::peekPrevItem(const uint256& id)
SHAMapLeafNode::pointer SHAMap::createLeaf(const SHAMapInnerNode& lowestParent, const uint256& id)
{
int depth=lowestParent.getDepth();
for(int depth=lowestParent.getDepth(); depth<SHAMapNode::leafDepth; depth++)
#ifdef DEBUG
std::cerr << "createLeaf(" << lowestParent.getString() << std::endl;
std::cerr << " for " << id.GetHex() << ")" << std::endl;
#endif
assert(!!id);
for(int depth=lowestParent.getDepth()+1; depth<SHAMapNode::leafDepth; depth++)
{
SHAMapInnerNode::pointer newNode(new SHAMapInnerNode(SHAMapNode(++depth, id)));
SHAMapInnerNode::pointer newNode(new SHAMapInnerNode(SHAMapNode(depth, id), mSeq));
#ifdef DEBUG
std::cerr << "create node " << newNode->getString() << std::endl;
#endif
mInnerNodeByID[*newNode]=newNode;
}
SHAMapLeafNode::pointer newLeaf(new SHAMapLeafNode(SHAMapNode(SHAMapNode::leafDepth, id)));
SHAMapLeafNode::pointer newLeaf(new SHAMapLeafNode(SHAMapNode(SHAMapNode::leafDepth, id), mSeq));
mLeafByID[*newLeaf]=newLeaf;
#ifdef DEBUG
std::cerr << "made leaf " << newLeaf->getString() << std::endl;
#endif
return newLeaf;
}
SHAMapItem::pointer SHAMap::peekItem(const uint256& id)
{
ScopedLock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(id, false);
SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false);
if(!leaf) return SHAMapItem::pointer();
return leaf->findItem(id);
}
@@ -273,7 +334,7 @@ SHAMapItem::pointer SHAMap::peekItem(const uint256& id)
bool SHAMap::hasItem(const uint256& id)
{ // does the tree have an item with this ID
ScopedLock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(id, false);
SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false);
if(!leaf) return false;
SHAMapItem::pointer item=leaf->findItem(id);
return (bool) item;
@@ -282,7 +343,7 @@ bool SHAMap::hasItem(const uint256& id)
bool SHAMap::delItem(const uint256& id)
{ // delete the item with this ID
ScopedLock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(id, false);
SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false);
if(!leaf) return false;
if(!leaf->delItem(id)) return false;
dirtyUp(id);
@@ -292,7 +353,7 @@ bool SHAMap::delItem(const uint256& id)
bool SHAMap::addGiveItem(const SHAMapItem::pointer item)
{ // add the specified item
ScopedLock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(item->getTag(), true);
SHAMapLeafNode::pointer leaf=walkToLeaf(item->getTag(), true, true);
if(!leaf) return false;
if(leaf->hasItem(item->getTag())) return false;
if(!leaf->addUpdateItem(item)) return false;
@@ -300,10 +361,15 @@ bool SHAMap::addGiveItem(const SHAMapItem::pointer item)
return true;
}
bool SHAMap::addItem(const SHAMapItem& i)
{
return addGiveItem(SHAMapItem::pointer(new SHAMapItem(i)));
}
bool SHAMap::updateGiveItem(SHAMapItem::pointer item)
{
ScopedLock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(item->getTag(), true);
SHAMapLeafNode::pointer leaf=walkToLeaf(item->getTag(), true, true);
if(!leaf) return false;
if(!leaf->addUpdateItem(item)) return false;
dirtyUp(item->getTag());
@@ -338,6 +404,14 @@ bool SHAMap::writeLeafNode(const uint256&, const SHAMapNode&, const std::vector<
void SHAMap::dump()
{
std::cerr << "SHAMap::dump" << std::endl;
SHAMapItem::pointer i=peekFirstItem();
while(i)
{
std::cerr << "Item: id=" << i->getTag().GetHex() << std::endl;
i=peekNextItem(i->getTag());
}
std::cerr << "SHAMap::dump done" << std::endl;
}
static std::vector<unsigned char>IntToVUC(int i)
@@ -358,6 +432,16 @@ bool SHAMap::TestSHAMap()
SHAMap sMap;
SHAMapItem i1(h1, IntToVUC(1)), i2(h2, IntToVUC(2)), i3(h3, IntToVUC(3)), i4(h4, IntToVUC(4)), i5(h5, IntToVUC(5));
#ifdef DEBUG
std::cerr << "addItem1" << std::endl;
#endif
sMap.addItem(i1);
#ifdef DEBUG
std::cerr << "addItem2" << std::endl;
#endif
sMap.addItem(i2);
sMap.dump();
sMap.dump();
}

View File

@@ -17,12 +17,11 @@ class SHAMap;
// The trees are designed for rapid synchronization and compression of differences
class SHAMapNode
class SHAMapNode
{ // Identifies a node in a SHA256 hash
public:
typedef boost::shared_ptr<SHAMapNode> pointer;
private:
static uint256 smMasks[11]; // AND with hash to get node id
@@ -56,7 +55,8 @@ public:
bool operator<=(const SHAMapNode&) const;
bool operator>=(const SHAMapNode&) const;
virtual void dump(void);
virtual std::string getString(void) const;
void dump(void);
static void ClassInit();
static uint256 getNodeID(int depth, const uint256& hash);
@@ -112,19 +112,27 @@ public:
private:
uint256 mHash;
std::list<SHAMapItem::pointer> mItems;
uint32 mSeq;
bool updateHash();
SHAMapLeafNode(const SHAMapLeafNode&); // no implementation
SHAMapLeafNode& operator=(const SHAMapLeafNode&); // no implementation
protected:
bool addUpdateItem(SHAMapItem::pointer);
bool delItem(const SHAMapItem::pointer i) { delItem(i->getTag()); }
bool delItem(const uint256& tag);
public:
SHAMapLeafNode(const SHAMapNode& nodeID);
SHAMapLeafNode(const SHAMapNode& nodeID, uint32 seq);
SHAMapLeafNode(const SHAMapLeafNode& node, uint32 seq);
virtual bool isPopulated(void) const { return true; }
uint32 getSeq(void) const { return mSeq; }
void setSeq(uint32 s) { mSeq=s; }
const uint256& getNodeHash() const { return mHash; }
bool isEmpty() const { return mItems.empty(); }
int getItemCount() const { return mItems.size(); }
@@ -150,15 +158,23 @@ public:
private:
uint256 mHash;
uint256 mHashes[32];
uint32 mSeq;
bool updateHash();
SHAMapInnerNode(const SHAMapInnerNode&); // no implementation
SHAMapInnerNode& operator=(const SHAMapInnerNode&); // no implementation
protected:
bool setChildHash(int m, const uint256& hash);
public:
SHAMapInnerNode(const SHAMapNode& id);
SHAMapInnerNode(const SHAMapNode& id, const std::vector<unsigned char>& contents);
SHAMapInnerNode(const SHAMapNode& id, uint32 seq);
SHAMapInnerNode(const SHAMapInnerNode& node, uint32 seq);
SHAMapInnerNode(const SHAMapNode& id, const std::vector<unsigned char>& contents, uint32 seq);
uint32 getSeq(void) const { return mSeq; }
void setSeq(uint32 s) { mSeq=s; }
virtual bool isPopulated(void) const { return true; }
@@ -183,7 +199,7 @@ public:
typedef boost::shared_ptr<SHAMap> pointer;
private:
int mLeafDataSize, mLeafDataOffset, mSeq;
uint32 mSeq;
mutable boost::recursive_mutex mLock;
std::map<SHAMapNode, SHAMapLeafNode::pointer> mLeafByID;
std::map<SHAMapNode, SHAMapInnerNode::pointer> mInnerNodeByID;
@@ -197,15 +213,19 @@ protected:
SHAMapLeafNode::pointer createLeaf(const SHAMapInnerNode& lowestParent, const uint256& id);
SHAMapLeafNode::pointer checkCacheLeaf(const SHAMapNode &);
SHAMapLeafNode::pointer walkToLeaf(const uint256& id, bool create);
SHAMapLeafNode::pointer walkToLeaf(const uint256& id, bool create, bool modify);
SHAMapLeafNode::pointer getLeaf(const SHAMapNode& id, const uint256& hash);
SHAMapInnerNode::pointer getInner(const SHAMapNode& id, const uint256& hash);
SHAMapLeafNode::pointer getLeaf(const SHAMapNode& id, const uint256& hash, bool modify);
SHAMapLeafNode::pointer returnLeaf(SHAMapLeafNode::pointer leaf, bool modify);
SHAMapInnerNode::pointer getInner(const SHAMapNode& id, const uint256& hash, bool modify);
SHAMapInnerNode::pointer returnNode(SHAMapInnerNode::pointer node, bool modify);
SHAMapItem::pointer firstBelow(SHAMapInnerNode::pointer);
SHAMapItem::pointer lastBelow(SHAMapInnerNode::pointer);
public:
// build new map
SHAMap();
// hold the map stable across operations

View File

@@ -3,6 +3,17 @@
#include "SHAMap.h"
#include <boost/foreach.hpp>
#include <boost/lexical_cast.hpp>
std::string SHAMapNode::getString() const
{
std::string ret="NodeID(";
ret+=boost::lexical_cast<std::string>(mDepth);
ret+=",";
ret+=mNodeID.GetHex();
ret+=")";
return ret;
}
uint256 SHAMapNode::smMasks[11];
@@ -51,7 +62,7 @@ void SHAMapNode::ClassInit()
for(i=0; i<64; i++) HexBuf[i]='0';
HexBuf[64]=0;
for(i=0; i<leafDepth; i++)
for(i=0; i<=leafDepth; i++)
{
smMasks[i].SetHex(HexBuf);
HexBuf[2*i]='1';
@@ -98,12 +109,18 @@ int SHAMapNode::selectBranch(const uint256 &hash)
void SHAMapNode::dump()
{
std::cerr << "MapNode(" << mNodeID.GetHex() << ", " << mDepth << ")" << std::endl;
std::cerr << getString() << std::endl;
}
SHAMapLeafNode::SHAMapLeafNode(const SHAMapNode& nodeID) : SHAMapNode(nodeID), mHash(0)
SHAMapLeafNode::SHAMapLeafNode(const SHAMapNode& nodeID, uint32 seq) : SHAMapNode(nodeID), mHash(0), mSeq(seq)
{
assert(nodeID.getDepth()==SHAMapNode::leafDepth);
assert(nodeID.isLeaf());
}
SHAMapLeafNode::SHAMapLeafNode(const SHAMapLeafNode& node, uint32 seq) : SHAMapNode(node),
mHash(node.mHash), mItems(node.mItems), mSeq(seq)
{
assert(node.isLeaf());
}
bool SHAMapLeafNode::hasItem(const uint256& item) const
@@ -202,17 +219,25 @@ void SHAMapLeafNode::dump()
std::cerr << " " << mItems.size() << " items" << std::endl;
}
SHAMapInnerNode::SHAMapInnerNode(const SHAMapNode& id) : SHAMapNode(id)
{
SHAMapInnerNode::SHAMapInnerNode(const SHAMapNode& id, uint32 seq) : SHAMapNode(id), mSeq(seq)
{ // can be root
assert(id.getDepth()<SHAMapNode::leafDepth);
}
SHAMapInnerNode::SHAMapInnerNode(const SHAMapNode& id, const std::vector<unsigned char>& contents)
: SHAMapNode(id)
SHAMapInnerNode::SHAMapInnerNode(const SHAMapNode& id, const std::vector<unsigned char>& contents, uint32 seq)
: SHAMapNode(id), mSeq(seq)
{
assert(!id.isLeaf());
Serializer s(contents);
for(int i=0; i<32; i++)
mHashes[i]=s.get256(i*32);
mHashes[i]=s.get256(i*32);
}
SHAMapInnerNode::SHAMapInnerNode(const SHAMapInnerNode& node, uint32 seq) : SHAMapNode(node), mHash(node.mHash),
mSeq(seq)
{
assert(!node.isLeaf());
memcpy(mHashes, node.mHashes, sizeof(mHashes));
}
bool SHAMapInnerNode::setChildHash(int m, const uint256 &hash)