Files
rippled/SHAMap.cpp
2011-12-20 22:01:06 -08:00

576 lines
16 KiB
C++

#include "Serializer.h"
#include "BitcoinUtil.h"
#include "SHAMap.h"
#include <boost/foreach.hpp>
#include <boost/lexical_cast.hpp>
SHAMap::SHAMap() : mSeq(0)
{
root=SHAMapInnerNode::pointer(new SHAMapInnerNode(SHAMapNode(SHAMapNode::rootDepth, uint256()), mSeq));
mInnerNodeByID[*root]=root;
}
void SHAMap::dirtyUp(const uint256& id)
{ // walk the tree up from through the inner nodes to the root
// update linking hashes and add nodes to dirty list
#ifdef DEBUG
std::cerr << "dirtyUp(" << id.GetHex() << ")" << std::endl;
#endif
SHAMapLeafNode::pointer leaf=mLeafByID[SHAMapNode(SHAMapNode::leafDepth, id)];
if(!leaf) throw SHAMapException(MissingNode);
uint256 hVal=leaf->getNodeHash();
if(mDirtyLeafNodes) mDirtyLeafNodes->insert(std::make_pair(SHAMapNode(*leaf), leaf));
if(!hVal)
{
#ifdef ST_DEBUG
std::cerr << " erasingL " << leaf->getString() << std::endl;
#endif
mLeafByID.erase(*leaf);
}
for(int depth=SHAMapNode::leafDepth-1; depth>=0; depth--)
{ // walk up the tree to the root updating nodes
SHAMapInnerNode::pointer node=mInnerNodeByID[SHAMapNode(depth, leaf->getNodeID())];
if(!node) throw SHAMapException(MissingNode);
if(!node->setChildHash(node->selectBranch(id), hVal))
{
#ifdef ST_DEBUG
std::cerr << " no change@ " << node->getString() << std::endl;
#endif
return;
}
#ifdef ST_DEBUG
std::cerr << "Dirty " << node->getString() << std::endl;
#endif
if(mDirtyInnerNodes) mDirtyInnerNodes->insert(std::make_pair(SHAMapNode(*node), node));
hVal=node->getNodeHash();
if(!hVal)
{
#ifdef ST_DEBUG
std::cerr << " erasingN " << node->getString() << std::endl;
#endif
mInnerNodeByID.erase(*node);
}
}
}
SHAMapLeafNode::pointer SHAMap::checkCacheLeaf(const SHAMapNode& iNode)
{
assert(iNode.isLeaf());
SHAMapLeafNode::pointer leaf=mLeafByID[iNode];
#ifdef ST_DEBUG
if(!leaf) std::cerr << "Leaf(" << iNode.getString() << ") not in cache" << std::endl;
else std::cerr << "Leaf(" << iNode.getString() << ") found in cache" << std::endl;
#endif
return leaf;
}
SHAMapLeafNode::pointer SHAMap::walkToLeaf(const uint256& id, bool create, bool modify)
{ // walk down to the leaf that would contain this ID
// is leaf node in cache
#ifdef DEBUG
std::cerr << "walkToLeaf(" << id.GetHex() << ")";
if(create) std::cerr << " create";
if(modify) std::cerr << " modify";
std::cerr << std::endl;
#endif
SHAMapLeafNode::pointer ln=checkCacheLeaf(SHAMapNode(SHAMapNode::leafDepth, id));
if(ln) return returnLeaf(ln, modify);
// walk tree to leaf
SHAMapInnerNode::pointer inNode=root;
for(int i=0; i<SHAMapNode::leafDepth; i++)
{
int branch=inNode->selectBranch(id);
if(branch<0) // somehow we got on the wrong branch
throw SHAMapException(InvalidNode);
if(inNode->isEmptyBranch(branch))
{ // no nodes below this one
#ifdef DEBUG
std::cerr << "No nodes below level " << i << ", branch " << branch << std::endl;
std::cerr << " terminal node is " << inNode->getString() << std::endl;
#endif
if(!create) return SHAMapLeafNode::pointer();
return createLeaf(*inNode, id);
}
if(inNode->isChildLeaf())
{ // child is leaf node
ln=getLeaf(inNode->getChildNodeID(branch), inNode->getChildHash(branch), modify);
if(ln==NULL)
{
if(!create) return SHAMapLeafNode::pointer();
return createLeaf(*inNode, id);
}
}
else
{ // child is another inner node
inNode=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), modify);
if(inNode==NULL) throw SHAMapException(InvalidNode);
}
}
assert(ln || !create);
return returnLeaf(ln, modify);
}
SHAMapLeafNode::pointer SHAMap::getLeaf(const SHAMapNode& id, const uint256& hash, bool modify)
{ // retrieve a leaf whose node hash is known
assert(!!hash);
if(!id.isLeaf()) return SHAMapLeafNode::pointer();
SHAMapLeafNode::pointer leaf=mLeafByID[id]; // is the leaf in memory
if(leaf) return returnLeaf(leaf, modify);
std::vector<unsigned char> leafData;
if(!fetchNode(hash, leafData)) throw SHAMapException(MissingNode);
leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(id, leafData, mSeq));
if(leaf->getNodeHash()!=hash) throw SHAMapException(InvalidNode);
mLeafByID[id]=leaf;
return leaf;
}
SHAMapInnerNode::pointer SHAMap::getInner(const SHAMapNode& id, const uint256& hash, bool modify)
{ // retrieve an inner node whose node hash is known
SHAMapInnerNode::pointer node=mInnerNodeByID[id];
if(node) return returnNode(node, modify);
std::vector<unsigned char> rawNode;
if(!fetchNode(hash, rawNode)) throw SHAMapException(MissingNode);
node=SHAMapInnerNode::pointer(new SHAMapInnerNode(id, rawNode, mSeq));
if(node->getNodeHash()!=hash) throw SHAMapException(InvalidNode);
mInnerNodeByID.insert(std::make_pair(id, node));
if(id.getDepth()==0) root=node;
return node;
}
SHAMapLeafNode::pointer SHAMap::returnLeaf(SHAMapLeafNode::pointer leaf, bool modify)
{ // make sure the leaf is suitable for the intended operation (copy on write)
if(leaf && modify && (leaf->getSeq()!=mSeq))
{
leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(*leaf, mSeq));
mLeafByID[*leaf]=leaf;
if(mDirtyLeafNodes) mDirtyLeafNodes->insert(std::make_pair(SHAMapNode(*leaf), leaf));
}
return leaf;
}
SHAMapInnerNode::pointer SHAMap::returnNode(SHAMapInnerNode::pointer node, bool modify)
{ // make sure the node is suitable for the intended operation (copy on write)
if(node && modify && (node->getSeq()!=mSeq))
{
#ifdef ST_DEBUG
std::cerr << "Node(" << node->getString() << ") bumpseq" << std::endl;
#endif
node=SHAMapInnerNode::pointer(new SHAMapInnerNode(*node, mSeq));
mInnerNodeByID.insert(std::make_pair(SHAMapNode(*node), node));
if(mDirtyInnerNodes) mDirtyInnerNodes->insert(std::make_pair(SHAMapNode(*node), node));
}
return node;
}
SHAMapItem::SHAMapItem(const uint256& tag, const std::vector<unsigned char>& data)
: mTag(tag), mData(data)
{ ; }
SHAMapItem::SHAMapItem(const uint160& tag, const std::vector<unsigned char>& data)
: mTag(tag.to256()), mData(data)
{ ; }
SHAMapItem::pointer SHAMap::peekFirstItem()
{
boost::recursive_mutex::scoped_lock sl(mLock);
return firstBelow(root);
}
SHAMapItem::pointer SHAMap::peekLastItem()
{
boost::recursive_mutex::scoped_lock sl(mLock);
return lastBelow(root);
}
SHAMapItem::pointer SHAMap::firstBelow(SHAMapInnerNode::pointer node)
{
#ifdef DEBUG
std::cerr << "firstBelow(" << node->getString() << ")" << std::endl;
#endif
int i;
while(!node->isChildLeaf())
{
for(i=0; i<32; i++)
if(!node->isEmptyBranch(i))
{
node=getInner(node->getChildNodeID(i), node->getChildHash(i), false);
if(!node) throw SHAMapException(MissingNode);
break;
}
if(i==32)
{
#ifdef ST_DEBUG
std::cerr << " node:" << node->getString() << " has no children" << std::endl;
#endif
return SHAMapItem::pointer();
}
}
assert(node->isChildLeaf());
for(int i=0; i<32; i++)
if(!node->isEmptyBranch(i))
{
SHAMapLeafNode::pointer mLeaf=getLeaf(node->getChildNodeID(i), node->getChildHash(i), false);
if(!mLeaf) throw SHAMapException(MissingNode);
SHAMapItem::pointer item=mLeaf->firstItem();
if(!item) throw SHAMapException(InvalidNode);
return item;
}
#ifdef ST_DEBUG
std::cerr << " node:" << node->getString() << " has no children" << std::endl;
#endif
return SHAMapItem::pointer();
}
SHAMapItem::pointer SHAMap::lastBelow(SHAMapInnerNode::pointer node)
{
boost::recursive_mutex::scoped_lock sl(mLock);
const uint256 zero;
int i;
while(!node->isChildLeaf())
{
for(i=31; i>=0; i--)
{
uint256 cHash(node->getChildHash(i));
if(cHash!=0)
{
node=getInner(node->getChildNodeID(i), cHash, false);
if(!node) return SHAMapItem::pointer();
break;
}
}
if(i<0) return SHAMapItem::pointer();
}
for(int i=31; i>=0; i--)
{
uint256 cHash=node->getChildHash(i);
if(cHash!=zero)
{
SHAMapLeafNode::pointer mLeaf=getLeaf(node->getChildNodeID(i), cHash, false);
if(!mLeaf) return SHAMapItem::pointer();
return mLeaf->lastItem();
}
}
return SHAMapItem::pointer();
}
SHAMapItem::pointer SHAMap::peekNextItem(const uint256& id)
{ // Get a pointer to the next item in the tree after a given item - item must be in tree
boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false);
if(!leaf)
{
#ifdef DEBUG
std::cerr << "peekNextItem: current not found" << std::endl;
#endif
return SHAMapItem::pointer();
}
// is there another item in this leaf? (there almost never will be)
SHAMapItem::pointer next=leaf->nextItem(id);
if(next) return next;
for(int depth=SHAMapNode::leafDepth-1; depth>=0; depth--)
{ // walk up the tree until we find a node with a subsequent child
SHAMapInnerNode::pointer node=mInnerNodeByID[SHAMapNode(depth, id)];
if(!node)
{
#ifdef DEBUG
std::cerr << "InnerNode missing: " << SHAMapNode(depth,id).getString() << std::endl;
#endif
throw SHAMapException(MissingNode);
}
#ifdef ST_DEBUG
std::cerr << " UpTo " << node->getString() << std::endl;
#endif
for(int i=node->selectBranch(id)+1; i<32; i++)
if(!!node->getChildHash(i))
{ // node has a subsequent child
SHAMapNode nextNode(node->getChildNodeID(i));
const uint256& nextHash(node->getChildHash(i));
if(nextNode.isLeaf())
{ // this is a terminal inner node
leaf=getLeaf(nextNode, nextHash, false);
if(!leaf) throw SHAMapException(MissingNode);
next=leaf->firstItem();
if(!next) throw SHAMapException(InvalidNode);
return next;
}
// the next item is the first item below this node
SHAMapInnerNode::pointer inner=getInner(nextNode, nextHash, false);
if(!inner) throw SHAMapException(MissingNode);
next=firstBelow(inner);
if(!next) throw SHAMapException(InvalidNode);
return next;
}
}
#ifdef ST_DEBUG
std::cerr << " peekNext off end" << std::endl;
#endif
// must be last item
return SHAMapItem::pointer();
}
SHAMapItem::pointer SHAMap::peekPrevItem(const uint256& id)
{
boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false);
if(!leaf) return SHAMapItem::pointer();
// is there another item in this leaf? (there almost never will be)
SHAMapItem::pointer prev=leaf->prevItem(id);
if(prev) return prev;
for(int depth=SHAMapNode::leafDepth-1; depth>=0; depth--)
{ // walk up the tree until we find a node with a previous child
SHAMapInnerNode::pointer node=mInnerNodeByID[SHAMapNode(depth, id)];
if(!node)
{
#ifdef DEBUG
std::cerr << "InnerNode missing: " << SHAMapNode(depth,id).getString() << std::endl;
#endif
throw SHAMapException(MissingNode);
}
for(int i=node->selectBranch(id)-1; i>=0; i--)
if(!!node->getChildHash(i))
{ // node has a subsequent child
SHAMapNode prevNode(node->getChildNodeID(i));
const uint256& prevHash(node->getChildHash(i));
if(prevNode.isLeaf())
{ // this is a terminal inner node
leaf=getLeaf(prevNode, prevHash, false);
if(!leaf) throw SHAMapException(MissingNode);
prev=leaf->firstItem();
if(!prev) throw SHAMapException(InvalidNode);
return prev;
}
// the next item is the first item below this node
SHAMapInnerNode::pointer inner=getInner(prevNode, prevHash, false);
if(!inner) throw SHAMapException(MissingNode);
prev=lastBelow(inner);
if(!prev) throw SHAMapException(InvalidNode);
return prev;
}
}
// must be last item
return SHAMapItem::pointer();
}
SHAMapLeafNode::pointer SHAMap::createLeaf(const SHAMapInnerNode& lowestParent, const uint256& id)
{ // caller must call dirtyUp if they populate the leaf
#ifdef DEBUG
std::cerr << "createLeaf(" << lowestParent.getString() << std::endl;
std::cerr << " for " << id.GetHex() << ")" << std::endl;
#endif
assert(!!id);
for(int depth=lowestParent.getDepth()+1; depth<SHAMapNode::leafDepth; depth++)
{
SHAMapInnerNode::pointer newNode(new SHAMapInnerNode(SHAMapNode(depth, id), mSeq));
#ifdef DEBUG
std::cerr << "create node " << newNode->getString() << std::endl;
#endif
mInnerNodeByID[*newNode]=newNode;
}
SHAMapLeafNode::pointer newLeaf(new SHAMapLeafNode(SHAMapNode(SHAMapNode::leafDepth, id), mSeq));
mLeafByID[*newLeaf]=newLeaf;
#ifdef DEBUG
std::cerr << "made leaf " << newLeaf->getString() << std::endl;
#endif
return newLeaf;
}
SHAMapItem::pointer SHAMap::peekItem(const uint256& id)
{
boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false);
if(!leaf) return SHAMapItem::pointer();
return leaf->findItem(id);
}
bool SHAMap::hasItem(const uint256& id)
{ // does the tree have an item with this ID
boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false);
if(!leaf) return false;
SHAMapItem::pointer item=leaf->findItem(id);
return (bool) item;
}
bool SHAMap::delItem(const uint256& id)
{ // delete the item with this ID
boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false);
if(!leaf) return false;
if(!leaf->delItem(id)) return false;
dirtyUp(id);
return true;
}
bool SHAMap::addGiveItem(const SHAMapItem::pointer item)
{ // add the specified item, does not update
boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(item->getTag(), true, true);
if(!leaf)
{
assert(false);
return false;
}
if(leaf->hasItem(item->getTag()))
{
#ifdef ST_DEBUG
std::cerr << "leaf has item we're adding" << std::endl;
#endif
return false;
}
if(!leaf->addUpdateItem(item, true))
{
assert(false);
return false;
}
dirtyUp(item->getTag());
return true;
}
bool SHAMap::addItem(const SHAMapItem& i)
{
return addGiveItem(SHAMapItem::pointer(new SHAMapItem(i)));
}
bool SHAMap::updateGiveItem(SHAMapItem::pointer item)
{
boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(item->getTag(), true, true);
if(!leaf) return false;
if(!leaf->addUpdateItem(item, true)) return false;
dirtyUp(item->getTag());
return true;
}
void SHAMapItem::dump()
{
std::cerr << "SHAMapItem(" << mTag.GetHex() << ") " << mData.size() << "bytes" << std::endl;
}
bool SHAMap::fetchNode(const uint256& hash, std::vector<unsigned char>& data)
{
HashedObject::pointer obj(HashedObject::retrieve(hash));
if(!obj) return false;
data=obj->getData();
return true;
}
int SHAMap::flushDirty(int maxNodes, HashedObjectType t, uint32 seq)
{
int flushed=0;
Serializer s;
if(mDirtyLeafNodes)
{
while(!mDirtyLeafNodes->empty())
{
SHAMapLeafNode::pointer& dln=mDirtyLeafNodes->begin()->second;
s.erase();
dln->addRaw(s);
HashedObject::store(t, seq, s.peekData(), s.getSHA512Half());
mDirtyLeafNodes->erase(mDirtyLeafNodes->begin());
if(flushed++>=maxNodes) return flushed;
}
}
if(mDirtyInnerNodes)
{
while(!mDirtyInnerNodes->empty())
{
SHAMapInnerNode::pointer& din=mDirtyInnerNodes->begin()->second;
s.erase();
din->addRaw(s);
HashedObject::store(t, seq, s.peekData(), s.getSHA512Half());
mDirtyInnerNodes->erase(mDirtyInnerNodes->begin());
if(flushed++>=maxNodes) return flushed;
}
}
return flushed;
}
void SHAMap::dump()
{
std::cerr << "SHAMap::dump" << std::endl;
SHAMapItem::pointer i=peekFirstItem();
while(i)
{
std::cerr << "Item: id=" << i->getTag().GetHex() << std::endl;
i=peekNextItem(i->getTag());
}
std::cerr << "SHAMap::dump done" << std::endl;
}
static std::vector<unsigned char>IntToVUC(int i)
{
std::vector<unsigned char> vuc;
vuc.push_back((unsigned char) i);
return vuc;
}
bool SHAMap::TestSHAMap()
{ // h3 and h4 differ only in the leaf, same terminal node (level 19)
uint256 h1, h2, h3, h4, h5;
h1.SetHex("092891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7");
h2.SetHex("436ccbac3347baa1f1e53baeef1f43334da88f1f6d70d963b833afd6dfa289fe");
h3.SetHex("b92891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
h4.SetHex("b92891fe4ef6cee585fdc6fda2e09eb4d386363158ec3321b8123e5a772c6ca8");
h5.SetHex("a92891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7");
SHAMap sMap;
SHAMapItem i1(h1, IntToVUC(1)), i2(h2, IntToVUC(2)), i3(h3, IntToVUC(3)), i4(h4, IntToVUC(4)), i5(h5, IntToVUC(5));
sMap.addItem(i2);
sMap.addItem(i1);
SHAMapItem::pointer i=sMap.peekFirstItem();
assert(!!i && (*i==i1));
i=sMap.peekNextItem(i->getTag());
assert(!!i && (*i==i2));
i=sMap.peekNextItem(i->getTag());
assert(!i);
sMap.addItem(i4);
sMap.delItem(i2.getTag());
sMap.addItem(i3);
i=sMap.peekFirstItem();
assert(!!i && (*i==i1));
i=sMap.peekNextItem(i->getTag());
assert(!!i && (*i==i3));
i=sMap.peekNextItem(i->getTag());
assert(!!i && (*i==i4));
i=sMap.peekNextItem(i->getTag());
assert(!i);
return true;
}