ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2025-11-27 22:45:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Major rewrite of the SHAMap code. This code performs much better

Browse Source 
than the original version, particularly for smaller maps.
This commit is contained in:
JoelKatz
2012-02-05 06:54:44 -08:00
parent 41ce5fa7f9
commit 30c9bf0ed2
7 changed files with 753 additions and 912 deletions
Download Patch File Download Diff File Expand all files Collapse all files

793
SHAMap.cpp
View File

@@ -1,6 +1,9 @@
#include <stack>
#include <boost/foreach.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/smart_ptr/make_shared.hpp>
#include "Serializer.h"
#include "BitcoinUtil.h"
@@ -8,205 +11,144 @@
SHAMap::SHAMap(uint32 seq) : mSeq(seq), mImmutable(false), mSynching(false)
{
root=SHAMapInnerNode::pointer(new SHAMapInnerNode(SHAMapNode(SHAMapNode::rootDepth, uint256()), mSeq));
mInnerNodeByID[*root]=root;
root=boost::make_shared<SHAMapTreeNode>(SHAMapNode(0, uint256()), mSeq);
root->makeInner();
mTNByID[*root]=root;
}
void SHAMap::dirtyUp(const uint256& id)
std::stack<SHAMapTreeNode::pointer> SHAMap::getStack(const uint256& id, bool include_nonmatching_leaf)
{
// Walk the tree as far as possible to the specified identifier
// produce a stack of nodes along the way, with the terminal node at the top
std::stack<SHAMapTreeNode::pointer> stack;
SHAMapTreeNode::pointer node=root;
while(!node->isLeaf())
{
stack.push(node);
int branch=node->selectBranch(id);
assert(branch>=0);
uint256 hash=node->getChildHash(branch);
if(!hash) return stack;
node=getNode(node->getChildNodeID(branch), node->getChildHash(branch), false);
if(!node)
{
if(mSynching) return stack;
throw SHAMapException(MissingNode);
}
}
if(include_nonmatching_leaf || (node->peekItem()->getTag()==id))
stack.push(node);
return stack;
}
void SHAMap::dirtyUp(std::stack<SHAMapTreeNode::pointer>& stack, const uint256& target, uint256 prevHash)
{ // walk the tree up from through the inner nodes to the root
// update linking hashes and add nodes to dirty list
assert(!mImmutable && !mSynching);
SHAMapLeafNode::pointer leaf=checkCacheLeaf(SHAMapNode(SHAMapNode::leafDepth, id));
if(!leaf) throw SHAMapException(MissingNode);
uint256 hVal=leaf->getNodeHash();
if(mDirtyLeafNodes) mDirtyLeafNodes->insert(std::make_pair(SHAMapNode(*leaf), leaf));
if(!hVal)
while(!stack.empty())
{
#ifdef ST_DEBUG
std::cerr << " erasingL " << leaf->getString() << std::endl;
#endif
mLeafByID.erase(*leaf);
}
SHAMapTreeNode::pointer node=stack.top();
stack.pop();
assert(node->isInnerNode());
for(int depth=SHAMapNode::leafDepth-1; depth>=0; depth--)
{ // walk up the tree to the root updating nodes
SHAMapInnerNode::pointer node=checkCacheNode(SHAMapNode(depth, leaf->getNodeID()));
if(!node) throw SHAMapException(MissingNode);
if(!node->setChildHash(node->selectBranch(id), hVal))
int branch=node->selectBranch(target);
assert(branch>=0);
returnNode(node, true);
if(!node->setChildHash(branch, prevHash))
{
#ifdef ST_DEBUG
std::cerr << " no change@ " << node->getString() << std::endl;
#endif
std::cerr << "dirtyUp terminates early" << std::endl;
assert(false);
return;
}
#ifdef ST_DEBUG
std::cerr << "Dirty " << node->getString() << std::endl;
std::cerr << "dirtyUp sets branch " << branch << " to " << prevHash.GetHex() << std::endl;
#endif
if(mDirtyInnerNodes) mDirtyInnerNodes->insert(std::make_pair(SHAMapNode(*node), node));
hVal=node->getNodeHash();
if(!hVal)
{
#ifdef ST_DEBUG
std::cerr << " erasingN " << node->getString() << std::endl;
#endif
mInnerNodeByID.erase(*node);
}
prevHash=node->getNodeHash();
assert(!!prevHash);
}
}
SHAMapLeafNode::pointer SHAMap::checkCacheLeaf(const SHAMapNode& iNode)
SHAMapTreeNode::pointer SHAMap::checkCacheNode(const SHAMapNode& iNode)
{
assert(iNode.isLeaf());
boost::unordered_map<SHAMapNode, SHAMapLeafNode::pointer>::iterator it=mLeafByID.find(iNode);
if(it==mLeafByID.end()) return SHAMapLeafNode::pointer();
boost::unordered_map<SHAMapNode, SHAMapTreeNode::pointer>::iterator it=mTNByID.find(iNode);
if(it==mTNByID.end()) return SHAMapTreeNode::pointer();
return it->second;
}
SHAMapInnerNode::pointer SHAMap::checkCacheNode(const SHAMapNode& iNode)
{
assert(!iNode.isLeaf());
boost::unordered_map<SHAMapNode, SHAMapInnerNode::pointer>::iterator it=mInnerNodeByID.find(iNode);
if(it==mInnerNodeByID.end()) return SHAMapInnerNode::pointer();
return it->second;
}
SHAMapTreeNode::pointer SHAMap::walkTo(const uint256& id, bool modify)
{ // walk down to the terminal node for this ID
SHAMapTreeNode::pointer inNode=root;
SHAMapLeafNode::pointer SHAMap::walkToLeaf(const uint256& id, bool create, bool modify)
{ // walk down to the leaf that would contain this ID
// is leaf node in cache
SHAMapLeafNode::pointer ln=checkCacheLeaf(SHAMapNode(SHAMapNode::leafDepth, id));
if(ln) return returnLeaf(ln, modify);
// walk tree to leaf
SHAMapInnerNode::pointer inNode=root;
for(int i=0; i<SHAMapNode::leafDepth; i++)
while(!inNode->isLeaf())
{
int branch=inNode->selectBranch(id);
if(branch<0) // somehow we got on the wrong branch
throw SHAMapException(InvalidNode);
if(inNode->isEmptyBranch(branch))
{ // no nodes below this one
if(!create) return SHAMapLeafNode::pointer();
return createLeaf(*inNode, id);
}
if(inNode->isChildLeaf())
{ // child is leaf node
ln=getLeaf(inNode->getChildNodeID(branch), inNode->getChildHash(branch), modify);
if(ln==NULL)
{
if(!create) return SHAMapLeafNode::pointer();
return createLeaf(*inNode, id);
}
}
else
{ // child is another inner node
inNode=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), modify);
if(inNode==NULL) throw SHAMapException(InvalidNode);
if(inNode->isEmptyBranch(branch)) return inNode;
uint256 childHash=inNode->getChildHash(branch);
if(!childHash) return inNode;
SHAMapTreeNode::pointer nextNode=getNode(inNode->getChildNodeID(branch), childHash, false);
if(!nextNode) throw SHAMapException(MissingNode);
inNode=nextNode;
}
if(modify) returnNode(inNode, true);
return inNode;
}
assert(ln || !create);
return returnLeaf(ln, modify);
}
SHAMapInnerNode::pointer SHAMap::walkTo(const SHAMapNode& id)
{ // walk down to this ID, as far as possible
SHAMapInnerNode::pointer inNode=root;
int i=0;
do
SHAMapTreeNode::pointer SHAMap::getNode(const SHAMapNode& id, const uint256& hash, bool modify)
{ // retrieve a node whose node hash is known
SHAMapTreeNode::pointer node=checkCacheNode(id);
if(node)
{
int branch=inNode->selectBranch(id.getNodeID());
if(branch<0) // somehow we got on the wrong branch
throw SHAMapException(InvalidNode);
if (inNode->isEmptyBranch(branch)) // we know no branches below this one
if(node->getNodeHash()!=hash)
{
#ifdef DEBUG
std::cerr << "Walk down empty branch" << std::endl;
std::cerr << "Attempt to get node, hash not in tree" << std::endl;
std::cerr << "ID: " << id.getString() << std::endl;
std::cerr << "TgtHash " << hash.GetHex() << std::endl;
std::cerr << "NodHash " << node->getNodeHash().GetHex() << std::endl;
dump();
#endif
return inNode;
throw SHAMapException(InvalidNode);
}
if(inNode->isChildLeaf()) // this is the last inner node
return inNode;
try
{
SHAMapInnerNode::pointer next=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), false);
if(!next) // we don't have the next node
{
#ifdef ST_DEBUG
std::cerr << "Unable to find node " << inNode->getChildNodeID(branch).getString() << std::endl;
#endif
return inNode;
}
assert(next->getDepth() == (inNode->getDepth()+1));
inNode=next;
}
catch (const SHAMapException& x)
{
if(x!=SHAMapException(MissingNode)) throw(x);
return inNode;
}
assert(i++ < SHAMapNode::leafDepth);
} while(1);
returnNode(node, modify);
return node;
}
SHAMapLeafNode::pointer SHAMap::getLeaf(const SHAMapNode& id, const uint256& hash, bool modify)
{ // retrieve a leaf whose node hash is known
assert(!!hash);
if(!id.isLeaf()) return SHAMapLeafNode::pointer();
std::vector<unsigned char> nodeData;
if(!fetchNode(hash, nodeData)) return SHAMapTreeNode::pointer();
SHAMapLeafNode::pointer leaf=checkCacheLeaf(id);
if(leaf) return returnLeaf(leaf, modify);
std::vector<unsigned char> leafData;
if(!fetchNode(hash, leafData)) return SHAMapLeafNode::pointer();
leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(id, leafData, mSeq));
if(leaf->getNodeHash()!=hash) throw SHAMapException(InvalidNode);
mLeafByID.insert(std::make_pair(id, leaf));
return leaf;
}
SHAMapInnerNode::pointer SHAMap::getInner(const SHAMapNode& id, const uint256& hash, bool modify)
{ // retrieve an inner node whose node hash is known
SHAMapInnerNode::pointer node=checkCacheNode(id);
if(node) return returnNode(node, modify);
std::vector<unsigned char> rawNode;
if(!fetchNode(hash, rawNode)) return SHAMapInnerNode::pointer();
node=SHAMapInnerNode::pointer(new SHAMapInnerNode(id, rawNode, mSeq));
node=boost::make_shared<SHAMapTreeNode>(id, nodeData, mSeq);
if(node->getNodeHash()!=hash) throw SHAMapException(InvalidNode);
mInnerNodeByID.insert(std::make_pair(id, node));
if(id.getDepth()==0) root=node;
if(!mTNByID.insert(std::make_pair(id, node)).second)
assert(false);
return node;
}
SHAMapLeafNode::pointer SHAMap::returnLeaf(SHAMapLeafNode::pointer leaf, bool modify)
{ // make sure the leaf is suitable for the intended operation (copy on write)
if(leaf && modify && (leaf->getSeq()!=mSeq))
{
leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(*leaf, mSeq));
mLeafByID[*leaf]=leaf;
if(mDirtyLeafNodes) mDirtyLeafNodes->insert(std::make_pair(SHAMapNode(*leaf), leaf));
}
return leaf;
}
SHAMapInnerNode::pointer SHAMap::returnNode(SHAMapInnerNode::pointer node, bool modify)
void SHAMap::returnNode(SHAMapTreeNode::pointer& node, bool modify)
{ // make sure the node is suitable for the intended operation (copy on write)
assert(node->isValid());
if(node && modify && (node->getSeq()!=mSeq))
{
#ifdef ST_DEBUG
std::cerr << "Node(" << node->getString() << ") bumpseq" << std::endl;
#ifdef DEBUG
std::cerr << "returnNode COW" << std::endl;
#endif
node=SHAMapInnerNode::pointer(new SHAMapInnerNode(*node, mSeq));
mInnerNodeByID.insert(std::make_pair(SHAMapNode(*node), node));
if(mDirtyInnerNodes) mDirtyInnerNodes->insert(std::make_pair(SHAMapNode(*node), node));
if(mDirtyNodes) (*mDirtyNodes)[*node]=node;
node=boost::make_shared<SHAMapTreeNode>(*node, mSeq);
assert(node->isValid());
mTNByID[*node]=node;
}
return node;
}
SHAMapItem::SHAMapItem(const uint256& tag, const std::vector<unsigned char>& data)
@@ -217,6 +159,57 @@ SHAMapItem::SHAMapItem(const uint160& tag, const std::vector<unsigned char>& dat
: mTag(tag.to256()), mData(data)
{ ; }
SHAMapItem::pointer SHAMap::firstBelow(SHAMapTreeNode::pointer node)
{
// Return the first item below this node
#ifdef ST_DEBUG
std::cerr << "firstBelow(" << node->getString() << ")" << std::endl;
#endif
do
{ // Walk down the tree
if(node->hasItem()) return node->peekItem();
bool foundNode=false;
for(int i=0; i<16; i++)
if(!node->isEmptyBranch(i))
{
#ifdef ST_DEBUG
std::cerr << " FB: node " << node->getString() << std::endl;
std::cerr << " has non-empty branch " << i << " : " <<
node->getChildNodeID(i).getString() << ", " << node->getChildHash(i).GetHex() << std::endl;
#endif
node=getNode(node->getChildNodeID(i), node->getChildHash(i), false);
if(!node) throw SHAMapException(MissingNode);
foundNode=true;
break;
}
if(!foundNode) return SHAMapItem::pointer();
} while(1);
}
SHAMapItem::pointer SHAMap::lastBelow(SHAMapTreeNode::pointer node)
{
#ifdef DEBUG
std::cerr << "lastBelow(" << node->getString() << ")" << std::endl;
#endif
do
{ // Walk down the tree
if(node->hasItem()) return node->peekItem();
bool foundNode=false;
for(int i=16; i>=0; i++)
if(!node->isEmptyBranch(i))
{
node=getNode(node->getChildNodeID(i), node->getChildHash(i), false);
if(!node) throw SHAMapException(MissingNode);
foundNode=true;
break;
}
if(!foundNode) return SHAMapItem::pointer();
} while(1);
}
SHAMapItem::pointer SHAMap::peekFirstItem()
{
boost::recursive_mutex::scoped_lock sl(mLock);
@@ -229,275 +222,242 @@ SHAMapItem::pointer SHAMap::peekLastItem()
return lastBelow(root);
}
SHAMapItem::pointer SHAMap::firstBelow(SHAMapInnerNode::pointer node)
{
#ifdef DEBUG
std::cerr << "firstBelow(" << node->getString() << ")" << std::endl;
#endif
int i;
while(!node->isChildLeaf())
{
for(i=0; i<32; i++)
if(!node->isEmptyBranch(i))
{
node=getInner(node->getChildNodeID(i), node->getChildHash(i), false);
if(!node) throw SHAMapException(MissingNode);
break;
}
if(i==32)
{
#ifdef ST_DEBUG
std::cerr << " node:" << node->getString() << " has no children" << std::endl;
#endif
return SHAMapItem::pointer();
}
}
assert(node->isChildLeaf());
for(int i=0; i<32; i++)
if(!node->isEmptyBranch(i))
{
SHAMapLeafNode::pointer mLeaf=getLeaf(node->getChildNodeID(i), node->getChildHash(i), false);
if(!mLeaf) throw SHAMapException(MissingNode);
SHAMapItem::pointer item=mLeaf->firstItem();
if(!item) throw SHAMapException(InvalidNode);
return item;
}
#ifdef ST_DEBUG
std::cerr << " node:" << node->getString() << " has no children" << std::endl;
#endif
return SHAMapItem::pointer();
}
SHAMapItem::pointer SHAMap::lastBelow(SHAMapInnerNode::pointer node)
{
boost::recursive_mutex::scoped_lock sl(mLock);
const uint256 zero;
int i;
while(!node->isChildLeaf())
{
for(i=31; i>=0; i--)
{
uint256 cHash(node->getChildHash(i));
if(cHash!=0)
{
node=getInner(node->getChildNodeID(i), cHash, false);
if(!node) return SHAMapItem::pointer();
break;
}
}
if(i<0) return SHAMapItem::pointer();
}
for(int i=31; i>=0; i--)
{
uint256 cHash=node->getChildHash(i);
if(cHash!=zero)
{
SHAMapLeafNode::pointer mLeaf=getLeaf(node->getChildNodeID(i), cHash, false);
if(!mLeaf) return SHAMapItem::pointer();
return mLeaf->lastItem();
}
}
return SHAMapItem::pointer();
}
SHAMapItem::pointer SHAMap::peekNextItem(const uint256& id)
{ // Get a pointer to the next item in the tree after a given item - item must be in tree
boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false);
if(!leaf)
std::stack<SHAMapTreeNode::pointer> stack=getStack(id, true);
while(!stack.empty())
{
#ifdef DEBUG
std::cerr << "peekNextItem: current not found" << std::endl;
#endif
return SHAMapItem::pointer();
}
SHAMapTreeNode::pointer node=stack.top();
stack.pop();
// is there another item in this leaf? (there almost never will be)
SHAMapItem::pointer next=leaf->nextItem(id);
if(next) return next;
for(int depth=SHAMapNode::leafDepth-1; depth>=0; depth--)
{ // walk up the tree until we find a node with a subsequent child
SHAMapInnerNode::pointer node=checkCacheNode(SHAMapNode(depth, id));
if(!node)
if(node->isLeaf())
{
#ifdef DEBUG
std::cerr << "InnerNode missing: " << SHAMapNode(depth,id).getString() << std::endl;
#endif
throw SHAMapException(MissingNode);
if(node->peekItem()->getTag()>id)
return node->peekItem();
}
#ifdef ST_DEBUG
std::cerr << " UpTo " << node->getString() << std::endl;
#endif
for(int i=node->selectBranch(id)+1; i<32; i++)
if(!!node->getChildHash(i))
{ // node has a subsequent child
SHAMapNode nextNode(node->getChildNodeID(i));
const uint256& nextHash(node->getChildHash(i));
if(nextNode.isLeaf())
{ // this is a terminal inner node
leaf=getLeaf(nextNode, nextHash, false);
if(!leaf) throw SHAMapException(MissingNode);
next=leaf->firstItem();
if(!next) throw SHAMapException(InvalidNode);
return next;
}
// the next item is the first item below this node
SHAMapInnerNode::pointer inner=getInner(nextNode, nextHash, false);
if(!inner) throw SHAMapException(MissingNode);
next=firstBelow(inner);
if(!next) throw SHAMapException(InvalidNode);
return next;
else for(int i=node->selectBranch(id)+1; i<16; i++)
if(!node->isEmptyBranch(i))
{
node=getNode(node->getChildNodeID(i), node->getChildHash(i), false);
if(!node) throw SHAMapException(MissingNode);
SHAMapItem::pointer item=firstBelow(node);
if(!item) throw SHAMapException(MissingNode);
return item;
}
}
#ifdef ST_DEBUG
std::cerr << " peekNext off end" << std::endl;
#endif
// must be last item
return SHAMapItem::pointer();
}
SHAMapItem::pointer SHAMap::peekPrevItem(const uint256& id)
{
{ // Get a pointer to the previous item in the tree after a given item - item must be in tree
boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false);
if(!leaf) return SHAMapItem::pointer();
// is there another item in this leaf? (there almost never will be)
SHAMapItem::pointer prev=leaf->prevItem(id);
if(prev) return prev;
for(int depth=SHAMapNode::leafDepth-1; depth>=0; depth--)
{ // walk up the tree until we find a node with a previous child
SHAMapInnerNode::pointer node=checkCacheNode(SHAMapNode(depth, id));
if(!node)
std::stack<SHAMapTreeNode::pointer> stack=getStack(id, true);
while(!stack.empty())
{
#ifdef DEBUG
std::cerr << "InnerNode missing: " << SHAMapNode(depth,id).getString() << std::endl;
#endif
throw SHAMapException(MissingNode);
}
for(int i=node->selectBranch(id)-1; i>=0; i--)
if(!!node->getChildHash(i))
{ // node has a subsequent child
SHAMapNode prevNode(node->getChildNodeID(i));
const uint256& prevHash(node->getChildHash(i));
SHAMapTreeNode::pointer node=stack.top();
stack.pop();
if(prevNode.isLeaf())
{ // this is a terminal inner node
leaf=getLeaf(prevNode, prevHash, false);
if(!leaf) throw SHAMapException(MissingNode);
prev=leaf->firstItem();
if(!prev) throw SHAMapException(InvalidNode);
return prev;
if(node->isLeaf())
{
if(node->peekItem()->getTag()<id)
return node->peekItem();
}
// the next item is the first item below this node
SHAMapInnerNode::pointer inner=getInner(prevNode, prevHash, false);
if(!inner) throw SHAMapException(MissingNode);
prev=lastBelow(inner);
if(!prev) throw SHAMapException(InvalidNode);
return prev;
else for(int i=node->selectBranch(id)-1; i>=0; i--)
if(!node->isEmptyBranch(i))
{
node=getNode(node->getChildNodeID(i), node->getChildHash(i), false);
if(!node) throw SHAMapException(MissingNode);
SHAMapItem::pointer item=firstBelow(node);
if(!item) throw SHAMapException(MissingNode);
return item;
}
}
// must be last item
return SHAMapItem::pointer();
}
SHAMapLeafNode::pointer SHAMap::createLeaf(const SHAMapInnerNode& lowestParent, const uint256& id)
{ // caller must call dirtyUp if they populate the leaf
#ifdef ST_DEBUG
std::cerr << "createLeaf(" << lowestParent.getString() << std::endl;
std::cerr << " for " << id.GetHex() << ")" << std::endl;
#endif
assert(!!id);
for(int depth=lowestParent.getDepth()+1; depth<SHAMapNode::leafDepth; depth++)
{
SHAMapInnerNode::pointer newNode(new SHAMapInnerNode(SHAMapNode(depth, id), mSeq));
mInnerNodeByID[*newNode]=newNode;
}
SHAMapLeafNode::pointer newLeaf(new SHAMapLeafNode(SHAMapNode(SHAMapNode::leafDepth, id), mSeq));
mLeafByID[*newLeaf]=newLeaf;
#ifdef ST_DEBUG
std::cerr << "made leaf " << newLeaf->getString() << std::endl;
#endif
return newLeaf;
}
SHAMapItem::pointer SHAMap::peekItem(const uint256& id)
{
boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false);
SHAMapTreeNode::pointer leaf=walkTo(id, false);
if(!leaf) return SHAMapItem::pointer();
return leaf->findItem(id);
return leaf->peekItem();
}
bool SHAMap::hasItem(const uint256& id)
{ // does the tree have an item with this ID
boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false);
SHAMapTreeNode::pointer leaf=walkTo(id, false);
if(!leaf) return false;
return leaf->hasItem(id);
SHAMapItem::pointer item=leaf->peekItem();
if(!item || item->getTag()!=id) return false;
return true;
}
bool SHAMap::delItem(const uint256& id)
{ // delete the item with this ID
boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false);
if(!leaf) return false;
if(!leaf->delItem(id)) return false;
dirtyUp(id);
std::stack<SHAMapTreeNode::pointer> stack=getStack(id, false);
if(stack.empty()) throw SHAMapException(MissingNode);
SHAMapTreeNode::pointer leaf=stack.top();
stack.pop();
if( !leaf || !leaf->hasItem() || (leaf->peekItem()->getTag()!=id) )
return false;
SHAMapTreeNode::TNType type=leaf->getType();
returnNode(leaf, true);
mTNByID.erase(*leaf);
uint256 prevHash;
while(!stack.empty())
{
SHAMapTreeNode::pointer node=stack.top();
stack.pop();
returnNode(node, true);
if(!node->setChildHash(node->selectBranch(id), prevHash))
return true;
if(!prevHash && !node->isRoot())
{ // we may have made this a node with 1 or 0 children
int bc=node->getBranchCount();
if(bc==0)
{
prevHash=uint256();
mTNByID.erase(*node);
}
else if(bc==1)
{ // pull up on the thread
SHAMapItem::pointer item=firstBelow(node);
assert(item);
node->setItem(item, type);
prevHash=node->getNodeHash();
}
}
else prevHash=node->getNodeHash();
}
return true;
}
bool SHAMap::addGiveItem(const SHAMapItem::pointer item)
bool SHAMap::addGiveItem(SHAMapItem::pointer item, bool isTransaction)
{ // add the specified item, does not update
boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(item->getTag(), true, true);
if(!leaf)
{
assert(false);
return false;
}
if(leaf->hasItem(item->getTag()))
{
#ifdef ST_DEBUG
std::cerr << "leaf has item we're adding" << std::endl;
std::cerr << "aGI " << item->getTag().GetHex() << std::endl;
#endif
return false;
}
if(!leaf->addUpdateItem(item, true))
uint256 tag=item->getTag();
SHAMapTreeNode::TNType type=isTransaction ? SHAMapTreeNode::TRANSACTION : SHAMapTreeNode::ACCOUNT_STATE;
boost::recursive_mutex::scoped_lock sl(mLock);
std::stack<SHAMapTreeNode::pointer> stack=getStack(tag, true);
if(stack.empty()) throw SHAMapException(MissingNode);
SHAMapTreeNode::pointer node=stack.top();
stack.pop();
if( node->isLeaf() && (node->peekItem()->getTag()==tag) )
{
assert(false);
std::cerr << "addGiveItem ends on leaf with same tag" << std::endl;
return false;
}
dirtyUp(item->getTag());
uint256 prevHash;
returnNode(node, true);
if(node->isInner())
{ // easy case, we end on an inner node
#ifdef ST_DEBUG
std::cerr << "aGI inner " << node->getString() << std::endl;
#endif
int branch=node->selectBranch(tag);
assert(node->isEmptyBranch(branch));
SHAMapTreeNode::pointer newNode=
boost::make_shared<SHAMapTreeNode>(node->getChildNodeID(branch), item, type, mSeq);
if(!mTNByID.insert(std::make_pair(SHAMapNode(*newNode), newNode)).second)
{
std::cerr << "Node: " << node->getString() << std::endl;
std::cerr << "NewNode: " << newNode->getString() << std::endl;
assert(false);
throw SHAMapException(InvalidNode);
}
node->setChildHash(branch, newNode->getNodeHash());
}
else
{ // this is a leaf node that has to be made an inner node holding two items
#ifdef ST_DEBUG
std::cerr << "aGI leaf " << node->getString() << std::endl;
#endif
SHAMapItem::pointer otherItem=node->peekItem();
assert(otherItem && (tag!=otherItem->getTag()) );
node->makeInner();
int b1, b2;
while( (b1=node->selectBranch(tag)) == (b2=node->selectBranch(otherItem->getTag())) )
{ // we need a new inner node, since both go on same branch at this level
#ifdef ST_DEBUG
std::cerr << "need new inner node at " << node->getDepth() << std::endl;
#endif
SHAMapTreeNode::pointer newNode=boost::make_shared<SHAMapTreeNode>(node->getChildNodeID(b1), mSeq);
newNode->makeInner();
if(!mTNByID.insert(std::make_pair(SHAMapNode(*newNode), newNode)).second)
assert(false);
stack.push(node);
node=newNode;
}
// we can add the two leaf nodes here
assert(node->isInner());
SHAMapTreeNode::pointer newNode=
boost::make_shared<SHAMapTreeNode>(node->getChildNodeID(b1), item, type, mSeq);
assert(newNode->isValid() && newNode->isLeaf());
if(!mTNByID.insert(std::make_pair(SHAMapNode(*newNode), newNode)).second)
assert(false);
node->setChildHash(b1, newNode->getNodeHash()); // OPTIMIZEME hash op not needed
newNode=boost::make_shared<SHAMapTreeNode>(node->getChildNodeID(b2), otherItem, type, mSeq);
assert(newNode->isValid() && newNode->isLeaf());
if(!mTNByID.insert(std::make_pair(SHAMapNode(*newNode), newNode)).second)
assert(false);
node->setChildHash(b2, newNode->getNodeHash());
}
prevHash=node->getNodeHash();
assert(!!prevHash);
dirtyUp(stack, tag, prevHash);
return true;
}
bool SHAMap::addItem(const SHAMapItem& i)
bool SHAMap::addItem(const SHAMapItem& i, bool isTransaction)
{
return addGiveItem(SHAMapItem::pointer(new SHAMapItem(i)));
return addGiveItem(boost::make_shared<SHAMapItem>(i), isTransaction);
}
bool SHAMap::updateGiveItem(SHAMapItem::pointer item)
{
bool SHAMap::updateGiveItem(SHAMapItem::pointer item, bool isTransaction)
{ // can't change the tag but can change the hash
uint256 tag=item->getTag();
boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(item->getTag(), true, true);
if(!leaf) return false;
if(!leaf->addUpdateItem(item, true)) return false;
dirtyUp(item->getTag());
std::stack<SHAMapTreeNode::pointer> stack=getStack(tag, true);
if(stack.empty()) throw SHAMapException(MissingNode);
SHAMapTreeNode::pointer node=stack.top();
stack.pop();
if(!node->isLeaf() || (node->peekItem()->getTag()==tag) )
return false;
returnNode(node, true);
if(!node->setItem(item, isTransaction ? SHAMapTreeNode::TRANSACTION : SHAMapTreeNode::ACCOUNT_STATE))
return true;
dirtyUp(stack, tag, node->getNodeHash());
return true;
}
@@ -519,28 +479,15 @@ int SHAMap::flushDirty(int maxNodes, HashedObjectType t, uint32 seq)
int flushed=0;
Serializer s;
if(mDirtyLeafNodes)
if(mDirtyNodes)
{
while(!mDirtyLeafNodes->empty())
while(!mDirtyNodes->empty())
{
SHAMapLeafNode::pointer& dln=mDirtyLeafNodes->begin()->second;
s.erase();
dln->addRaw(s);
HashedObject::store(t, seq, s.peekData(), s.getSHA512Half());
mDirtyLeafNodes->erase(mDirtyLeafNodes->begin());
if(flushed++>=maxNodes) return flushed;
}
}
if(mDirtyInnerNodes)
{
while(!mDirtyInnerNodes->empty())
{
SHAMapInnerNode::pointer& din=mDirtyInnerNodes->begin()->second;
SHAMapTreeNode::pointer& din=mDirtyNodes->begin()->second;
s.erase();
din->addRaw(s);
HashedObject::store(t, seq, s.peekData(), s.getSHA512Half());
mDirtyInnerNodes->erase(mDirtyInnerNodes->begin());
mDirtyNodes->erase(mDirtyNodes->begin());
if(flushed++>=maxNodes) return flushed;
}
}
@@ -548,48 +495,30 @@ int SHAMap::flushDirty(int maxNodes, HashedObjectType t, uint32 seq)
return flushed;
}
SHAMapInnerNode::pointer SHAMap::getInnerNode(const SHAMapNode& node)
SHAMapTreeNode::pointer SHAMap::getNode(const SHAMapNode& nodeID)
{
boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapInnerNode::pointer inNode=root;
for(int i=0; i<SHAMapNode::leafDepth; i++)
{
if(inNode->getDepth()==node.getDepth())
{
if((*inNode)!=node) return SHAMapInnerNode::pointer();
return inNode;
}
int branch=inNode->selectBranch(node.getNodeID());
if( (branch<0) || (inNode->isEmptyBranch(branch)) ) return SHAMapInnerNode::pointer();
inNode=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), false);
if(!inNode) return inNode;
}
if(inNode->getDepth()==node.getDepth())
{
if((*inNode)!=node) return SHAMapInnerNode::pointer();
return inNode;
}
return SHAMapInnerNode::pointer();
}
SHAMapLeafNode::pointer SHAMap::getLeafNode(const SHAMapNode& leaf)
SHAMapTreeNode::pointer node=checkCacheNode(nodeID);
if(node) return node;
node=root;
while(nodeID!=*node)
{
boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapInnerNode::pointer inNode=root;
for(int i=0; inNode->getDepth()<SHAMapNode::leafDepth; i++)
{
int branch=inNode->selectBranch(leaf.getNodeID());
if( (branch<0) || (inNode->isEmptyBranch(branch)) ) return SHAMapLeafNode::pointer();
inNode=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), false);
if(!inNode) return SHAMapLeafNode::pointer();
int branch=node->selectBranch(nodeID.getNodeID());
assert(branch>=0);
if( (branch<0) || (node->isEmptyBranch(branch)) )
return SHAMapTreeNode::pointer();
node=getNode(node->getChildNodeID(branch), node->getChildHash(branch), false);
if(!node) throw SHAMapException(MissingNode);
}
int branch=inNode->selectBranch(leaf.getNodeID());
if( (branch<0) || (inNode->isEmptyBranch(branch)) ) return SHAMapLeafNode::pointer();
return getLeaf(inNode->getChildNodeID(branch), inNode->getChildHash(branch), false);
return node;
}
void SHAMap::dump()
{
#if 0
std::cerr << "SHAMap::dump" << std::endl;
SHAMapItem::pointer i=peekFirstItem();
while(i)
@@ -598,11 +527,22 @@ void SHAMap::dump()
i=peekNextItem(i->getTag());
}
std::cerr << "SHAMap::dump done" << std::endl;
#endif
std::cerr << " MAP Contains" << std::endl;
boost::recursive_mutex::scoped_lock sl(mLock);
for(boost::unordered_map<SHAMapNode, SHAMapTreeNode::pointer, hash_SMN>::iterator it=mTNByID.begin();
it!=mTNByID.end(); ++it)
{
std::cerr << it->second->getString() << std::endl;
}
}
static std::vector<unsigned char>IntToVUC(int i)
{
std::vector<unsigned char> vuc;
for(int i=0; i<32; i++)
vuc.push_back((unsigned char) i);
return vuc;
}
@@ -619,19 +559,29 @@ bool SHAMap::TestSHAMap()
SHAMap sMap;
SHAMapItem i1(h1, IntToVUC(1)), i2(h2, IntToVUC(2)), i3(h3, IntToVUC(3)), i4(h4, IntToVUC(4)), i5(h5, IntToVUC(5));
sMap.addItem(i2);
sMap.addItem(i1);
if(!sMap.addItem(i2, true))
{
assert(false);
return false;
}
if(!sMap.addItem(i1, true))
{
assert(false);
return false;
}
SHAMapItem::pointer i=sMap.peekFirstItem();
SHAMapItem::pointer i;
i=sMap.peekFirstItem();
assert(!!i && (*i==i1));
i=sMap.peekNextItem(i->getTag());
assert(!!i && (*i==i2));
i=sMap.peekNextItem(i->getTag());
assert(!i);
sMap.addItem(i4);
sMap.addItem(i4, true);
sMap.delItem(i2.getTag());
sMap.addItem(i3);
sMap.addItem(i3, true);
i=sMap.peekFirstItem();
assert(!!i && (*i==i1));
@@ -642,9 +592,8 @@ bool SHAMap::TestSHAMap()
i=sMap.peekNextItem(i->getTag());
assert(!i);
if(!syncTest());
if(!syncTest())
return false;
return true;
}

182
SHAMap.h
View File

@@ -3,7 +3,7 @@
#include <list>
#include <map>
#include <deque>
#include <stack>
#include <boost/shared_ptr.hpp>
#include <boost/enable_shared_from_this.hpp>
@@ -18,7 +18,6 @@
class SHAMap;
// A tree-like map of SHA256 hashes
// 21 levels, consisting of a root, 19 interior node levels, and leaves
// The trees are designed for rapid synchronization and compression of differences
@@ -28,29 +27,27 @@ public:
typedef boost::shared_ptr<SHAMapNode> pointer;
private:
static uint256 smMasks[21]; // AND with hash to get node id
static uint256 smMasks[64]; // AND with hash to get node id
uint256 mNodeID;
int mDepth;
public:
// 0 is root, 20 is leaf
static const int rootDepth=0;
static const int leafDepth=20;
SHAMapNode() : mDepth(0) { ; }
SHAMapNode(int depth, const uint256& hash);
int getDepth() const { return mDepth; }
const uint256& getNodeID() const { return mNodeID; }
bool isRoot() const { return mDepth==0; }
bool isLeaf() const { return mDepth==leafDepth; }
bool isChildLeaf() const { return mDepth==(leafDepth-1); }
bool isInner() const { return !isRoot() && !isLeaf(); }
virtual bool isPopulated() const { return false; }
SHAMapNode getParentNodeID() const { return SHAMapNode(mDepth-1, mNodeID); }
SHAMapNode getParentNodeID() const
{
assert(mDepth);
return SHAMapNode(mDepth-1, mNodeID);
}
SHAMapNode getChildNodeID(int m) const;
int selectBranch(const uint256& hash) const;
@@ -62,6 +59,7 @@ public:
bool operator!=(const uint256&) const;
bool operator<=(const SHAMapNode&) const;
bool operator>=(const SHAMapNode&) const;
bool isRoot() const { return mDepth==0; }
virtual std::string getString() const;
void dump() const;
@@ -100,6 +98,8 @@ public:
const uint256& getTag() const { return mTag; }
std::vector<unsigned char> getData() const { return mData; }
const std::vector<unsigned char>& peekData() const { return mData; }
void addRaw(Serializer &s) { s.addRaw(mData); }
void addRaw(std::vector<unsigned char>& s) { s.insert(s.end(), mData.begin(), mData.end()); }
void updateData(const std::vector<unsigned char>& data) { mData=data; }
@@ -118,93 +118,74 @@ public:
virtual void dump();
};
class SHAMapLeafNode : public SHAMapNode
class SHAMapTreeNode : public SHAMapNode
{
friend class SHAMap;
public:
typedef boost::shared_ptr<SHAMapLeafNode> pointer;
typedef boost::shared_ptr<SHAMapTreeNode> pointer;
private:
uint256 mHash;
std::list<SHAMapItem::pointer> mItems;
uint32 mSeq;
bool updateHash();
SHAMapLeafNode(const SHAMapLeafNode&); // no implementation
SHAMapLeafNode& operator=(const SHAMapLeafNode&); // no implementation
protected:
bool addUpdateItem(SHAMapItem::pointer, bool doHash);
bool delItem(const SHAMapItem::pointer i) { return delItem(i->getTag()); }
bool delItem(const uint256& tag);
public:
SHAMapLeafNode(const SHAMapNode& nodeID, uint32 seq);
SHAMapLeafNode(const SHAMapLeafNode& node, uint32 seq);
SHAMapLeafNode(const SHAMapNode& id, const std::vector<unsigned char>& contents, uint32 seq);
void addRaw(Serializer &);
virtual bool isPopulated() const { return true; }
uint32 getSeq() const { return mSeq; }
void setSeq(uint32 s) { mSeq=s; }
const uint256& getNodeHash() const { return mHash; }
bool isEmpty() const { return mItems.empty(); }
int getItemCount() const { return mItems.size(); }
bool hasItem(const uint256& item) const;
SHAMapItem::pointer findItem(const uint256& tag);
SHAMapItem::pointer firstItem();
SHAMapItem::pointer lastItem();
SHAMapItem::pointer nextItem(const uint256& tag);
SHAMapItem::pointer prevItem(const uint256& tag);
virtual void dump();
enum TNType
{
ERROR =0,
INNER =1,
TRANSACTION =2,
ACCOUNT_STATE =3
};
class SHAMapInnerNode : public SHAMapNode
{
friend class SHAMap;
public:
typedef boost::shared_ptr<SHAMapInnerNode> pointer;
private:
uint256 mHash;
uint256 mHashes[32];
uint256 mHashes[16];
SHAMapItem::pointer mItem;
uint32 mSeq;
bool mFullBelow; // we have all nodes below this node
TNType mType;
bool mFullBelow;
bool updateHash();
SHAMapInnerNode(const SHAMapInnerNode&); // no implementation
SHAMapInnerNode& operator=(const SHAMapInnerNode&); // no implementation
protected:
bool setChildHash(int m, const uint256& hash);
SHAMapTreeNode(const SHAMapTreeNode&); // no implementation
SHAMapTreeNode& operator=(const SHAMapTreeNode&); // no implementation
public:
SHAMapInnerNode(const SHAMapNode& id, uint32 seq);
SHAMapInnerNode(const SHAMapInnerNode& node, uint32 seq);
SHAMapInnerNode(const SHAMapNode& id, const std::vector<unsigned char>& contents, uint32 seq);
SHAMapTreeNode(const SHAMapNode& nodeID, uint32 seq); // empty node
SHAMapTreeNode(const SHAMapTreeNode& node, uint32 seq); // copy node from older tree
SHAMapTreeNode(const SHAMapNode& nodeID, SHAMapItem::pointer item, TNType type, uint32 seq);
// raw node functions
SHAMapTreeNode(const SHAMapNode& id, const std::vector<unsigned char>& contents, uint32 seq); // raw node
void addRaw(Serializer &);
uint32 getSeq() const { return mSeq; }
void setSeq(uint32 s) { mSeq=s; }
virtual bool isPopulated() const { return true; }
// node functions
uint32 getSeq() const { return mSeq; }
void setSeq(uint32 s) { mSeq=s; }
const uint256& getNodeHash() const { return mHash; }
TNType getType() const { return mType; }
// type functions
bool isLeaf() const { return mType==TRANSACTION || mType==ACCOUNT_STATE; }
bool isInner() const { return mType==INNER; }
bool isValid() const { return mType!=ERROR; }
bool isTransaction() const { return mType!=TRANSACTION; }
bool isAccountState() const { return mType!=ACCOUNT_STATE; }
// inner node functions
bool isInnerNode() const { return !mItem; }
bool setChildHash(int m, const uint256& hash);
const uint256& getChildHash(int m) const;
bool isEmptyBranch(int m) const { return !mHashes[m]; }
int getBranchCount() const;
void makeInner();
// item node function
bool hasItem() const { return !!mItem; }
SHAMapItem::pointer peekItem() { return mItem; }
bool setItem(SHAMapItem::pointer& i, TNType type);
// sync functions
bool isFullBelow(void) const { return mFullBelow; }
void setFullBelow(void) { mFullBelow=true; }
bool isEmptyBranch(int m) const { return !mHashes[m]; }
const uint256& getNodeHash() const { return mHash; }
const uint256& getChildHash(int m) const;
bool isEmpty() const;
virtual void dump();
virtual std::string getString() const;
@@ -225,32 +206,27 @@ public:
private:
uint32 mSeq;
mutable boost::recursive_mutex mLock;
boost::unordered_map<SHAMapNode, SHAMapLeafNode::pointer, hash_SMN> mLeafByID;
boost::unordered_map<SHAMapNode, SHAMapInnerNode::pointer, hash_SMN> mInnerNodeByID;
boost::unordered_map<SHAMapNode, SHAMapTreeNode::pointer, hash_SMN> mTNByID;
boost::shared_ptr<std::map<SHAMapNode, SHAMapLeafNode::pointer> > mDirtyLeafNodes;
boost::shared_ptr<std::map<SHAMapNode, SHAMapInnerNode::pointer> > mDirtyInnerNodes;
boost::shared_ptr<std::map<SHAMapNode, SHAMapTreeNode::pointer> > mDirtyNodes;
SHAMapInnerNode::pointer root;
SHAMapTreeNode::pointer root;
bool mImmutable, mSynching;
protected:
void dirtyUp(const uint256& id);
SHAMapLeafNode::pointer createLeaf(const SHAMapInnerNode& lowestParent, const uint256& id);
SHAMapLeafNode::pointer checkCacheLeaf(const SHAMapNode&);
SHAMapLeafNode::pointer walkToLeaf(const uint256& id, bool create, bool modify);
SHAMapLeafNode::pointer getLeaf(const SHAMapNode& id, const uint256& hash, bool modify);
SHAMapLeafNode::pointer returnLeaf(SHAMapLeafNode::pointer leaf, bool modify);
void dirtyUp(std::stack<SHAMapTreeNode::pointer>& stack, const uint256& target, uint256 prevHash);
std::stack<SHAMapTreeNode::pointer> getStack(const uint256& id, bool include_nonmatching_leaf);
SHAMapTreeNode::pointer walkTo(const uint256& id, bool modify);
SHAMapTreeNode::pointer checkCacheNode(const SHAMapNode&);
void returnNode(SHAMapTreeNode::pointer&, bool modify);
SHAMapInnerNode::pointer checkCacheNode(const SHAMapNode&);
SHAMapInnerNode::pointer getInner(const SHAMapNode& id, const uint256& hash, bool modify);
SHAMapInnerNode::pointer returnNode(SHAMapInnerNode::pointer node, bool modify);
SHAMapInnerNode::pointer walkTo(const SHAMapNode& id);
SHAMapTreeNode::pointer getNode(const SHAMapNode& id);
SHAMapTreeNode::pointer getNode(const SHAMapNode& id, const uint256& hash, bool modify);
SHAMapItem::pointer firstBelow(SHAMapInnerNode::pointer);
SHAMapItem::pointer lastBelow(SHAMapInnerNode::pointer);
SHAMapItem::pointer firstBelow(SHAMapTreeNode::pointer);
SHAMapItem::pointer lastBelow(SHAMapTreeNode::pointer);
public:
@@ -260,32 +236,20 @@ public:
// hold the map stable across operations
ScopedLock Lock() const { return ScopedLock(mLock); }
// inner node access functions
bool hasInnerNode(const SHAMapNode& id);
bool giveInnerNode(SHAMapInnerNode::pointer);
SHAMapInnerNode::pointer getInnerNode(const SHAMapNode&);
// leaf node access functions
bool hasLeafNode(const SHAMapNode& id);
bool giveLeafNode(SHAMapLeafNode::pointer);
SHAMapLeafNode::pointer getLeafNode(const SHAMapNode&);
// generic node functions
std::vector<unsigned char> getRawNode(const SHAMapNode& id);
bool addRawNode(const SHAMapNode& nodeID, std::vector<unsigned char> rawNode);
bool hasNode(const SHAMapNode& id);
// normal hash access functions
bool hasItem(const uint256& id);
bool delItem(const uint256& id);
bool addItem(const SHAMapItem& i);
bool updateItem(const SHAMapItem& i);
bool addItem(const SHAMapItem& i, bool isTransaction);
bool updateItem(const SHAMapItem& i, bool isTransaction);
SHAMapItem getItem(const uint256& id);
uint256 getHash() const { return root->getNodeHash(); }
uint256 getHash() { return root->getNodeHash(); }
// save a copy if you have a temporary anyway
bool updateGiveItem(SHAMapItem::pointer);
bool addGiveItem(SHAMapItem::pointer);
bool updateGiveItem(SHAMapItem::pointer, bool isTransaction);
bool addGiveItem(SHAMapItem::pointer, bool isTransaction);
// save a copy if you only need a temporary
SHAMapItem::pointer peekItem(const uint256& id);

7
SHAMapDiff.cpp
View File

@@ -20,11 +20,15 @@ class SHAMapDiffNode
mNodeID(id), mOurHash(ourHash), mOtherHash(otherHash) { ; }
};
bool SHAMap::compare(SHAMap::pointer otherMap, SHAMapDiff& differences, int maxCount)
{ // compare two hash trees, add up to maxCount differences to the difference table
// return value: true=complete table of differences given, false=too many differences
// throws on corrupt tables or missing nodes
#if 0
// FIXME: Temporarily disabled
std::stack<SHAMapDiffNode> nodeStack; // track nodes we've pushed
nodeStack.push(SHAMapDiffNode(SHAMapNode(), getHash(), otherMap->getHash()));
@@ -143,5 +147,8 @@ bool SHAMap::compare(SHAMap::pointer otherMap, SHAMapDiff& differences, int maxC
}
}
}
#endif
return true;
}

353
SHAMapNodes.cpp
View File

@@ -3,12 +3,14 @@
#include <boost/foreach.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/smart_ptr/make_shared.hpp>
#include <openssl/sha.h>
#include "Serializer.h"
#include "BitcoinUtil.h"
#include "SHAMap.h"
std::string SHAMapNode::getString() const
{
std::string ret="NodeID(";
@@ -19,7 +21,7 @@ std::string SHAMapNode::getString() const
return ret;
}
uint256 SHAMapNode::smMasks[21];
uint256 SHAMapNode::smMasks[64];
bool SHAMapNode::operator<(const SHAMapNode &s) const
{
@@ -72,52 +74,57 @@ bool SHAMapNode::operator!=(const uint256 &s) const
void SHAMapNode::ClassInit()
{ // set up the depth masks
uint256 selector;
for(int i=0; i<=leafDepth; i++)
for(int i=0; i<64; i+=2)
{
smMasks[i]=selector;
*(selector.begin()+i)=0x1F;
*(selector.begin()+(i/2))=0x0F;
smMasks[i+1]=selector;
*(selector.begin()+(i/2))=0xFF;
}
}
uint256 SHAMapNode::getNodeID(int depth, const uint256& hash)
{
assert(depth>=0 && depth<=leafDepth);
assert(depth>=0 && depth<64);
return hash & smMasks[depth];
}
SHAMapNode::SHAMapNode(int depth, const uint256 &hash) : mDepth(depth)
{ // canonicalize the hash to a node ID for this depth
assert(depth>=0 && depth<=leafDepth);
assert(depth>=0 && depth<64);
mNodeID = getNodeID(depth, hash);
}
SHAMapNode SHAMapNode::getChildNodeID(int m) const
{ // This can be optimized to avoid the << if needed
assert(!isLeaf());
assert((m>=0) && (m<32));
assert((m>=0) && (m<16));
uint256 branch=m;
branch<<=mDepth*8;
branch<<=mDepth*4;
return SHAMapNode(mDepth+1, mNodeID | branch);
}
int SHAMapNode::selectBranch(const uint256& hash) const
{
if(isLeaf()) // no nodes under this node
{ // Which branch would contain the specified hash
if(mDepth==63)
{
assert(false);
return -1;
}
if((hash&smMasks[mDepth])!=mNodeID)
{
std::cerr << "selectBranch(" << getString() << std::endl;
std::cerr << " " << hash.GetHex() << " off branch" << std::endl;
assert(false);
return -1; // does not go under this node
}
uint256 selector(hash & smMasks[mDepth+1]);
int branch=*(selector.begin()+mDepth);
assert(branch>=0 && branch<32);
int branch=*(hash.begin()+(mDepth/2));
if(mDepth%2) branch>>=4;
else branch&=0xf;
assert(branch>=0 && branch<16);
return branch;
}
@@ -126,259 +133,211 @@ void SHAMapNode::dump() const
std::cerr << getString() << std::endl;
}
SHAMapLeafNode::SHAMapLeafNode(const SHAMapNode& nodeID, uint32 seq) : SHAMapNode(nodeID), mHash(0), mSeq(seq)
SHAMapTreeNode::SHAMapTreeNode(const SHAMapNode& nodeID, uint32 seq) : SHAMapNode(nodeID), mHash(0), mSeq(seq),
mType(ERROR), mFullBelow(false)
{
assert(nodeID.isLeaf());
}
SHAMapLeafNode::SHAMapLeafNode(const SHAMapLeafNode& node, uint32 seq) : SHAMapNode(node),
mHash(node.mHash), mItems(node.mItems), mSeq(seq)
SHAMapTreeNode::SHAMapTreeNode(const SHAMapTreeNode& node, uint32 seq) : SHAMapNode(node),
mHash(node.mHash), mItem(node.mItem), mSeq(seq), mType(node.mType), mFullBelow(false)
{
assert(node.isLeaf());
if(node.mItem)
mItem=boost::make_shared<SHAMapItem>(*node.mItem);
else
memcpy(mHashes, node.mHashes, sizeof(mHashes));
}
SHAMapLeafNode::SHAMapLeafNode(const SHAMapNode& id, const std::vector<unsigned char>& rawLeaf, uint32 seq)
: SHAMapNode(id), mSeq(seq)
SHAMapTreeNode::SHAMapTreeNode(const SHAMapNode& node, SHAMapItem::pointer item, TNType type, uint32 seq) :
SHAMapNode(node), mItem(item), mSeq(seq), mType(type), mFullBelow(true)
{
Serializer s(rawLeaf);
int pos=0;
while(pos<s.getLength())
{
uint256 id=s.get256(pos);
pos+=32;
uint16 len;
if(!s.get16(len, pos)) throw SHAMapException(InvalidNode);
pos+=2;
if(!id || !len || ((pos+len)>s.getLength())) throw SHAMapException(InvalidNode);
addUpdateItem(SHAMapItem::pointer(new SHAMapItem(id, s.getRaw(pos, len))), false);
pos+=len;
}
assert(item->peekData().size()>=32);
updateHash();
}
void SHAMapLeafNode::addRaw(Serializer &s)
SHAMapTreeNode::SHAMapTreeNode(const SHAMapNode& id, const std::vector<unsigned char>& rawNode, uint32 seq)
: SHAMapNode(id), mSeq(seq), mType(ERROR), mFullBelow(false)
{
BOOST_FOREACH(SHAMapItem::pointer& nodeItem, mItems)
{
s.add256(nodeItem->getTag());
s.add16(nodeItem->peekData().size());
s.addRaw(nodeItem->peekData());
Serializer s(rawNode);
int type=s.removeLastByte();
int len=s.getLength();
if( (type<0) || (type>3) || (len<33) ) throw SHAMapException(InvalidNode);
if(type==0)
{ // transaction
mItem=boost::make_shared<SHAMapItem>(s.getSHA512Half(), s.peekData());
mType=TRANSACTION;
}
else if(type==1)
{ // account state
uint160 u;
s.get160(u, len-20);
s.chop(20);
if(!u) throw SHAMapException(InvalidNode);
mItem=boost::make_shared<SHAMapItem>(u, s.peekData());
mType=ACCOUNT_STATE;
}
else if(type==2)
{ // full inner
if(len!=512) throw SHAMapException(InvalidNode);
for(int i=0; i<16; i++)
s.get256(mHashes[i], i*32);
mType=INNER;
}
else if(type==3)
{ // compressed inner
for(int i=0; i<(len/33); i++)
{
int pos;
s.get1(pos, 32+(i*33));
if( (pos<0) || (pos>=16)) throw SHAMapException(InvalidNode);
s.get256(mHashes[pos], i*33);
}
mType=INNER;
}
bool SHAMapLeafNode::hasItem(const uint256& item) const
{
BOOST_FOREACH(const SHAMapItem::pointer& nodeItem, mItems)
if(nodeItem->getTag()==item) return true;
return false;
updateHash();
}
bool SHAMapLeafNode::addUpdateItem(SHAMapItem::pointer item, bool doHash)
{ // The node will almost never have more than one item in it
#ifdef ST_DEBUG
std::cerr << "Leaf(" << getString() << ")" << std::endl;
std::cerr << " addi(" << item->getTag().GetHex() << std::endl;
#endif
std::list<SHAMapItem::pointer>::iterator it;
for(it=mItems.begin(); it!=mItems.end(); ++it)
void SHAMapTreeNode::addRaw(Serializer &s)
{
SHAMapItem &nodeItem=**it;
if(nodeItem.getTag()==item->getTag())
{
if(nodeItem.peekData()==item->peekData())
return false; // no change
nodeItem.updateData(item->peekData());
if(doHash) return updateHash();
return true;
}
if(nodeItem.getTag()>item->getTag())
{
mItems.insert(it, item);
if(doHash) return updateHash();
return true;
}
}
mItems.push_back(item);
if(mType==ERROR) throw SHAMapException(InvalidNode);
if(doHash) return updateHash();
return true;
if(mType==TRANSACTION)
{
mItem->addRaw(s);
s.add1(0);
assert(s.getLength()>32);
return;
}
bool SHAMapLeafNode::delItem(const uint256& tag)
if(mType==ACCOUNT_STATE)
{
std::list<SHAMapItem::pointer>::iterator it;
for(it=mItems.begin(); it!=mItems.end(); ++it)
{
if((*it)->getTag()==tag)
{
mItems.erase(it);
return updateHash();
}
}
return false;
mItem->addRaw(s);
assert(s.getLength()>20);
s.add160(mItem->getTag().to160());
s.add1(1);
return;
}
SHAMapItem::pointer SHAMapLeafNode::findItem(const uint256& tag)
if(getBranchCount()<5)
{ // compressed node
for(int i=0; i<16; i++)
if(!!mHashes[i])
{
BOOST_FOREACH(SHAMapItem::pointer& it, mItems)
if(it->getTag() == tag) return it;
return SHAMapItem::pointer();
s.add256(mHashes[i]);
s.add1(i);
}
s.add1(3);
return;
}
SHAMapItem::pointer SHAMapLeafNode::firstItem()
{
if(mItems.empty()) return SHAMapItem::pointer();
return *(mItems.begin());
for(int i=0; i<16; i++)
s.add256(mHashes[i]);
s.add1(2);
}
SHAMapItem::pointer SHAMapLeafNode::nextItem(const uint256& tag)
{
#ifdef ST_DEBUG
std::cerr << "LeafNode::nextItem(" << tag.GetHex() << std::endl;
BOOST_FOREACH(SHAMapItem::pointer& it, mItems)
std::cerr << " item(" << it->getTag().GetHex() << std::endl;
#endif
std::list<SHAMapItem::pointer>::iterator it;
for(it=mItems.begin(); it!=mItems.end(); ++it)
{
if((*it)->getTag()==tag)
{
if(++it==mItems.end()) return SHAMapItem::pointer();
return *it;
}
}
#ifdef DEBUG
std::cerr << "nextItem(!found)" << std::endl;
#endif
return SHAMapItem::pointer();
}
SHAMapItem::pointer SHAMapLeafNode::prevItem(const uint256& tag)
{
std::list<SHAMapItem::pointer>::reverse_iterator it;
for(it=mItems.rbegin(); it!=mItems.rend(); ++it)
{
if((*it)->getTag()==tag)
{
++it;
if(it==mItems.rend()) return SHAMapItem::pointer();
return *it;
}
}
return SHAMapItem::pointer();
}
SHAMapItem::pointer SHAMapLeafNode::lastItem()
{
if(mItems.empty()) return SHAMapItem::pointer();
return *(mItems.rbegin());
}
bool SHAMapLeafNode::updateHash()
bool SHAMapTreeNode::updateHash()
{
uint256 nh;
if(!mItems.empty())
if(mType==INNER)
{
bool empty=true;
for(int i=0; i<16; i++)
if(!!mHashes[i])
{
empty=false;
break;
}
if(!empty)
{
uint256 j[2];
SHA512(reinterpret_cast<unsigned char *>(mHashes), sizeof(mHashes), (unsigned char *) j);
nh=j[0];
assert(!!nh);
}
}
else if(mType==ACCOUNT_STATE)
{
Serializer s;
BOOST_FOREACH(const SHAMapItem::pointer &mi, mItems)
s.addRaw(mi->peekData());
mItem->addRaw(s);
s.add160(mItem->getTag().to160());
nh=s.getSHA512Half();
}
else if(mType==TRANSACTION)
nh=mItem->getTag();
else assert(false);
if(nh==mHash) return false;
mHash=nh;
return true;
}
void SHAMapLeafNode::dump()
bool SHAMapTreeNode::setItem(SHAMapItem::pointer& i, TNType type)
{
std::cerr << "SHAMapLeafNode(" << getNodeID().GetHex() << ")" << std::endl;
std::cerr << " " << mItems.size() << " items" << std::endl;
uint256 hash=getNodeHash();
mType=type;
mItem=i;
return getNodeHash()==hash;
}
SHAMapInnerNode::SHAMapInnerNode(const SHAMapNode& id, uint32 seq) : SHAMapNode(id), mSeq(seq), mFullBelow(false)
{ // can be root
assert(id.getDepth()<SHAMapNode::leafDepth);
}
SHAMapInnerNode::SHAMapInnerNode(const SHAMapNode& id, const std::vector<unsigned char>& contents, uint32 seq)
: SHAMapNode(id), mSeq(seq), mFullBelow(false)
int SHAMapTreeNode::getBranchCount() const
{
assert(!id.isLeaf());
assert(contents.size()==32*256/8);
Serializer s(contents);
for(int i=0; i<32; i++)
mHashes[i]=s.get256(i*32);
updateHash();
int ret=0;
for(int i=0; i<16; i++)
if(!mHashes[i]) ret++;
return ret;
}
SHAMapInnerNode::SHAMapInnerNode(const SHAMapInnerNode& node, uint32 seq) : SHAMapNode(node), mHash(node.mHash),
mSeq(seq), mFullBelow(false)
void SHAMapTreeNode::makeInner()
{
assert(!node.isLeaf());
memcpy(mHashes, node.mHashes, sizeof(mHashes));
updateHash();
mItem=SHAMapItem::pointer();
memset(mHashes, 0, sizeof(mHashes));
mType=INNER;
mHash.zero();
}
std::string SHAMapInnerNode::getString() const
void SHAMapTreeNode::dump()
{
std::cerr << "SHAMapTreeNode(" << getNodeID().GetHex() << ")" << std::endl;
}
std::string SHAMapTreeNode::getString() const
{
std::string ret="NodeID(";
ret+=boost::lexical_cast<std::string>(getDepth());
ret+=",";
ret+=getNodeID().GetHex();
ret+=")";
for(int i=0; i<32; i++)
if(isInner())
{
for(int i=0; i<16; i++)
if(!isEmptyBranch(i))
{
ret+=",b";
ret+=boost::lexical_cast<std::string>(i);
}
}
if(isLeaf())
{
ret+=",leaf";
}
return ret;
}
void SHAMapInnerNode::addRaw(Serializer &s)
bool SHAMapTreeNode::setChildHash(int m, const uint256 &hash)
{
for(int i=0; i<32; i++)
s.add256(mHashes[i]);
}
bool SHAMapInnerNode::setChildHash(int m, const uint256 &hash)
{
assert( (m>=0) && (m<32) );
assert( (m>=0) && (m<16) );
assert(mType==INNER);
if(mHashes[m]==hash)
return false;
mHashes[m]=hash;
return updateHash();
}
const uint256& SHAMapInnerNode::getChildHash(int m) const
const uint256& SHAMapTreeNode::getChildHash(int m) const
{
assert( (m>=0) && (m<32) );
assert( (m>=0) && (m<16) && (mType==INNER) );
return mHashes[m];
}
bool SHAMapInnerNode::isEmpty() const
{
for(int i=0; i<32; i++)
if(mHashes[i]!=0) return false;
return true;
}
bool SHAMapInnerNode::updateHash()
{
uint256 j[2];
if(!isEmpty())
SHA512(reinterpret_cast<unsigned char *>(mHashes), sizeof(mHashes), (unsigned char *) j);
if(mHash==j[0]) return false;
mHash=j[0];
return true;
}
void SHAMapInnerNode::dump()
{
std::cerr << "SHAMapInnerNode(" << getDepth() << ", " << getNodeID().GetHex() << ")" << std::endl;
int children=0;
for(int i=0; i<32; i++)
if(!!mHashes[i]) children++;
std::cerr << " " << children << " child(ren)" << std::endl;
}

257
SHAMapSync.cpp
View File

@@ -1,6 +1,8 @@
#include <stack>
#include <boost/make_shared.hpp>
#include <openssl/rand.h>
#include "SHAMap.h"
@@ -9,6 +11,8 @@ void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint2
{
boost::recursive_mutex::scoped_lock sl(mLock);
assert(root->isValid());
if(root->isFullBelow())
{
#ifdef GMN_DEBUG
@@ -17,46 +21,42 @@ void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint2
return;
}
std::stack<SHAMapInnerNode::pointer> stack;
if(!root->isInner())
{
std::cerr << "synching empty tree" << std::endl;
return;
}
std::stack<SHAMapTreeNode::pointer> stack;
stack.push(root);
while( (max>0) && (!stack.empty()) )
{
SHAMapInnerNode::pointer node=stack.top();
SHAMapTreeNode::pointer node=stack.top();
stack.pop();
#ifdef GMN_DEBUG
std::cerr << "gMN: popped " << node->getString() << std::endl;
#endif
for(int i=0; i<32; i++)
{
for(int i=0; i<16; i++)
if(!node->isEmptyBranch(i))
{
#ifdef GNMN_DEBUG
#ifdef GMN_DEBUG
std::cerr << "gMN: " << node->getString() << " has non-empty branch " << i << std::endl;
#endif
bool missing=false;
SHAMapNode childNID=node->getChildNodeID(i);
if(node->isChildLeaf())
{ // do we have this leaf node?
if(!getLeaf(childNID, node->getChildHash(i), false)) missing=true;
}
else
SHAMapTreeNode::pointer desc=getNode(node->getChildNodeID(i), node->getChildHash(i), false);
if(desc)
{
SHAMapInnerNode::pointer desc=getInner(childNID, node->getChildHash(i), false);
if(!desc)
missing=true;
else if(!desc->isFullBelow())
if(desc->isInner() && !desc->isFullBelow())
stack.push(desc);
}
if(missing && max-->0)
else if(max-- > 0)
{
#ifdef GMN_DEBUG
std::cerr << "gMN: need " << node->getChildNodeID(i).getString() << std::endl;
#endif
nodeIDs.push_back(childNID);
}
nodeIDs.push_back(node->getChildNodeID(i));
}
}
}
@@ -66,21 +66,11 @@ bool SHAMap::getNodeFat(const SHAMapNode& wanted, std::vector<SHAMapNode>& nodeI
std::list<std::vector<unsigned char> >& rawNodes)
{
boost::recursive_mutex::scoped_lock sl(mLock);
if(wanted.isLeaf())
{ // no fat way to get a leaf
SHAMapLeafNode::pointer leaf=getLeafNode(wanted);
if(!leaf) return false;
nodeIDs.push_back(*leaf);
Serializer s;
leaf->addRaw(s);
rawNodes.push_back(s.peekData());
return true;
}
SHAMapInnerNode::pointer node=getInnerNode(wanted);
SHAMapTreeNode::pointer node=getNode(wanted);
if(!node)
{
assert(false);
assert(false); // Remove for release, this can happen if we get a bogus request
return false;
}
@@ -92,46 +82,21 @@ bool SHAMap::getNodeFat(const SHAMapNode& wanted, std::vector<SHAMapNode>& nodeI
if(wanted.isRoot()) // don't get a fat root
return true;
bool ret=true;
for(int i=0; i<32; i++)
{
for(int i=0; i<16; i++)
if(!node->isEmptyBranch(i))
{
if(node->isChildLeaf())
SHAMapTreeNode::pointer nextNode=getNode(node->getChildNodeID(i), node->getChildHash(i), false);
assert(nextNode);
if(nextNode)
{
SHAMapLeafNode::pointer leaf=getLeaf(node->getChildNodeID(i), node->getChildHash(i), false);
if(!leaf)
{
assert(false);
ret=false;
}
else
{
nodeIDs.push_back(*leaf);
nodeIDs.push_back(*nextNode);
Serializer s;
leaf->addRaw(s);
nextNode->addRaw(s);
rawNodes.push_back(s.peekData());
}
}
else
{
SHAMapInnerNode::pointer ino=getInner(node->getChildNodeID(i), node->getChildHash(i), false);
if(!ino)
{
assert(false);
ret=false;
}
else
{
nodeIDs.push_back(*ino);
Serializer s;
ino->addRaw(s);
rawNodes.push_back(s.peekData());
}
}
}
}
return ret;
return true;
}
bool SHAMap::addRootNode(const std::vector<unsigned char>& rootNode)
@@ -139,7 +104,7 @@ bool SHAMap::addRootNode(const std::vector<unsigned char>& rootNode)
boost::recursive_mutex::scoped_lock sl(mLock);
// we already have a root node
if(root->getNodeHash()!=0)
if(!!root->getNodeHash())
{
#ifdef DEBUG
std::cerr << "got root node, already have one" << std::endl;
@@ -147,17 +112,19 @@ bool SHAMap::addRootNode(const std::vector<unsigned char>& rootNode)
return true;
}
SHAMapInnerNode::pointer node=SHAMapInnerNode::pointer(new SHAMapInnerNode(SHAMapNode(), rootNode, 0));
SHAMapTreeNode::pointer node=boost::make_shared<SHAMapTreeNode>(SHAMapNode(), rootNode, 0);
if(!node) return false;
#ifdef DEBUG
node->dump();
#endif
returnNode(root, true);
root=node;
mInnerNodeByID[*node]=node;
if(mDirtyInnerNodes) (*mDirtyInnerNodes)[*node]=node;
mTNByID[*root]=root;
if(!root->getNodeHash()) root->setFullBelow();
return true;
}
@@ -166,7 +133,7 @@ bool SHAMap::addRootNode(const uint256& hash, const std::vector<unsigned char>&
boost::recursive_mutex::scoped_lock sl(mLock);
// we already have a root node
if(root->getNodeHash()!=0)
if(!!root->getNodeHash())
{
#ifdef DEBUG
std::cerr << "got root node, already have one" << std::endl;
@@ -175,14 +142,15 @@ bool SHAMap::addRootNode(const uint256& hash, const std::vector<unsigned char>&
return true;
}
SHAMapInnerNode::pointer node=SHAMapInnerNode::pointer(new SHAMapInnerNode(SHAMapNode(), rootNode, 0));
SHAMapTreeNode::pointer node=boost::make_shared<SHAMapTreeNode>(SHAMapNode(), rootNode, 0);
if(!node) return false;
if(node->getNodeHash()!=hash) return false;
returnNode(root, true);
root=node;
mInnerNodeByID[*node]=node;
if(mDirtyInnerNodes) (*mDirtyInnerNodes)[*node]=node;
mTNByID[*root]=root;
if(!root->getNodeHash()) root->setFullBelow();
return true;
}
@@ -193,23 +161,19 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vector<unsigned cha
boost::recursive_mutex::scoped_lock sl(mLock);
if(node.isLeaf())
{
if(checkCacheLeaf(node)) return true;
}
else
{
if(checkCacheNode(node)) return true;
}
SHAMapInnerNode::pointer iNode=walkTo(node);
std::stack<SHAMapTreeNode::pointer> stack=getStack(node.getNodeID(), true);
if(stack.empty()) return false;
SHAMapTreeNode::pointer iNode=stack.top();
if(!iNode)
{ // we should always have a root
assert(false);
return true;
}
if(iNode->getDepth()==node.getDepth())
if(iNode->isLeaf() || (iNode->getDepth()==node.getDepth()))
{
#ifdef DEBUG
std::cerr << "got inner node, already had it (late)" << std::endl;
@@ -236,81 +200,46 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vector<unsigned cha
uint256 hash=iNode->getChildHash(branch);
if(!hash) return false;
if(node.isLeaf())
{ // leaf node
SHAMapLeafNode::pointer leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(node, rawNode, mSeq));
if( (leaf->getNodeHash()!=hash) || (node!=(*leaf)) )
{
#ifdef DEBUG
std::cerr << "leaf fails consistency check" << std::endl;
#endif
SHAMapTreeNode::pointer newNode=boost::make_shared<SHAMapTreeNode>(node, rawNode, mSeq);
if(hash!=newNode->getNodeHash()) // these aren't the droids we're looking for
return false;
}
mLeafByID[node]=leaf;
if(mDirtyLeafNodes) (*mDirtyLeafNodes)[node]=leaf;
// FIXME: This should check all sources
SHAMapInnerNode::pointer pNode=checkCacheNode(node.getParentNodeID());
if(!pNode)
mTNByID[*newNode]=newNode;
if(!newNode->isLeaf())
return true; // only a leaf can fill a branch
// did this new leaf cause its parents to fill up
do
{
assert(false);
return false;
}
for(int i=0; i<32; i++)
if(!checkCacheLeaf(pNode->getChildNodeID(i)))
return true;
pNode->setFullBelow();
while(!pNode->isRoot())
iNode=stack.top();
stack.pop();
assert(iNode->isInner());
for(int i=0; i<16; i++)
if(!iNode->isEmptyBranch(i))
{
pNode=checkCacheNode(pNode->getParentNodeID());
if(!pNode)
{
assert(false);
return false;
SHAMapTreeNode::pointer nextNode=getNode(iNode->getChildNodeID(i), iNode->getChildHash(i), false);
if(!nextNode) return true;
if(nextNode->isInner() && !nextNode->isFullBelow()) return true;
}
for(int i=0; i<32; i++)
if(!checkCacheNode(pNode->getChildNodeID(i)))
return true;
pNode->setFullBelow();
}
return true;
}
SHAMapInnerNode::pointer newNode=SHAMapInnerNode::pointer(new SHAMapInnerNode(node, rawNode, mSeq));
if( (newNode->getNodeHash()!=hash) || (node!=newNode->getNodeID()) )
{
#ifdef DEBUG
std::cerr << "inner node fails consistency check" << std::endl;
std::cerr << " Built: " << newNode->getString() << " h=" << newNode->getNodeHash().GetHex() << std::endl;
std::cerr << "Expected: " << node.getString() << " h=" << hash.GetHex() << std::endl;
#endif
return false;
}
mInnerNodeByID[node]=newNode;
if(mDirtyInnerNodes) (*mDirtyInnerNodes)[node]=newNode;
#ifdef ST_DEBUG
std::cerr << "Hooked: " << node.getString() << std::endl;
#endif
iNode->setFullBelow();
} while(!stack.empty());
return true;
}
bool SHAMap::deepCompare(SHAMap& other)
{ // Intended for debug/test only
std::stack<SHAMapInnerNode::pointer> stack;
std::stack<SHAMapTreeNode::pointer> stack;
boost::recursive_mutex::scoped_lock sl(mLock);
stack.push(root);
while(!stack.empty())
{
SHAMapInnerNode::pointer node=stack.top();
SHAMapTreeNode::pointer node=stack.top();
stack.pop();
SHAMapInnerNode::pointer otherNode;
SHAMapTreeNode::pointer otherNode;
if(node->isRoot()) otherNode=other.root;
else otherNode=other.getInner(*node, node->getNodeHash(), false);
else otherNode=other.getNode(*node, node->getNodeHash(), false);
if(!otherNode)
{
@@ -327,7 +256,22 @@ bool SHAMap::deepCompare(SHAMap& other)
std::cerr << "Comparing inner nodes " << node->getString() << std::endl;
#endif
for(int i=0; i<32; i++)
if(node->getNodeHash() != otherNode->getNodeHash())
return false;
if(node->isLeaf())
{
if(!otherNode->isLeaf())
return false;
if(node->peekItem()->getTag()!=otherNode->peekItem()->getTag())
return false;
if(node->peekItem()->getData()!=otherNode->peekItem()->getData())
return false;
}
else if(node->isInner())
{
if(!otherNode->isInner())
return false;
for(int i=0; i<16; i++)
{
if(node->isEmptyBranch(i))
{
@@ -336,29 +280,7 @@ bool SHAMap::deepCompare(SHAMap& other)
}
else
{
if(node->isChildLeaf())
{ //
SHAMapLeafNode::pointer leaf=getLeaf(node->getChildNodeID(i), node->getChildHash(i), false);
if(!leaf)
{
std::cerr << "unable to fetch leaf" << std::endl;
return false;
}
SHAMapLeafNode::pointer otherLeaf=other.getLeaf(*leaf, leaf->getNodeHash(), false);
if(!otherLeaf)
{
std::cerr << "unable to fetch other leaf" << std::endl;
return false;
}
if(leaf->getNodeHash()!=otherLeaf->getNodeHash())
{
std::cerr << "leaf hash mismatch" << std::endl;
return false;
}
}
else
{ // do we have this inner node?
SHAMapInnerNode::pointer next=getInner(node->getChildNodeID(i), node->getChildHash(i), false);
SHAMapTreeNode::pointer next=getNode(node->getChildNodeID(i), node->getChildHash(i), false);
if(!next)
{
std::cerr << "unable to fetch inner node" << std::endl;
@@ -385,15 +307,16 @@ bool SHAMap::syncTest()
for(int i=0; i<items; i++)
{
Serializer s;
int dlen=rand()%30+4;
int dlen=rand()%30+10;
for(int d=0; d<dlen; d++)
s.add32(rand());
uint256 id=s.getSHA512Half();
source.addItem(SHAMapItem(id, s.peekData()));
uint160 id=s.getRIPEMD160();
source.addItem(SHAMapItem(id, s.peekData()), false);
#ifdef ST_DEBUG
std::cerr << "Item: " << id.GetHex() << std::endl;
#endif
}
source.setImmutable();
#ifdef DEBUG
@@ -443,7 +366,7 @@ bool SHAMap::syncTest()
hashes.clear();
// get the list of nodes we know we need
destination.getMissingNodes(nodeIDs, hashes, 1024);
destination.getMissingNodes(nodeIDs, hashes, 512);
if(!nodeIDs.size()) break;
#ifdef DEBUG
@@ -491,7 +414,7 @@ bool SHAMap::syncTest()
destination.clearSynching();
#ifdef DEBUG
std::cerr << "SYNCHING COMPLETE" << std::endl;
std::cerr << "SYNCHING COMPLETE " << items << " items, " << nodes << " nodes" << std::endl;
#endif
if(!source.deepCompare(destination))

34
Serializer.cpp
View File

@@ -114,6 +114,40 @@ uint256 Serializer::get256(int offset) const
return ret;
}
int Serializer::add1(unsigned char byte)
{
int ret=mData.size();
mData.push_back(byte);
return ret;
}
bool Serializer::get1(int& byte, int offset) const
{
if(offset>=mData.size()) return false;
byte=mData[offset];
return true;
}
bool Serializer::chop(int bytes)
{
if(bytes>mData.size()) return false;
mData.resize(mData.size()-bytes);
return true;
}
int Serializer::removeLastByte()
{
int size=mData.size()-1;
if(size<0)
{
assert(false);
return -1;
}
int ret=mData[size];
mData.resize(size);
return ret;
}
bool Serializer::getRaw(std::vector<unsigned char>& o, int offset, int length) const
{
if((offset+length)>mData.size()) return false;

5
Serializer.h
View File

@@ -21,6 +21,7 @@ class Serializer
Serializer(const std::vector<unsigned char> &data) : mData(data) { ; }
// assemble functions
int add1(unsigned char byte);
int add16(uint16);
int add32(uint32); // ledger indexes, account sequence
int add64(uint64); // timestamps, amounts
@@ -30,6 +31,8 @@ class Serializer
int addRaw(const void *ptr, int len);
// disassemble functions
bool get1(int&, int offset) const;
bool get1(unsigned char&, int offset) const;
bool get16(uint16&, int offset) const;
bool get32(uint32&, int offset) const;
bool get64(uint64&, int offset) const;
@@ -53,6 +56,8 @@ class Serializer
std::vector<unsigned char> getData() const { return mData; }
void secureErase() { memset(&(mData.front()), 0, mData.size()); erase(); }
void erase() { mData.clear(); }
int removeLastByte();
bool chop(int num);
// signature functions
bool checkSignature(int pubkeyOffset, int signatureOffset) const;