ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2025-12-03 17:35:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Major rewrite of the SHAMap code. This code performs much better

Browse Source 
than the original version, particularly for smaller maps.
This commit is contained in:
JoelKatz
2012-02-05 06:54:44 -08:00
parent 41ce5fa7f9
commit 30c9bf0ed2
7 changed files with 753 additions and 912 deletions
Download Patch File Download Diff File Expand all files Collapse all files

797
SHAMap.cpp
View File

@@ -1,6 +1,9 @@
#include <stack>
#include <boost/foreach.hpp> #include <boost/foreach.hpp>
#include <boost/lexical_cast.hpp> #include <boost/lexical_cast.hpp>
#include <boost/smart_ptr/make_shared.hpp>
#include "Serializer.h" #include "Serializer.h"
#include "BitcoinUtil.h" #include "BitcoinUtil.h"
@@ -8,205 +11,144 @@
SHAMap::SHAMap(uint32 seq) : mSeq(seq), mImmutable(false), mSynching(false) SHAMap::SHAMap(uint32 seq) : mSeq(seq), mImmutable(false), mSynching(false)
{ {
root=SHAMapInnerNode::pointer(new SHAMapInnerNode(SHAMapNode(SHAMapNode::rootDepth, uint256()), mSeq)); root=boost::make_shared<SHAMapTreeNode>(SHAMapNode(0, uint256()), mSeq);
mInnerNodeByID[*root]=root; root->makeInner();
mTNByID[*root]=root;
} }
void SHAMap::dirtyUp(const uint256& id) std::stack<SHAMapTreeNode::pointer> SHAMap::getStack(const uint256& id, bool include_nonmatching_leaf)
{
// Walk the tree as far as possible to the specified identifier
// produce a stack of nodes along the way, with the terminal node at the top
std::stack<SHAMapTreeNode::pointer> stack;
SHAMapTreeNode::pointer node=root;
while(!node->isLeaf())
{
stack.push(node);
int branch=node->selectBranch(id);
assert(branch>=0);
uint256 hash=node->getChildHash(branch);
if(!hash) return stack;
node=getNode(node->getChildNodeID(branch), node->getChildHash(branch), false);
if(!node)
{
if(mSynching) return stack;
throw SHAMapException(MissingNode);
}
}
if(include_nonmatching_leaf || (node->peekItem()->getTag()==id))
stack.push(node);
return stack;
}
void SHAMap::dirtyUp(std::stack<SHAMapTreeNode::pointer>& stack, const uint256& target, uint256 prevHash)
{ // walk the tree up from through the inner nodes to the root { // walk the tree up from through the inner nodes to the root
// update linking hashes and add nodes to dirty list // update linking hashes and add nodes to dirty list
assert(!mImmutable && !mSynching); assert(!mImmutable && !mSynching);
SHAMapLeafNode::pointer leaf=checkCacheLeaf(SHAMapNode(SHAMapNode::leafDepth, id));
if(!leaf) throw SHAMapException(MissingNode);
uint256 hVal=leaf->getNodeHash(); while(!stack.empty())
if(mDirtyLeafNodes) mDirtyLeafNodes->insert(std::make_pair(SHAMapNode(*leaf), leaf));
if(!hVal)
{ {
#ifdef ST_DEBUG SHAMapTreeNode::pointer node=stack.top();
std::cerr << " erasingL " << leaf->getString() << std::endl; stack.pop();
#endif assert(node->isInnerNode());
mLeafByID.erase(*leaf);
}
for(int depth=SHAMapNode::leafDepth-1; depth>=0; depth--) int branch=node->selectBranch(target);
{ // walk up the tree to the root updating nodes assert(branch>=0);
SHAMapInnerNode::pointer node=checkCacheNode(SHAMapNode(depth, leaf->getNodeID()));
if(!node) throw SHAMapException(MissingNode); returnNode(node, true);
if(!node->setChildHash(node->selectBranch(id), hVal))
if(!node->setChildHash(branch, prevHash))
{ {
#ifdef ST_DEBUG std::cerr << "dirtyUp terminates early" << std::endl;
std::cerr << " no change@ " << node->getString() << std::endl; assert(false);
#endif
return; return;
} }
#ifdef ST_DEBUG #ifdef ST_DEBUG
std::cerr << "Dirty " << node->getString() << std::endl; std::cerr << "dirtyUp sets branch " << branch << " to " << prevHash.GetHex() << std::endl;
#endif #endif
if(mDirtyInnerNodes) mDirtyInnerNodes->insert(std::make_pair(SHAMapNode(*node), node)); prevHash=node->getNodeHash();
hVal=node->getNodeHash(); assert(!!prevHash);
if(!hVal)
{
#ifdef ST_DEBUG
std::cerr << " erasingN " << node->getString() << std::endl;
#endif
mInnerNodeByID.erase(*node);
}
} }
} }
SHAMapLeafNode::pointer SHAMap::checkCacheLeaf(const SHAMapNode& iNode) SHAMapTreeNode::pointer SHAMap::checkCacheNode(const SHAMapNode& iNode)
{ {
assert(iNode.isLeaf()); boost::unordered_map<SHAMapNode, SHAMapTreeNode::pointer>::iterator it=mTNByID.find(iNode);
boost::unordered_map<SHAMapNode, SHAMapLeafNode::pointer>::iterator it=mLeafByID.find(iNode); if(it==mTNByID.end()) return SHAMapTreeNode::pointer();
if(it==mLeafByID.end()) return SHAMapLeafNode::pointer();
return it->second; return it->second;
} }
SHAMapInnerNode::pointer SHAMap::checkCacheNode(const SHAMapNode& iNode) SHAMapTreeNode::pointer SHAMap::walkTo(const uint256& id, bool modify)
{ { // walk down to the terminal node for this ID
assert(!iNode.isLeaf());
boost::unordered_map<SHAMapNode, SHAMapInnerNode::pointer>::iterator it=mInnerNodeByID.find(iNode);
if(it==mInnerNodeByID.end()) return SHAMapInnerNode::pointer();
return it->second;
}
SHAMapTreeNode::pointer inNode=root;
SHAMapLeafNode::pointer SHAMap::walkToLeaf(const uint256& id, bool create, bool modify) while(!inNode->isLeaf())
{ // walk down to the leaf that would contain this ID
// is leaf node in cache
SHAMapLeafNode::pointer ln=checkCacheLeaf(SHAMapNode(SHAMapNode::leafDepth, id));
if(ln) return returnLeaf(ln, modify);
// walk tree to leaf
SHAMapInnerNode::pointer inNode=root;
for(int i=0; i<SHAMapNode::leafDepth; i++)
{ {
int branch=inNode->selectBranch(id); int branch=inNode->selectBranch(id);
if(branch<0) // somehow we got on the wrong branch if(inNode->isEmptyBranch(branch)) return inNode;
throw SHAMapException(InvalidNode); uint256 childHash=inNode->getChildHash(branch);
if(inNode->isEmptyBranch(branch)) if(!childHash) return inNode;
{ // no nodes below this one
if(!create) return SHAMapLeafNode::pointer();
return createLeaf(*inNode, id);
}
if(inNode->isChildLeaf())
{ // child is leaf node
ln=getLeaf(inNode->getChildNodeID(branch), inNode->getChildHash(branch), modify);
if(ln==NULL)
{
if(!create) return SHAMapLeafNode::pointer();
return createLeaf(*inNode, id);
}
}
else
{ // child is another inner node
inNode=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), modify);
if(inNode==NULL) throw SHAMapException(InvalidNode);
}
}
assert(ln || !create); SHAMapTreeNode::pointer nextNode=getNode(inNode->getChildNodeID(branch), childHash, false);
return returnLeaf(ln, modify); if(!nextNode) throw SHAMapException(MissingNode);
inNode=nextNode;
}
if(modify) returnNode(inNode, true);
return inNode;
} }
SHAMapInnerNode::pointer SHAMap::walkTo(const SHAMapNode& id) SHAMapTreeNode::pointer SHAMap::getNode(const SHAMapNode& id, const uint256& hash, bool modify)
{ // walk down to this ID, as far as possible { // retrieve a node whose node hash is known
SHAMapInnerNode::pointer inNode=root; SHAMapTreeNode::pointer node=checkCacheNode(id);
int i=0; if(node)
do
{ {
int branch=inNode->selectBranch(id.getNodeID()); if(node->getNodeHash()!=hash)
if(branch<0) // somehow we got on the wrong branch
throw SHAMapException(InvalidNode);
if (inNode->isEmptyBranch(branch)) // we know no branches below this one
{ {
#ifdef DEBUG #ifdef DEBUG
std::cerr << "Walk down empty branch" << std::endl; std::cerr << "Attempt to get node, hash not in tree" << std::endl;
std::cerr << "ID: " << id.getString() << std::endl;
std::cerr << "TgtHash " << hash.GetHex() << std::endl;
std::cerr << "NodHash " << node->getNodeHash().GetHex() << std::endl;
dump();
#endif #endif
return inNode; throw SHAMapException(InvalidNode);
} }
if(inNode->isChildLeaf()) // this is the last inner node returnNode(node, modify);
return inNode; return node;
}
try std::vector<unsigned char> nodeData;
{ if(!fetchNode(hash, nodeData)) return SHAMapTreeNode::pointer();
SHAMapInnerNode::pointer next=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), false);
if(!next) // we don't have the next node
{
#ifdef ST_DEBUG
std::cerr << "Unable to find node " << inNode->getChildNodeID(branch).getString() << std::endl;
#endif
return inNode;
}
assert(next->getDepth() == (inNode->getDepth()+1));
inNode=next;
}
catch (const SHAMapException& x)
{
if(x!=SHAMapException(MissingNode)) throw(x);
return inNode;
}
assert(i++ < SHAMapNode::leafDepth);
} while(1);
}
SHAMapLeafNode::pointer SHAMap::getLeaf(const SHAMapNode& id, const uint256& hash, bool modify) node=boost::make_shared<SHAMapTreeNode>(id, nodeData, mSeq);
{ // retrieve a leaf whose node hash is known
assert(!!hash);
if(!id.isLeaf()) return SHAMapLeafNode::pointer();
SHAMapLeafNode::pointer leaf=checkCacheLeaf(id);
if(leaf) return returnLeaf(leaf, modify);
std::vector<unsigned char> leafData;
if(!fetchNode(hash, leafData)) return SHAMapLeafNode::pointer();
leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(id, leafData, mSeq));
if(leaf->getNodeHash()!=hash) throw SHAMapException(InvalidNode);
mLeafByID.insert(std::make_pair(id, leaf));
return leaf;
}
SHAMapInnerNode::pointer SHAMap::getInner(const SHAMapNode& id, const uint256& hash, bool modify)
{ // retrieve an inner node whose node hash is known
SHAMapInnerNode::pointer node=checkCacheNode(id);
if(node) return returnNode(node, modify);
std::vector<unsigned char> rawNode;
if(!fetchNode(hash, rawNode)) return SHAMapInnerNode::pointer();
node=SHAMapInnerNode::pointer(new SHAMapInnerNode(id, rawNode, mSeq));
if(node->getNodeHash()!=hash) throw SHAMapException(InvalidNode); if(node->getNodeHash()!=hash) throw SHAMapException(InvalidNode);
mInnerNodeByID.insert(std::make_pair(id, node)); if(!mTNByID.insert(std::make_pair(id, node)).second)
if(id.getDepth()==0) root=node; assert(false);
return node; return node;
} }
SHAMapLeafNode::pointer SHAMap::returnLeaf(SHAMapLeafNode::pointer leaf, bool modify) void SHAMap::returnNode(SHAMapTreeNode::pointer& node, bool modify)
{ // make sure the leaf is suitable for the intended operation (copy on write)
if(leaf && modify && (leaf->getSeq()!=mSeq))
{
leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(*leaf, mSeq));
mLeafByID[*leaf]=leaf;
if(mDirtyLeafNodes) mDirtyLeafNodes->insert(std::make_pair(SHAMapNode(*leaf), leaf));
}
return leaf;
}
SHAMapInnerNode::pointer SHAMap::returnNode(SHAMapInnerNode::pointer node, bool modify)
{ // make sure the node is suitable for the intended operation (copy on write) { // make sure the node is suitable for the intended operation (copy on write)
assert(node->isValid());
if(node && modify && (node->getSeq()!=mSeq)) if(node && modify && (node->getSeq()!=mSeq))
{ {
#ifdef ST_DEBUG #ifdef DEBUG
std::cerr << "Node(" << node->getString() << ") bumpseq" << std::endl; std::cerr << "returnNode COW" << std::endl;
#endif #endif
node=SHAMapInnerNode::pointer(new SHAMapInnerNode(*node, mSeq)); if(mDirtyNodes) (*mDirtyNodes)[*node]=node;
mInnerNodeByID.insert(std::make_pair(SHAMapNode(*node), node)); node=boost::make_shared<SHAMapTreeNode>(*node, mSeq);
if(mDirtyInnerNodes) mDirtyInnerNodes->insert(std::make_pair(SHAMapNode(*node), node)); assert(node->isValid());
mTNByID[*node]=node;
} }
return node;
} }
SHAMapItem::SHAMapItem(const uint256& tag, const std::vector<unsigned char>& data) SHAMapItem::SHAMapItem(const uint256& tag, const std::vector<unsigned char>& data)
@@ -217,6 +159,57 @@ SHAMapItem::SHAMapItem(const uint160& tag, const std::vector<unsigned char>& dat
: mTag(tag.to256()), mData(data) : mTag(tag.to256()), mData(data)
{ ; } { ; }
SHAMapItem::pointer SHAMap::firstBelow(SHAMapTreeNode::pointer node)
{
// Return the first item below this node
#ifdef ST_DEBUG
std::cerr << "firstBelow(" << node->getString() << ")" << std::endl;
#endif
do
{ // Walk down the tree
if(node->hasItem()) return node->peekItem();
bool foundNode=false;
for(int i=0; i<16; i++)
if(!node->isEmptyBranch(i))
{
#ifdef ST_DEBUG
std::cerr << " FB: node " << node->getString() << std::endl;
std::cerr << " has non-empty branch " << i << " : " <<
node->getChildNodeID(i).getString() << ", " << node->getChildHash(i).GetHex() << std::endl;
#endif
node=getNode(node->getChildNodeID(i), node->getChildHash(i), false);
if(!node) throw SHAMapException(MissingNode);
foundNode=true;
break;
}
if(!foundNode) return SHAMapItem::pointer();
} while(1);
}
SHAMapItem::pointer SHAMap::lastBelow(SHAMapTreeNode::pointer node)
{
#ifdef DEBUG
std::cerr << "lastBelow(" << node->getString() << ")" << std::endl;
#endif
do
{ // Walk down the tree
if(node->hasItem()) return node->peekItem();
bool foundNode=false;
for(int i=16; i>=0; i++)
if(!node->isEmptyBranch(i))
{
node=getNode(node->getChildNodeID(i), node->getChildHash(i), false);
if(!node) throw SHAMapException(MissingNode);
foundNode=true;
break;
}
if(!foundNode) return SHAMapItem::pointer();
} while(1);
}
SHAMapItem::pointer SHAMap::peekFirstItem() SHAMapItem::pointer SHAMap::peekFirstItem()
{ {
boost::recursive_mutex::scoped_lock sl(mLock); boost::recursive_mutex::scoped_lock sl(mLock);
@@ -229,275 +222,242 @@ SHAMapItem::pointer SHAMap::peekLastItem()
return lastBelow(root); return lastBelow(root);
} }
SHAMapItem::pointer SHAMap::firstBelow(SHAMapInnerNode::pointer node)
{
#ifdef DEBUG
std::cerr << "firstBelow(" << node->getString() << ")" << std::endl;
#endif
int i;
while(!node->isChildLeaf())
{
for(i=0; i<32; i++)
if(!node->isEmptyBranch(i))
{
node=getInner(node->getChildNodeID(i), node->getChildHash(i), false);
if(!node) throw SHAMapException(MissingNode);
break;
}
if(i==32)
{
#ifdef ST_DEBUG
std::cerr << " node:" << node->getString() << " has no children" << std::endl;
#endif
return SHAMapItem::pointer();
}
}
assert(node->isChildLeaf());
for(int i=0; i<32; i++)
if(!node->isEmptyBranch(i))
{
SHAMapLeafNode::pointer mLeaf=getLeaf(node->getChildNodeID(i), node->getChildHash(i), false);
if(!mLeaf) throw SHAMapException(MissingNode);
SHAMapItem::pointer item=mLeaf->firstItem();
if(!item) throw SHAMapException(InvalidNode);
return item;
}
#ifdef ST_DEBUG
std::cerr << " node:" << node->getString() << " has no children" << std::endl;
#endif
return SHAMapItem::pointer();
}
SHAMapItem::pointer SHAMap::lastBelow(SHAMapInnerNode::pointer node)
{
boost::recursive_mutex::scoped_lock sl(mLock);
const uint256 zero;
int i;
while(!node->isChildLeaf())
{
for(i=31; i>=0; i--)
{
uint256 cHash(node->getChildHash(i));
if(cHash!=0)
{
node=getInner(node->getChildNodeID(i), cHash, false);
if(!node) return SHAMapItem::pointer();
break;
}
}
if(i<0) return SHAMapItem::pointer();
}
for(int i=31; i>=0; i--)
{
uint256 cHash=node->getChildHash(i);
if(cHash!=zero)
{
SHAMapLeafNode::pointer mLeaf=getLeaf(node->getChildNodeID(i), cHash, false);
if(!mLeaf) return SHAMapItem::pointer();
return mLeaf->lastItem();
}
}
return SHAMapItem::pointer();
}
SHAMapItem::pointer SHAMap::peekNextItem(const uint256& id) SHAMapItem::pointer SHAMap::peekNextItem(const uint256& id)
{ // Get a pointer to the next item in the tree after a given item - item must be in tree { // Get a pointer to the next item in the tree after a given item - item must be in tree
boost::recursive_mutex::scoped_lock sl(mLock); boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false); std::stack<SHAMapTreeNode::pointer> stack=getStack(id, true);
if(!leaf) while(!stack.empty())
{ {
#ifdef DEBUG SHAMapTreeNode::pointer node=stack.top();
std::cerr << "peekNextItem: current not found" << std::endl; stack.pop();
#endif
return SHAMapItem::pointer();
}
// is there another item in this leaf? (there almost never will be) if(node->isLeaf())
SHAMapItem::pointer next=leaf->nextItem(id);
if(next) return next;
for(int depth=SHAMapNode::leafDepth-1; depth>=0; depth--)
{ // walk up the tree until we find a node with a subsequent child
SHAMapInnerNode::pointer node=checkCacheNode(SHAMapNode(depth, id));
if(!node)
{ {
#ifdef DEBUG if(node->peekItem()->getTag()>id)
std::cerr << "InnerNode missing: " << SHAMapNode(depth,id).getString() << std::endl; return node->peekItem();
#endif
throw SHAMapException(MissingNode);
} }
#ifdef ST_DEBUG else for(int i=node->selectBranch(id)+1; i<16; i++)
std::cerr << " UpTo " << node->getString() << std::endl; if(!node->isEmptyBranch(i))
#endif {
for(int i=node->selectBranch(id)+1; i<32; i++) node=getNode(node->getChildNodeID(i), node->getChildHash(i), false);
if(!!node->getChildHash(i)) if(!node) throw SHAMapException(MissingNode);
{ // node has a subsequent child SHAMapItem::pointer item=firstBelow(node);
SHAMapNode nextNode(node->getChildNodeID(i)); if(!item) throw SHAMapException(MissingNode);
const uint256& nextHash(node->getChildHash(i)); return item;
if(nextNode.isLeaf())
{ // this is a terminal inner node
leaf=getLeaf(nextNode, nextHash, false);
if(!leaf) throw SHAMapException(MissingNode);
next=leaf->firstItem();
if(!next) throw SHAMapException(InvalidNode);
return next;
}
// the next item is the first item below this node
SHAMapInnerNode::pointer inner=getInner(nextNode, nextHash, false);
if(!inner) throw SHAMapException(MissingNode);
next=firstBelow(inner);
if(!next) throw SHAMapException(InvalidNode);
return next;
} }
} }
#ifdef ST_DEBUG
std::cerr << " peekNext off end" << std::endl;
#endif
// must be last item // must be last item
return SHAMapItem::pointer(); return SHAMapItem::pointer();
} }
SHAMapItem::pointer SHAMap::peekPrevItem(const uint256& id) SHAMapItem::pointer SHAMap::peekPrevItem(const uint256& id)
{ { // Get a pointer to the previous item in the tree after a given item - item must be in tree
boost::recursive_mutex::scoped_lock sl(mLock); boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false); std::stack<SHAMapTreeNode::pointer> stack=getStack(id, true);
if(!leaf) return SHAMapItem::pointer(); while(!stack.empty())
{
SHAMapTreeNode::pointer node=stack.top();
stack.pop();
// is there another item in this leaf? (there almost never will be) if(node->isLeaf())
SHAMapItem::pointer prev=leaf->prevItem(id);
if(prev) return prev;
for(int depth=SHAMapNode::leafDepth-1; depth>=0; depth--)
{ // walk up the tree until we find a node with a previous child
SHAMapInnerNode::pointer node=checkCacheNode(SHAMapNode(depth, id));
if(!node)
{ {
#ifdef DEBUG if(node->peekItem()->getTag()<id)
std::cerr << "InnerNode missing: " << SHAMapNode(depth,id).getString() << std::endl; return node->peekItem();
#endif
throw SHAMapException(MissingNode);
} }
for(int i=node->selectBranch(id)-1; i>=0; i--) else for(int i=node->selectBranch(id)-1; i>=0; i--)
if(!!node->getChildHash(i)) if(!node->isEmptyBranch(i))
{ // node has a subsequent child {
SHAMapNode prevNode(node->getChildNodeID(i)); node=getNode(node->getChildNodeID(i), node->getChildHash(i), false);
const uint256& prevHash(node->getChildHash(i)); if(!node) throw SHAMapException(MissingNode);
SHAMapItem::pointer item=firstBelow(node);
if(prevNode.isLeaf()) if(!item) throw SHAMapException(MissingNode);
{ // this is a terminal inner node return item;
leaf=getLeaf(prevNode, prevHash, false);
if(!leaf) throw SHAMapException(MissingNode);
prev=leaf->firstItem();
if(!prev) throw SHAMapException(InvalidNode);
return prev;
}
// the next item is the first item below this node
SHAMapInnerNode::pointer inner=getInner(prevNode, prevHash, false);
if(!inner) throw SHAMapException(MissingNode);
prev=lastBelow(inner);
if(!prev) throw SHAMapException(InvalidNode);
return prev;
} }
} }
// must be last item // must be last item
return SHAMapItem::pointer(); return SHAMapItem::pointer();
} }
SHAMapLeafNode::pointer SHAMap::createLeaf(const SHAMapInnerNode& lowestParent, const uint256& id)
{ // caller must call dirtyUp if they populate the leaf
#ifdef ST_DEBUG
std::cerr << "createLeaf(" << lowestParent.getString() << std::endl;
std::cerr << " for " << id.GetHex() << ")" << std::endl;
#endif
assert(!!id);
for(int depth=lowestParent.getDepth()+1; depth<SHAMapNode::leafDepth; depth++)
{
SHAMapInnerNode::pointer newNode(new SHAMapInnerNode(SHAMapNode(depth, id), mSeq));
mInnerNodeByID[*newNode]=newNode;
}
SHAMapLeafNode::pointer newLeaf(new SHAMapLeafNode(SHAMapNode(SHAMapNode::leafDepth, id), mSeq));
mLeafByID[*newLeaf]=newLeaf;
#ifdef ST_DEBUG
std::cerr << "made leaf " << newLeaf->getString() << std::endl;
#endif
return newLeaf;
}
SHAMapItem::pointer SHAMap::peekItem(const uint256& id) SHAMapItem::pointer SHAMap::peekItem(const uint256& id)
{ {
boost::recursive_mutex::scoped_lock sl(mLock); boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false); SHAMapTreeNode::pointer leaf=walkTo(id, false);
if(!leaf) return SHAMapItem::pointer(); if(!leaf) return SHAMapItem::pointer();
return leaf->findItem(id); return leaf->peekItem();
} }
bool SHAMap::hasItem(const uint256& id) bool SHAMap::hasItem(const uint256& id)
{ // does the tree have an item with this ID { // does the tree have an item with this ID
boost::recursive_mutex::scoped_lock sl(mLock); boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false); SHAMapTreeNode::pointer leaf=walkTo(id, false);
if(!leaf) return false; if(!leaf) return false;
return leaf->hasItem(id); SHAMapItem::pointer item=leaf->peekItem();
if(!item || item->getTag()!=id) return false;
return true;
} }
bool SHAMap::delItem(const uint256& id) bool SHAMap::delItem(const uint256& id)
{ // delete the item with this ID { // delete the item with this ID
boost::recursive_mutex::scoped_lock sl(mLock); boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false);
if(!leaf) return false; std::stack<SHAMapTreeNode::pointer> stack=getStack(id, false);
if(!leaf->delItem(id)) return false; if(stack.empty()) throw SHAMapException(MissingNode);
dirtyUp(id);
SHAMapTreeNode::pointer leaf=stack.top();
stack.pop();
if( !leaf || !leaf->hasItem() || (leaf->peekItem()->getTag()!=id) )
return false;
SHAMapTreeNode::TNType type=leaf->getType();
returnNode(leaf, true);
mTNByID.erase(*leaf);
uint256 prevHash;
while(!stack.empty())
{
SHAMapTreeNode::pointer node=stack.top();
stack.pop();
returnNode(node, true);
if(!node->setChildHash(node->selectBranch(id), prevHash))
return true;
if(!prevHash && !node->isRoot())
{ // we may have made this a node with 1 or 0 children
int bc=node->getBranchCount();
if(bc==0)
{
prevHash=uint256();
mTNByID.erase(*node);
}
else if(bc==1)
{ // pull up on the thread
SHAMapItem::pointer item=firstBelow(node);
assert(item);
node->setItem(item, type);
prevHash=node->getNodeHash();
}
}
else prevHash=node->getNodeHash();
}
return true; return true;
} }
bool SHAMap::addGiveItem(const SHAMapItem::pointer item) bool SHAMap::addGiveItem(SHAMapItem::pointer item, bool isTransaction)
{ // add the specified item, does not update { // add the specified item, does not update
boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(item->getTag(), true, true);
if(!leaf)
{
assert(false);
return false;
}
if(leaf->hasItem(item->getTag()))
{
#ifdef ST_DEBUG #ifdef ST_DEBUG
std::cerr << "leaf has item we're adding" << std::endl; std::cerr << "aGI " << item->getTag().GetHex() << std::endl;
#endif #endif
return false;
} uint256 tag=item->getTag();
if(!leaf->addUpdateItem(item, true)) SHAMapTreeNode::TNType type=isTransaction ? SHAMapTreeNode::TRANSACTION : SHAMapTreeNode::ACCOUNT_STATE;
boost::recursive_mutex::scoped_lock sl(mLock);
std::stack<SHAMapTreeNode::pointer> stack=getStack(tag, true);
if(stack.empty()) throw SHAMapException(MissingNode);
SHAMapTreeNode::pointer node=stack.top();
stack.pop();
if( node->isLeaf() && (node->peekItem()->getTag()==tag) )
{ {
assert(false); std::cerr << "addGiveItem ends on leaf with same tag" << std::endl;
return false; return false;
} }
dirtyUp(item->getTag());
uint256 prevHash;
returnNode(node, true);
if(node->isInner())
{ // easy case, we end on an inner node
#ifdef ST_DEBUG
std::cerr << "aGI inner " << node->getString() << std::endl;
#endif
int branch=node->selectBranch(tag);
assert(node->isEmptyBranch(branch));
SHAMapTreeNode::pointer newNode=
boost::make_shared<SHAMapTreeNode>(node->getChildNodeID(branch), item, type, mSeq);
if(!mTNByID.insert(std::make_pair(SHAMapNode(*newNode), newNode)).second)
{
std::cerr << "Node: " << node->getString() << std::endl;
std::cerr << "NewNode: " << newNode->getString() << std::endl;
assert(false);
throw SHAMapException(InvalidNode);
}
node->setChildHash(branch, newNode->getNodeHash());
}
else
{ // this is a leaf node that has to be made an inner node holding two items
#ifdef ST_DEBUG
std::cerr << "aGI leaf " << node->getString() << std::endl;
#endif
SHAMapItem::pointer otherItem=node->peekItem();
assert(otherItem && (tag!=otherItem->getTag()) );
node->makeInner();
int b1, b2;
while( (b1=node->selectBranch(tag)) == (b2=node->selectBranch(otherItem->getTag())) )
{ // we need a new inner node, since both go on same branch at this level
#ifdef ST_DEBUG
std::cerr << "need new inner node at " << node->getDepth() << std::endl;
#endif
SHAMapTreeNode::pointer newNode=boost::make_shared<SHAMapTreeNode>(node->getChildNodeID(b1), mSeq);
newNode->makeInner();
if(!mTNByID.insert(std::make_pair(SHAMapNode(*newNode), newNode)).second)
assert(false);
stack.push(node);
node=newNode;
}
// we can add the two leaf nodes here
assert(node->isInner());
SHAMapTreeNode::pointer newNode=
boost::make_shared<SHAMapTreeNode>(node->getChildNodeID(b1), item, type, mSeq);
assert(newNode->isValid() && newNode->isLeaf());
if(!mTNByID.insert(std::make_pair(SHAMapNode(*newNode), newNode)).second)
assert(false);
node->setChildHash(b1, newNode->getNodeHash()); // OPTIMIZEME hash op not needed
newNode=boost::make_shared<SHAMapTreeNode>(node->getChildNodeID(b2), otherItem, type, mSeq);
assert(newNode->isValid() && newNode->isLeaf());
if(!mTNByID.insert(std::make_pair(SHAMapNode(*newNode), newNode)).second)
assert(false);
node->setChildHash(b2, newNode->getNodeHash());
}
prevHash=node->getNodeHash();
assert(!!prevHash);
dirtyUp(stack, tag, prevHash);
return true; return true;
} }
bool SHAMap::addItem(const SHAMapItem& i) bool SHAMap::addItem(const SHAMapItem& i, bool isTransaction)
{ {
return addGiveItem(SHAMapItem::pointer(new SHAMapItem(i))); return addGiveItem(boost::make_shared<SHAMapItem>(i), isTransaction);
} }
bool SHAMap::updateGiveItem(SHAMapItem::pointer item) bool SHAMap::updateGiveItem(SHAMapItem::pointer item, bool isTransaction)
{ { // can't change the tag but can change the hash
uint256 tag=item->getTag();
boost::recursive_mutex::scoped_lock sl(mLock); boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapLeafNode::pointer leaf=walkToLeaf(item->getTag(), true, true);
if(!leaf) return false; std::stack<SHAMapTreeNode::pointer> stack=getStack(tag, true);
if(!leaf->addUpdateItem(item, true)) return false; if(stack.empty()) throw SHAMapException(MissingNode);
dirtyUp(item->getTag());
SHAMapTreeNode::pointer node=stack.top();
stack.pop();
if(!node->isLeaf() || (node->peekItem()->getTag()==tag) )
return false;
returnNode(node, true);
if(!node->setItem(item, isTransaction ? SHAMapTreeNode::TRANSACTION : SHAMapTreeNode::ACCOUNT_STATE))
return true;
dirtyUp(stack, tag, node->getNodeHash());
return true; return true;
} }
@@ -519,28 +479,15 @@ int SHAMap::flushDirty(int maxNodes, HashedObjectType t, uint32 seq)
int flushed=0; int flushed=0;
Serializer s; Serializer s;
if(mDirtyLeafNodes) if(mDirtyNodes)
{ {
while(!mDirtyLeafNodes->empty()) while(!mDirtyNodes->empty())
{ {
SHAMapLeafNode::pointer& dln=mDirtyLeafNodes->begin()->second; SHAMapTreeNode::pointer& din=mDirtyNodes->begin()->second;
s.erase();
dln->addRaw(s);
HashedObject::store(t, seq, s.peekData(), s.getSHA512Half());
mDirtyLeafNodes->erase(mDirtyLeafNodes->begin());
if(flushed++>=maxNodes) return flushed;
}
}
if(mDirtyInnerNodes)
{
while(!mDirtyInnerNodes->empty())
{
SHAMapInnerNode::pointer& din=mDirtyInnerNodes->begin()->second;
s.erase(); s.erase();
din->addRaw(s); din->addRaw(s);
HashedObject::store(t, seq, s.peekData(), s.getSHA512Half()); HashedObject::store(t, seq, s.peekData(), s.getSHA512Half());
mDirtyInnerNodes->erase(mDirtyInnerNodes->begin()); mDirtyNodes->erase(mDirtyNodes->begin());
if(flushed++>=maxNodes) return flushed; if(flushed++>=maxNodes) return flushed;
} }
} }
@@ -548,48 +495,30 @@ int SHAMap::flushDirty(int maxNodes, HashedObjectType t, uint32 seq)
return flushed; return flushed;
} }
SHAMapInnerNode::pointer SHAMap::getInnerNode(const SHAMapNode& node) SHAMapTreeNode::pointer SHAMap::getNode(const SHAMapNode& nodeID)
{ {
boost::recursive_mutex::scoped_lock sl(mLock); boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapInnerNode::pointer inNode=root;
for(int i=0; i<SHAMapNode::leafDepth; i++)
{
if(inNode->getDepth()==node.getDepth())
{
if((*inNode)!=node) return SHAMapInnerNode::pointer();
return inNode;
}
int branch=inNode->selectBranch(node.getNodeID());
if( (branch<0) || (inNode->isEmptyBranch(branch)) ) return SHAMapInnerNode::pointer();
inNode=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), false);
if(!inNode) return inNode;
}
if(inNode->getDepth()==node.getDepth())
{
if((*inNode)!=node) return SHAMapInnerNode::pointer();
return inNode;
}
return SHAMapInnerNode::pointer();
}
SHAMapLeafNode::pointer SHAMap::getLeafNode(const SHAMapNode& leaf) SHAMapTreeNode::pointer node=checkCacheNode(nodeID);
{ if(node) return node;
boost::recursive_mutex::scoped_lock sl(mLock);
SHAMapInnerNode::pointer inNode=root; node=root;
for(int i=0; inNode->getDepth()<SHAMapNode::leafDepth; i++) while(nodeID!=*node)
{ {
int branch=inNode->selectBranch(leaf.getNodeID()); int branch=node->selectBranch(nodeID.getNodeID());
if( (branch<0) || (inNode->isEmptyBranch(branch)) ) return SHAMapLeafNode::pointer(); assert(branch>=0);
inNode=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), false); if( (branch<0) || (node->isEmptyBranch(branch)) )
if(!inNode) return SHAMapLeafNode::pointer(); return SHAMapTreeNode::pointer();
node=getNode(node->getChildNodeID(branch), node->getChildHash(branch), false);
if(!node) throw SHAMapException(MissingNode);
} }
int branch=inNode->selectBranch(leaf.getNodeID()); return node;
if( (branch<0) || (inNode->isEmptyBranch(branch)) ) return SHAMapLeafNode::pointer();
return getLeaf(inNode->getChildNodeID(branch), inNode->getChildHash(branch), false);
} }
void SHAMap::dump() void SHAMap::dump()
{ {
#if 0
std::cerr << "SHAMap::dump" << std::endl; std::cerr << "SHAMap::dump" << std::endl;
SHAMapItem::pointer i=peekFirstItem(); SHAMapItem::pointer i=peekFirstItem();
while(i) while(i)
@@ -598,12 +527,23 @@ void SHAMap::dump()
i=peekNextItem(i->getTag()); i=peekNextItem(i->getTag());
} }
std::cerr << "SHAMap::dump done" << std::endl; std::cerr << "SHAMap::dump done" << std::endl;
#endif
std::cerr << " MAP Contains" << std::endl;
boost::recursive_mutex::scoped_lock sl(mLock);
for(boost::unordered_map<SHAMapNode, SHAMapTreeNode::pointer, hash_SMN>::iterator it=mTNByID.begin();
it!=mTNByID.end(); ++it)
{
std::cerr << it->second->getString() << std::endl;
}
} }
static std::vector<unsigned char>IntToVUC(int i) static std::vector<unsigned char>IntToVUC(int i)
{ {
std::vector<unsigned char> vuc; std::vector<unsigned char> vuc;
vuc.push_back((unsigned char) i); for(int i=0; i<32; i++)
vuc.push_back((unsigned char) i);
return vuc; return vuc;
} }
@@ -619,19 +559,29 @@ bool SHAMap::TestSHAMap()
SHAMap sMap; SHAMap sMap;
SHAMapItem i1(h1, IntToVUC(1)), i2(h2, IntToVUC(2)), i3(h3, IntToVUC(3)), i4(h4, IntToVUC(4)), i5(h5, IntToVUC(5)); SHAMapItem i1(h1, IntToVUC(1)), i2(h2, IntToVUC(2)), i3(h3, IntToVUC(3)), i4(h4, IntToVUC(4)), i5(h5, IntToVUC(5));
sMap.addItem(i2); if(!sMap.addItem(i2, true))
sMap.addItem(i1); {
assert(false);
return false;
}
if(!sMap.addItem(i1, true))
{
assert(false);
return false;
}
SHAMapItem::pointer i=sMap.peekFirstItem(); SHAMapItem::pointer i;
i=sMap.peekFirstItem();
assert(!!i && (*i==i1)); assert(!!i && (*i==i1));
i=sMap.peekNextItem(i->getTag()); i=sMap.peekNextItem(i->getTag());
assert(!!i && (*i==i2)); assert(!!i && (*i==i2));
i=sMap.peekNextItem(i->getTag()); i=sMap.peekNextItem(i->getTag());
assert(!i); assert(!i);
sMap.addItem(i4); sMap.addItem(i4, true);
sMap.delItem(i2.getTag()); sMap.delItem(i2.getTag());
sMap.addItem(i3); sMap.addItem(i3, true);
i=sMap.peekFirstItem(); i=sMap.peekFirstItem();
assert(!!i && (*i==i1)); assert(!!i && (*i==i1));
@@ -642,9 +592,8 @@ bool SHAMap::TestSHAMap()
i=sMap.peekNextItem(i->getTag()); i=sMap.peekNextItem(i->getTag());
assert(!i); assert(!i);
if(!syncTest()); if(!syncTest())
return false; return false;
return true; return true;
} }

170
SHAMap.h
View File

@@ -3,7 +3,7 @@
#include <list> #include <list>
#include <map> #include <map>
#include <deque> #include <stack>
#include <boost/shared_ptr.hpp> #include <boost/shared_ptr.hpp>
#include <boost/enable_shared_from_this.hpp> #include <boost/enable_shared_from_this.hpp>
@@ -18,7 +18,6 @@
class SHAMap; class SHAMap;
// A tree-like map of SHA256 hashes // A tree-like map of SHA256 hashes
// 21 levels, consisting of a root, 19 interior node levels, and leaves
// The trees are designed for rapid synchronization and compression of differences // The trees are designed for rapid synchronization and compression of differences
@@ -28,29 +27,27 @@ public:
typedef boost::shared_ptr<SHAMapNode> pointer; typedef boost::shared_ptr<SHAMapNode> pointer;
private: private:
static uint256 smMasks[21]; // AND with hash to get node id static uint256 smMasks[64]; // AND with hash to get node id
uint256 mNodeID; uint256 mNodeID;
int mDepth; int mDepth;
public: public:
// 0 is root, 20 is leaf
static const int rootDepth=0; static const int rootDepth=0;
static const int leafDepth=20;
SHAMapNode() : mDepth(0) { ; } SHAMapNode() : mDepth(0) { ; }
SHAMapNode(int depth, const uint256& hash); SHAMapNode(int depth, const uint256& hash);
int getDepth() const { return mDepth; } int getDepth() const { return mDepth; }
const uint256& getNodeID() const { return mNodeID; } const uint256& getNodeID() const { return mNodeID; }
bool isRoot() const { return mDepth==0; }
bool isLeaf() const { return mDepth==leafDepth; }
bool isChildLeaf() const { return mDepth==(leafDepth-1); }
bool isInner() const { return !isRoot() && !isLeaf(); }
virtual bool isPopulated() const { return false; } virtual bool isPopulated() const { return false; }
SHAMapNode getParentNodeID() const { return SHAMapNode(mDepth-1, mNodeID); } SHAMapNode getParentNodeID() const
{
assert(mDepth);
return SHAMapNode(mDepth-1, mNodeID);
}
SHAMapNode getChildNodeID(int m) const; SHAMapNode getChildNodeID(int m) const;
int selectBranch(const uint256& hash) const; int selectBranch(const uint256& hash) const;
@@ -62,6 +59,7 @@ public:
bool operator!=(const uint256&) const; bool operator!=(const uint256&) const;
bool operator<=(const SHAMapNode&) const; bool operator<=(const SHAMapNode&) const;
bool operator>=(const SHAMapNode&) const; bool operator>=(const SHAMapNode&) const;
bool isRoot() const { return mDepth==0; }
virtual std::string getString() const; virtual std::string getString() const;
void dump() const; void dump() const;
@@ -100,6 +98,8 @@ public:
const uint256& getTag() const { return mTag; } const uint256& getTag() const { return mTag; }
std::vector<unsigned char> getData() const { return mData; } std::vector<unsigned char> getData() const { return mData; }
const std::vector<unsigned char>& peekData() const { return mData; } const std::vector<unsigned char>& peekData() const { return mData; }
void addRaw(Serializer &s) { s.addRaw(mData); }
void addRaw(std::vector<unsigned char>& s) { s.insert(s.end(), mData.begin(), mData.end()); }
void updateData(const std::vector<unsigned char>& data) { mData=data; } void updateData(const std::vector<unsigned char>& data) { mData=data; }
@@ -118,93 +118,74 @@ public:
virtual void dump(); virtual void dump();
}; };
class SHAMapLeafNode : public SHAMapNode class SHAMapTreeNode : public SHAMapNode
{ {
friend class SHAMap; friend class SHAMap;
public: public:
typedef boost::shared_ptr<SHAMapLeafNode> pointer; typedef boost::shared_ptr<SHAMapTreeNode> pointer;
enum TNType
{
ERROR =0,
INNER =1,
TRANSACTION =2,
ACCOUNT_STATE =3
};
private: private:
uint256 mHash; uint256 mHash;
std::list<SHAMapItem::pointer> mItems; uint256 mHashes[16];
SHAMapItem::pointer mItem;
uint32 mSeq; uint32 mSeq;
TNType mType;
bool mFullBelow;
bool updateHash(); bool updateHash();
SHAMapLeafNode(const SHAMapLeafNode&); // no implementation SHAMapTreeNode(const SHAMapTreeNode&); // no implementation
SHAMapLeafNode& operator=(const SHAMapLeafNode&); // no implementation SHAMapTreeNode& operator=(const SHAMapTreeNode&); // no implementation
protected:
bool addUpdateItem(SHAMapItem::pointer, bool doHash);
bool delItem(const SHAMapItem::pointer i) { return delItem(i->getTag()); }
bool delItem(const uint256& tag);
public: public:
SHAMapLeafNode(const SHAMapNode& nodeID, uint32 seq); SHAMapTreeNode(const SHAMapNode& nodeID, uint32 seq); // empty node
SHAMapLeafNode(const SHAMapLeafNode& node, uint32 seq); SHAMapTreeNode(const SHAMapTreeNode& node, uint32 seq); // copy node from older tree
SHAMapLeafNode(const SHAMapNode& id, const std::vector<unsigned char>& contents, uint32 seq); SHAMapTreeNode(const SHAMapNode& nodeID, SHAMapItem::pointer item, TNType type, uint32 seq);
// raw node functions
SHAMapTreeNode(const SHAMapNode& id, const std::vector<unsigned char>& contents, uint32 seq); // raw node
void addRaw(Serializer &); void addRaw(Serializer &);
virtual bool isPopulated() const { return true; } virtual bool isPopulated() const { return true; }
// node functions
uint32 getSeq() const { return mSeq; } uint32 getSeq() const { return mSeq; }
void setSeq(uint32 s) { mSeq=s; } void setSeq(uint32 s) { mSeq=s; }
const uint256& getNodeHash() const { return mHash; } const uint256& getNodeHash() const { return mHash; }
bool isEmpty() const { return mItems.empty(); } TNType getType() const { return mType; }
int getItemCount() const { return mItems.size(); }
bool hasItem(const uint256& item) const; // type functions
SHAMapItem::pointer findItem(const uint256& tag); bool isLeaf() const { return mType==TRANSACTION || mType==ACCOUNT_STATE; }
SHAMapItem::pointer firstItem(); bool isInner() const { return mType==INNER; }
SHAMapItem::pointer lastItem(); bool isValid() const { return mType!=ERROR; }
SHAMapItem::pointer nextItem(const uint256& tag); bool isTransaction() const { return mType!=TRANSACTION; }
SHAMapItem::pointer prevItem(const uint256& tag); bool isAccountState() const { return mType!=ACCOUNT_STATE; }
virtual void dump(); // inner node functions
}; bool isInnerNode() const { return !mItem; }
class SHAMapInnerNode : public SHAMapNode
{
friend class SHAMap;
public:
typedef boost::shared_ptr<SHAMapInnerNode> pointer;
private:
uint256 mHash;
uint256 mHashes[32];
uint32 mSeq;
bool mFullBelow; // we have all nodes below this node
bool updateHash();
SHAMapInnerNode(const SHAMapInnerNode&); // no implementation
SHAMapInnerNode& operator=(const SHAMapInnerNode&); // no implementation
protected:
bool setChildHash(int m, const uint256& hash); bool setChildHash(int m, const uint256& hash);
const uint256& getChildHash(int m) const;
bool isEmptyBranch(int m) const { return !mHashes[m]; }
int getBranchCount() const;
void makeInner();
public: // item node function
SHAMapInnerNode(const SHAMapNode& id, uint32 seq); bool hasItem() const { return !!mItem; }
SHAMapInnerNode(const SHAMapInnerNode& node, uint32 seq); SHAMapItem::pointer peekItem() { return mItem; }
SHAMapInnerNode(const SHAMapNode& id, const std::vector<unsigned char>& contents, uint32 seq); bool setItem(SHAMapItem::pointer& i, TNType type);
void addRaw(Serializer&);
uint32 getSeq() const { return mSeq; }
void setSeq(uint32 s) { mSeq=s; }
virtual bool isPopulated() const { return true; }
// sync functions
bool isFullBelow(void) const { return mFullBelow; } bool isFullBelow(void) const { return mFullBelow; }
void setFullBelow(void) { mFullBelow=true; } void setFullBelow(void) { mFullBelow=true; }
bool isEmptyBranch(int m) const { return !mHashes[m]; }
const uint256& getNodeHash() const { return mHash; }
const uint256& getChildHash(int m) const;
bool isEmpty() const;
virtual void dump(); virtual void dump();
virtual std::string getString() const; virtual std::string getString() const;
@@ -225,32 +206,27 @@ public:
private: private:
uint32 mSeq; uint32 mSeq;
mutable boost::recursive_mutex mLock; mutable boost::recursive_mutex mLock;
boost::unordered_map<SHAMapNode, SHAMapLeafNode::pointer, hash_SMN> mLeafByID; boost::unordered_map<SHAMapNode, SHAMapTreeNode::pointer, hash_SMN> mTNByID;
boost::unordered_map<SHAMapNode, SHAMapInnerNode::pointer, hash_SMN> mInnerNodeByID;
boost::shared_ptr<std::map<SHAMapNode, SHAMapLeafNode::pointer> > mDirtyLeafNodes; boost::shared_ptr<std::map<SHAMapNode, SHAMapTreeNode::pointer> > mDirtyNodes;
boost::shared_ptr<std::map<SHAMapNode, SHAMapInnerNode::pointer> > mDirtyInnerNodes;
SHAMapInnerNode::pointer root; SHAMapTreeNode::pointer root;
bool mImmutable, mSynching; bool mImmutable, mSynching;
protected: protected:
void dirtyUp(const uint256& id);
SHAMapLeafNode::pointer createLeaf(const SHAMapInnerNode& lowestParent, const uint256& id); void dirtyUp(std::stack<SHAMapTreeNode::pointer>& stack, const uint256& target, uint256 prevHash);
SHAMapLeafNode::pointer checkCacheLeaf(const SHAMapNode&); std::stack<SHAMapTreeNode::pointer> getStack(const uint256& id, bool include_nonmatching_leaf);
SHAMapLeafNode::pointer walkToLeaf(const uint256& id, bool create, bool modify); SHAMapTreeNode::pointer walkTo(const uint256& id, bool modify);
SHAMapLeafNode::pointer getLeaf(const SHAMapNode& id, const uint256& hash, bool modify); SHAMapTreeNode::pointer checkCacheNode(const SHAMapNode&);
SHAMapLeafNode::pointer returnLeaf(SHAMapLeafNode::pointer leaf, bool modify); void returnNode(SHAMapTreeNode::pointer&, bool modify);
SHAMapInnerNode::pointer checkCacheNode(const SHAMapNode&); SHAMapTreeNode::pointer getNode(const SHAMapNode& id);
SHAMapInnerNode::pointer getInner(const SHAMapNode& id, const uint256& hash, bool modify); SHAMapTreeNode::pointer getNode(const SHAMapNode& id, const uint256& hash, bool modify);
SHAMapInnerNode::pointer returnNode(SHAMapInnerNode::pointer node, bool modify);
SHAMapInnerNode::pointer walkTo(const SHAMapNode& id);
SHAMapItem::pointer firstBelow(SHAMapInnerNode::pointer); SHAMapItem::pointer firstBelow(SHAMapTreeNode::pointer);
SHAMapItem::pointer lastBelow(SHAMapInnerNode::pointer); SHAMapItem::pointer lastBelow(SHAMapTreeNode::pointer);
public: public:
@@ -260,32 +236,20 @@ public:
// hold the map stable across operations // hold the map stable across operations
ScopedLock Lock() const { return ScopedLock(mLock); } ScopedLock Lock() const { return ScopedLock(mLock); }
// inner node access functions bool hasNode(const SHAMapNode& id);
bool hasInnerNode(const SHAMapNode& id);
bool giveInnerNode(SHAMapInnerNode::pointer);
SHAMapInnerNode::pointer getInnerNode(const SHAMapNode&);
// leaf node access functions
bool hasLeafNode(const SHAMapNode& id);
bool giveLeafNode(SHAMapLeafNode::pointer);
SHAMapLeafNode::pointer getLeafNode(const SHAMapNode&);
// generic node functions
std::vector<unsigned char> getRawNode(const SHAMapNode& id);
bool addRawNode(const SHAMapNode& nodeID, std::vector<unsigned char> rawNode);
// normal hash access functions // normal hash access functions
bool hasItem(const uint256& id); bool hasItem(const uint256& id);
bool delItem(const uint256& id); bool delItem(const uint256& id);
bool addItem(const SHAMapItem& i); bool addItem(const SHAMapItem& i, bool isTransaction);
bool updateItem(const SHAMapItem& i); bool updateItem(const SHAMapItem& i, bool isTransaction);
SHAMapItem getItem(const uint256& id); SHAMapItem getItem(const uint256& id);
uint256 getHash() const { return root->getNodeHash(); } uint256 getHash() const { return root->getNodeHash(); }
uint256 getHash() { return root->getNodeHash(); } uint256 getHash() { return root->getNodeHash(); }
// save a copy if you have a temporary anyway // save a copy if you have a temporary anyway
bool updateGiveItem(SHAMapItem::pointer); bool updateGiveItem(SHAMapItem::pointer, bool isTransaction);
bool addGiveItem(SHAMapItem::pointer); bool addGiveItem(SHAMapItem::pointer, bool isTransaction);
// save a copy if you only need a temporary // save a copy if you only need a temporary
SHAMapItem::pointer peekItem(const uint256& id); SHAMapItem::pointer peekItem(const uint256& id);

7
SHAMapDiff.cpp
View File

@@ -20,11 +20,15 @@ class SHAMapDiffNode
mNodeID(id), mOurHash(ourHash), mOtherHash(otherHash) { ; } mNodeID(id), mOurHash(ourHash), mOtherHash(otherHash) { ; }
}; };
bool SHAMap::compare(SHAMap::pointer otherMap, SHAMapDiff& differences, int maxCount) bool SHAMap::compare(SHAMap::pointer otherMap, SHAMapDiff& differences, int maxCount)
{ // compare two hash trees, add up to maxCount differences to the difference table { // compare two hash trees, add up to maxCount differences to the difference table
// return value: true=complete table of differences given, false=too many differences // return value: true=complete table of differences given, false=too many differences
// throws on corrupt tables or missing nodes // throws on corrupt tables or missing nodes
#if 0
// FIXME: Temporarily disabled
std::stack<SHAMapDiffNode> nodeStack; // track nodes we've pushed std::stack<SHAMapDiffNode> nodeStack; // track nodes we've pushed
nodeStack.push(SHAMapDiffNode(SHAMapNode(), getHash(), otherMap->getHash())); nodeStack.push(SHAMapDiffNode(SHAMapNode(), getHash(), otherMap->getHash()));
@@ -143,5 +147,8 @@ bool SHAMap::compare(SHAMap::pointer otherMap, SHAMapDiff& differences, int maxC
} }
} }
} }
#endif
return true; return true;
} }

373
SHAMapNodes.cpp
View File

@@ -3,12 +3,14 @@
#include <boost/foreach.hpp> #include <boost/foreach.hpp>
#include <boost/lexical_cast.hpp> #include <boost/lexical_cast.hpp>
#include <boost/smart_ptr/make_shared.hpp>
#include <openssl/sha.h> #include <openssl/sha.h>
#include "Serializer.h" #include "Serializer.h"
#include "BitcoinUtil.h" #include "BitcoinUtil.h"
#include "SHAMap.h" #include "SHAMap.h"
std::string SHAMapNode::getString() const std::string SHAMapNode::getString() const
{ {
std::string ret="NodeID("; std::string ret="NodeID(";
@@ -19,7 +21,7 @@ std::string SHAMapNode::getString() const
return ret; return ret;
} }
uint256 SHAMapNode::smMasks[21]; uint256 SHAMapNode::smMasks[64];
bool SHAMapNode::operator<(const SHAMapNode &s) const bool SHAMapNode::operator<(const SHAMapNode &s) const
{ {
@@ -72,52 +74,57 @@ bool SHAMapNode::operator!=(const uint256 &s) const
void SHAMapNode::ClassInit() void SHAMapNode::ClassInit()
{ // set up the depth masks { // set up the depth masks
uint256 selector; uint256 selector;
for(int i=0; i<=leafDepth; i++) for(int i=0; i<64; i+=2)
{ {
smMasks[i]=selector; smMasks[i]=selector;
*(selector.begin()+i)=0x1F; *(selector.begin()+(i/2))=0x0F;
smMasks[i+1]=selector;
*(selector.begin()+(i/2))=0xFF;
} }
} }
uint256 SHAMapNode::getNodeID(int depth, const uint256& hash) uint256 SHAMapNode::getNodeID(int depth, const uint256& hash)
{ {
assert(depth>=0 && depth<=leafDepth); assert(depth>=0 && depth<64);
return hash & smMasks[depth]; return hash & smMasks[depth];
} }
SHAMapNode::SHAMapNode(int depth, const uint256 &hash) : mDepth(depth) SHAMapNode::SHAMapNode(int depth, const uint256 &hash) : mDepth(depth)
{ // canonicalize the hash to a node ID for this depth { // canonicalize the hash to a node ID for this depth
assert(depth>=0 && depth<=leafDepth); assert(depth>=0 && depth<64);
mNodeID = getNodeID(depth, hash); mNodeID = getNodeID(depth, hash);
} }
SHAMapNode SHAMapNode::getChildNodeID(int m) const SHAMapNode SHAMapNode::getChildNodeID(int m) const
{ // This can be optimized to avoid the << if needed { // This can be optimized to avoid the << if needed
assert(!isLeaf()); assert((m>=0) && (m<16));
assert((m>=0) && (m<32));
uint256 branch=m; uint256 branch=m;
branch<<=mDepth*8; branch<<=mDepth*4;
return SHAMapNode(mDepth+1, mNodeID | branch); return SHAMapNode(mDepth+1, mNodeID | branch);
} }
int SHAMapNode::selectBranch(const uint256 &hash) const int SHAMapNode::selectBranch(const uint256& hash) const
{ { // Which branch would contain the specified hash
if(isLeaf()) // no nodes under this node if(mDepth==63)
{ {
assert(false); assert(false);
return -1; return -1;
} }
if((hash&smMasks[mDepth])!=mNodeID) if((hash&smMasks[mDepth])!=mNodeID)
{ {
std::cerr << "selectBranch(" << getString() << std::endl;
std::cerr << " " << hash.GetHex() << " off branch" << std::endl;
assert(false); assert(false);
return -1; // does not go under this node return -1; // does not go under this node
} }
uint256 selector(hash & smMasks[mDepth+1]); int branch=*(hash.begin()+(mDepth/2));
int branch=*(selector.begin()+mDepth); if(mDepth%2) branch>>=4;
assert(branch>=0 && branch<32); else branch&=0xf;
assert(branch>=0 && branch<16);
return branch; return branch;
} }
@@ -126,259 +133,211 @@ void SHAMapNode::dump() const
std::cerr << getString() << std::endl; std::cerr << getString() << std::endl;
} }
SHAMapLeafNode::SHAMapLeafNode(const SHAMapNode& nodeID, uint32 seq) : SHAMapNode(nodeID), mHash(0), mSeq(seq) SHAMapTreeNode::SHAMapTreeNode(const SHAMapNode& nodeID, uint32 seq) : SHAMapNode(nodeID), mHash(0), mSeq(seq),
mType(ERROR), mFullBelow(false)
{ {
assert(nodeID.isLeaf());
} }
SHAMapLeafNode::SHAMapLeafNode(const SHAMapLeafNode& node, uint32 seq) : SHAMapNode(node), SHAMapTreeNode::SHAMapTreeNode(const SHAMapTreeNode& node, uint32 seq) : SHAMapNode(node),
mHash(node.mHash), mItems(node.mItems), mSeq(seq) mHash(node.mHash), mItem(node.mItem), mSeq(seq), mType(node.mType), mFullBelow(false)
{ {
assert(node.isLeaf()); if(node.mItem)
mItem=boost::make_shared<SHAMapItem>(*node.mItem);
else
memcpy(mHashes, node.mHashes, sizeof(mHashes));
} }
SHAMapLeafNode::SHAMapLeafNode(const SHAMapNode& id, const std::vector<unsigned char>& rawLeaf, uint32 seq) SHAMapTreeNode::SHAMapTreeNode(const SHAMapNode& node, SHAMapItem::pointer item, TNType type, uint32 seq) :
: SHAMapNode(id), mSeq(seq) SHAMapNode(node), mItem(item), mSeq(seq), mType(type), mFullBelow(true)
{ {
Serializer s(rawLeaf); assert(item->peekData().size()>=32);
int pos=0;
while(pos<s.getLength())
{
uint256 id=s.get256(pos);
pos+=32;
uint16 len;
if(!s.get16(len, pos)) throw SHAMapException(InvalidNode);
pos+=2;
if(!id || !len || ((pos+len)>s.getLength())) throw SHAMapException(InvalidNode);
addUpdateItem(SHAMapItem::pointer(new SHAMapItem(id, s.getRaw(pos, len))), false);
pos+=len;
}
updateHash(); updateHash();
} }
void SHAMapLeafNode::addRaw(Serializer &s) SHAMapTreeNode::SHAMapTreeNode(const SHAMapNode& id, const std::vector<unsigned char>& rawNode, uint32 seq)
: SHAMapNode(id), mSeq(seq), mType(ERROR), mFullBelow(false)
{ {
BOOST_FOREACH(SHAMapItem::pointer& nodeItem, mItems) Serializer s(rawNode);
{
s.add256(nodeItem->getTag()); int type=s.removeLastByte();
s.add16(nodeItem->peekData().size()); int len=s.getLength();
s.addRaw(nodeItem->peekData()); if( (type<0) || (type>3) || (len<33) ) throw SHAMapException(InvalidNode);
if(type==0)
{ // transaction
mItem=boost::make_shared<SHAMapItem>(s.getSHA512Half(), s.peekData());
mType=TRANSACTION;
} }
} else if(type==1)
{ // account state
bool SHAMapLeafNode::hasItem(const uint256& item) const uint160 u;
{ s.get160(u, len-20);
BOOST_FOREACH(const SHAMapItem::pointer& nodeItem, mItems) s.chop(20);
if(nodeItem->getTag()==item) return true; if(!u) throw SHAMapException(InvalidNode);
return false; mItem=boost::make_shared<SHAMapItem>(u, s.peekData());
} mType=ACCOUNT_STATE;
bool SHAMapLeafNode::addUpdateItem(SHAMapItem::pointer item, bool doHash)
{ // The node will almost never have more than one item in it
#ifdef ST_DEBUG
std::cerr << "Leaf(" << getString() << ")" << std::endl;
std::cerr << " addi(" << item->getTag().GetHex() << std::endl;
#endif
std::list<SHAMapItem::pointer>::iterator it;
for(it=mItems.begin(); it!=mItems.end(); ++it)
{
SHAMapItem &nodeItem=**it;
if(nodeItem.getTag()==item->getTag())
{
if(nodeItem.peekData()==item->peekData())
return false; // no change
nodeItem.updateData(item->peekData());
if(doHash) return updateHash();
return true;
}
if(nodeItem.getTag()>item->getTag())
{
mItems.insert(it, item);
if(doHash) return updateHash();
return true;
}
} }
mItems.push_back(item); else if(type==2)
{ // full inner
if(doHash) return updateHash(); if(len!=512) throw SHAMapException(InvalidNode);
return true; for(int i=0; i<16; i++)
} s.get256(mHashes[i], i*32);
mType=INNER;
bool SHAMapLeafNode::delItem(const uint256& tag)
{
std::list<SHAMapItem::pointer>::iterator it;
for(it=mItems.begin(); it!=mItems.end(); ++it)
{
if((*it)->getTag()==tag)
{
mItems.erase(it);
return updateHash();
}
} }
return false; else if(type==3)
} { // compressed inner
for(int i=0; i<(len/33); i++)
SHAMapItem::pointer SHAMapLeafNode::findItem(const uint256& tag)
{
BOOST_FOREACH(SHAMapItem::pointer& it, mItems)
if(it->getTag() == tag) return it;
return SHAMapItem::pointer();
}
SHAMapItem::pointer SHAMapLeafNode::firstItem()
{
if(mItems.empty()) return SHAMapItem::pointer();
return *(mItems.begin());
}
SHAMapItem::pointer SHAMapLeafNode::nextItem(const uint256& tag)
{
#ifdef ST_DEBUG
std::cerr << "LeafNode::nextItem(" << tag.GetHex() << std::endl;
BOOST_FOREACH(SHAMapItem::pointer& it, mItems)
std::cerr << " item(" << it->getTag().GetHex() << std::endl;
#endif
std::list<SHAMapItem::pointer>::iterator it;
for(it=mItems.begin(); it!=mItems.end(); ++it)
{
if((*it)->getTag()==tag)
{ {
if(++it==mItems.end()) return SHAMapItem::pointer(); int pos;
return *it; s.get1(pos, 32+(i*33));
if( (pos<0) || (pos>=16)) throw SHAMapException(InvalidNode);
s.get256(mHashes[pos], i*33);
} }
mType=INNER;
} }
#ifdef DEBUG
std::cerr << "nextItem(!found)" << std::endl; updateHash();
#endif
return SHAMapItem::pointer();
} }
SHAMapItem::pointer SHAMapLeafNode::prevItem(const uint256& tag) void SHAMapTreeNode::addRaw(Serializer &s)
{ {
std::list<SHAMapItem::pointer>::reverse_iterator it; if(mType==ERROR) throw SHAMapException(InvalidNode);
for(it=mItems.rbegin(); it!=mItems.rend(); ++it)
if(mType==TRANSACTION)
{ {
if((*it)->getTag()==tag) mItem->addRaw(s);
{ s.add1(0);
++it; assert(s.getLength()>32);
if(it==mItems.rend()) return SHAMapItem::pointer(); return;
return *it;
}
} }
return SHAMapItem::pointer();
if(mType==ACCOUNT_STATE)
{
mItem->addRaw(s);
assert(s.getLength()>20);
s.add160(mItem->getTag().to160());
s.add1(1);
return;
}
if(getBranchCount()<5)
{ // compressed node
for(int i=0; i<16; i++)
if(!!mHashes[i])
{
s.add256(mHashes[i]);
s.add1(i);
}
s.add1(3);
return;
}
for(int i=0; i<16; i++)
s.add256(mHashes[i]);
s.add1(2);
} }
SHAMapItem::pointer SHAMapLeafNode::lastItem() bool SHAMapTreeNode::updateHash()
{
if(mItems.empty()) return SHAMapItem::pointer();
return *(mItems.rbegin());
}
bool SHAMapLeafNode::updateHash()
{ {
uint256 nh; uint256 nh;
if(!mItems.empty())
if(mType==INNER)
{
bool empty=true;
for(int i=0; i<16; i++)
if(!!mHashes[i])
{
empty=false;
break;
}
if(!empty)
{
uint256 j[2];
SHA512(reinterpret_cast<unsigned char *>(mHashes), sizeof(mHashes), (unsigned char *) j);
nh=j[0];
assert(!!nh);
}
}
else if(mType==ACCOUNT_STATE)
{ {
Serializer s; Serializer s;
BOOST_FOREACH(const SHAMapItem::pointer &mi, mItems) mItem->addRaw(s);
s.addRaw(mi->peekData()); s.add160(mItem->getTag().to160());
nh=s.getSHA512Half(); nh=s.getSHA512Half();
} }
else if(mType==TRANSACTION)
nh=mItem->getTag();
else assert(false);
if(nh==mHash) return false; if(nh==mHash) return false;
mHash=nh; mHash=nh;
return true; return true;
} }
void SHAMapLeafNode::dump() bool SHAMapTreeNode::setItem(SHAMapItem::pointer& i, TNType type)
{ {
std::cerr << "SHAMapLeafNode(" << getNodeID().GetHex() << ")" << std::endl; uint256 hash=getNodeHash();
std::cerr << " " << mItems.size() << " items" << std::endl; mType=type;
mItem=i;
return getNodeHash()==hash;
} }
SHAMapInnerNode::SHAMapInnerNode(const SHAMapNode& id, uint32 seq) : SHAMapNode(id), mSeq(seq), mFullBelow(false) int SHAMapTreeNode::getBranchCount() const
{ // can be root
assert(id.getDepth()<SHAMapNode::leafDepth);
}
SHAMapInnerNode::SHAMapInnerNode(const SHAMapNode& id, const std::vector<unsigned char>& contents, uint32 seq)
: SHAMapNode(id), mSeq(seq), mFullBelow(false)
{ {
assert(!id.isLeaf()); int ret=0;
assert(contents.size()==32*256/8); for(int i=0; i<16; i++)
Serializer s(contents); if(!mHashes[i]) ret++;
for(int i=0; i<32; i++) return ret;
mHashes[i]=s.get256(i*32);
updateHash();
} }
SHAMapInnerNode::SHAMapInnerNode(const SHAMapInnerNode& node, uint32 seq) : SHAMapNode(node), mHash(node.mHash), void SHAMapTreeNode::makeInner()
mSeq(seq), mFullBelow(false)
{ {
assert(!node.isLeaf()); mItem=SHAMapItem::pointer();
memcpy(mHashes, node.mHashes, sizeof(mHashes)); memset(mHashes, 0, sizeof(mHashes));
updateHash(); mType=INNER;
mHash.zero();
} }
std::string SHAMapInnerNode::getString() const void SHAMapTreeNode::dump()
{
std::cerr << "SHAMapTreeNode(" << getNodeID().GetHex() << ")" << std::endl;
}
std::string SHAMapTreeNode::getString() const
{ {
std::string ret="NodeID("; std::string ret="NodeID(";
ret+=boost::lexical_cast<std::string>(getDepth()); ret+=boost::lexical_cast<std::string>(getDepth());
ret+=","; ret+=",";
ret+=getNodeID().GetHex(); ret+=getNodeID().GetHex();
ret+=")"; ret+=")";
for(int i=0; i<32; i++) if(isInner())
if(!isEmptyBranch(i)) {
{ for(int i=0; i<16; i++)
ret+=",b"; if(!isEmptyBranch(i))
ret+=boost::lexical_cast<std::string>(i); {
} ret+=",b";
ret+=boost::lexical_cast<std::string>(i);
}
}
if(isLeaf())
{
ret+=",leaf";
}
return ret; return ret;
} }
void SHAMapInnerNode::addRaw(Serializer &s) bool SHAMapTreeNode::setChildHash(int m, const uint256 &hash)
{ {
for(int i=0; i<32; i++) assert( (m>=0) && (m<16) );
s.add256(mHashes[i]); assert(mType==INNER);
}
bool SHAMapInnerNode::setChildHash(int m, const uint256 &hash)
{
assert( (m>=0) && (m<32) );
if(mHashes[m]==hash) if(mHashes[m]==hash)
return false; return false;
mHashes[m]=hash; mHashes[m]=hash;
return updateHash(); return updateHash();
} }
const uint256& SHAMapInnerNode::getChildHash(int m) const const uint256& SHAMapTreeNode::getChildHash(int m) const
{ {
assert( (m>=0) && (m<32) ); assert( (m>=0) && (m<16) && (mType==INNER) );
return mHashes[m]; return mHashes[m];
} }
bool SHAMapInnerNode::isEmpty() const
{
for(int i=0; i<32; i++)
if(mHashes[i]!=0) return false;
return true;
}
bool SHAMapInnerNode::updateHash()
{
uint256 j[2];
if(!isEmpty())
SHA512(reinterpret_cast<unsigned char *>(mHashes), sizeof(mHashes), (unsigned char *) j);
if(mHash==j[0]) return false;
mHash=j[0];
return true;
}
void SHAMapInnerNode::dump()
{
std::cerr << "SHAMapInnerNode(" << getDepth() << ", " << getNodeID().GetHex() << ")" << std::endl;
int children=0;
for(int i=0; i<32; i++)
if(!!mHashes[i]) children++;
std::cerr << " " << children << " child(ren)" << std::endl;
}

279
SHAMapSync.cpp
View File

@@ -1,6 +1,8 @@
#include <stack> #include <stack>
#include <boost/make_shared.hpp>
#include <openssl/rand.h> #include <openssl/rand.h>
#include "SHAMap.h" #include "SHAMap.h"
@@ -8,6 +10,8 @@
void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint256>& hashes, int max) void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint256>& hashes, int max)
{ {
boost::recursive_mutex::scoped_lock sl(mLock); boost::recursive_mutex::scoped_lock sl(mLock);
assert(root->isValid());
if(root->isFullBelow()) if(root->isFullBelow())
{ {
@@ -17,48 +21,44 @@ void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint2
return; return;
} }
std::stack<SHAMapInnerNode::pointer> stack; if(!root->isInner())
{
std::cerr << "synching empty tree" << std::endl;
return;
}
std::stack<SHAMapTreeNode::pointer> stack;
stack.push(root); stack.push(root);
while( (max>0) && (!stack.empty()) ) while( (max>0) && (!stack.empty()) )
{ {
SHAMapInnerNode::pointer node=stack.top(); SHAMapTreeNode::pointer node=stack.top();
stack.pop(); stack.pop();
#ifdef GMN_DEBUG #ifdef GMN_DEBUG
std::cerr << "gMN: popped " << node->getString() << std::endl; std::cerr << "gMN: popped " << node->getString() << std::endl;
#endif #endif
for(int i=0; i<32; i++) for(int i=0; i<16; i++)
{
if(!node->isEmptyBranch(i)) if(!node->isEmptyBranch(i))
{ {
#ifdef GNMN_DEBUG #ifdef GMN_DEBUG
std::cerr << "gMN: " << node->getString() << " has non-empty branch " << i << std::endl; std::cerr << "gMN: " << node->getString() << " has non-empty branch " << i << std::endl;
#endif #endif
bool missing=false; SHAMapTreeNode::pointer desc=getNode(node->getChildNodeID(i), node->getChildHash(i), false);
SHAMapNode childNID=node->getChildNodeID(i); if(desc)
if(node->isChildLeaf())
{ // do we have this leaf node?
if(!getLeaf(childNID, node->getChildHash(i), false)) missing=true;
}
else
{ {
SHAMapInnerNode::pointer desc=getInner(childNID, node->getChildHash(i), false); if(desc->isInner() && !desc->isFullBelow())
if(!desc)
missing=true;
else if(!desc->isFullBelow())
stack.push(desc); stack.push(desc);
} }
if(missing && max-->0) else if(max-- > 0)
{ {
#ifdef GMN_DEBUG #ifdef GMN_DEBUG
std::cerr << "gMN: need " << node->getChildNodeID(i).getString() << std::endl; std::cerr << "gMN: need " << node->getChildNodeID(i).getString() << std::endl;
#endif #endif
nodeIDs.push_back(childNID); nodeIDs.push_back(node->getChildNodeID(i));
} }
} }
}
} }
} }
@@ -66,21 +66,11 @@ bool SHAMap::getNodeFat(const SHAMapNode& wanted, std::vector<SHAMapNode>& nodeI
std::list<std::vector<unsigned char> >& rawNodes) std::list<std::vector<unsigned char> >& rawNodes)
{ {
boost::recursive_mutex::scoped_lock sl(mLock); boost::recursive_mutex::scoped_lock sl(mLock);
if(wanted.isLeaf())
{ // no fat way to get a leaf
SHAMapLeafNode::pointer leaf=getLeafNode(wanted);
if(!leaf) return false;
nodeIDs.push_back(*leaf);
Serializer s;
leaf->addRaw(s);
rawNodes.push_back(s.peekData());
return true;
}
SHAMapInnerNode::pointer node=getInnerNode(wanted); SHAMapTreeNode::pointer node=getNode(wanted);
if(!node) if(!node)
{ {
assert(false); assert(false); // Remove for release, this can happen if we get a bogus request
return false; return false;
} }
@@ -92,46 +82,21 @@ bool SHAMap::getNodeFat(const SHAMapNode& wanted, std::vector<SHAMapNode>& nodeI
if(wanted.isRoot()) // don't get a fat root if(wanted.isRoot()) // don't get a fat root
return true; return true;
bool ret=true; for(int i=0; i<16; i++)
for(int i=0; i<32; i++)
{
if(!node->isEmptyBranch(i)) if(!node->isEmptyBranch(i))
{ {
if(node->isChildLeaf()) SHAMapTreeNode::pointer nextNode=getNode(node->getChildNodeID(i), node->getChildHash(i), false);
{ assert(nextNode);
SHAMapLeafNode::pointer leaf=getLeaf(node->getChildNodeID(i), node->getChildHash(i), false); if(nextNode)
if(!leaf) {
{ nodeIDs.push_back(*nextNode);
assert(false); Serializer s;
ret=false; nextNode->addRaw(s);
} rawNodes.push_back(s.peekData());
else
{
nodeIDs.push_back(*leaf);
Serializer s;
leaf->addRaw(s);
rawNodes.push_back(s.peekData());
}
}
else
{
SHAMapInnerNode::pointer ino=getInner(node->getChildNodeID(i), node->getChildHash(i), false);
if(!ino)
{
assert(false);
ret=false;
}
else
{
nodeIDs.push_back(*ino);
Serializer s;
ino->addRaw(s);
rawNodes.push_back(s.peekData());
}
} }
} }
}
return ret; return true;
} }
bool SHAMap::addRootNode(const std::vector<unsigned char>& rootNode) bool SHAMap::addRootNode(const std::vector<unsigned char>& rootNode)
@@ -139,7 +104,7 @@ bool SHAMap::addRootNode(const std::vector<unsigned char>& rootNode)
boost::recursive_mutex::scoped_lock sl(mLock); boost::recursive_mutex::scoped_lock sl(mLock);
// we already have a root node // we already have a root node
if(root->getNodeHash()!=0) if(!!root->getNodeHash())
{ {
#ifdef DEBUG #ifdef DEBUG
std::cerr << "got root node, already have one" << std::endl; std::cerr << "got root node, already have one" << std::endl;
@@ -147,17 +112,19 @@ bool SHAMap::addRootNode(const std::vector<unsigned char>& rootNode)
return true; return true;
} }
SHAMapInnerNode::pointer node=SHAMapInnerNode::pointer(new SHAMapInnerNode(SHAMapNode(), rootNode, 0)); SHAMapTreeNode::pointer node=boost::make_shared<SHAMapTreeNode>(SHAMapNode(), rootNode, 0);
if(!node) return false; if(!node) return false;
#ifdef DEBUG #ifdef DEBUG
node->dump(); node->dump();
#endif #endif
returnNode(root, true);
root=node; root=node;
mInnerNodeByID[*node]=node; mTNByID[*root]=root;
if(mDirtyInnerNodes) (*mDirtyInnerNodes)[*node]=node;
if(!root->getNodeHash()) root->setFullBelow(); if(!root->getNodeHash()) root->setFullBelow();
return true; return true;
} }
@@ -166,7 +133,7 @@ bool SHAMap::addRootNode(const uint256& hash, const std::vector<unsigned char>&
boost::recursive_mutex::scoped_lock sl(mLock); boost::recursive_mutex::scoped_lock sl(mLock);
// we already have a root node // we already have a root node
if(root->getNodeHash()!=0) if(!!root->getNodeHash())
{ {
#ifdef DEBUG #ifdef DEBUG
std::cerr << "got root node, already have one" << std::endl; std::cerr << "got root node, already have one" << std::endl;
@@ -175,14 +142,15 @@ bool SHAMap::addRootNode(const uint256& hash, const std::vector<unsigned char>&
return true; return true;
} }
SHAMapInnerNode::pointer node=SHAMapInnerNode::pointer(new SHAMapInnerNode(SHAMapNode(), rootNode, 0)); SHAMapTreeNode::pointer node=boost::make_shared<SHAMapTreeNode>(SHAMapNode(), rootNode, 0);
if(!node) return false; if(!node) return false;
if(node->getNodeHash()!=hash) return false; if(node->getNodeHash()!=hash) return false;
returnNode(root, true);
root=node; root=node;
mInnerNodeByID[*node]=node; mTNByID[*root]=root;
if(mDirtyInnerNodes) (*mDirtyInnerNodes)[*node]=node;
if(!root->getNodeHash()) root->setFullBelow(); if(!root->getNodeHash()) root->setFullBelow();
return true; return true;
} }
@@ -193,23 +161,19 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vector<unsigned cha
boost::recursive_mutex::scoped_lock sl(mLock); boost::recursive_mutex::scoped_lock sl(mLock);
if(node.isLeaf()) if(checkCacheNode(node)) return true;
{
if(checkCacheLeaf(node)) return true;
}
else
{
if(checkCacheNode(node)) return true;
}
SHAMapInnerNode::pointer iNode=walkTo(node); std::stack<SHAMapTreeNode::pointer> stack=getStack(node.getNodeID(), true);
if(stack.empty()) return false;
SHAMapTreeNode::pointer iNode=stack.top();
if(!iNode) if(!iNode)
{ // we should always have a root { // we should always have a root
assert(false); assert(false);
return true; return true;
} }
if(iNode->getDepth()==node.getDepth()) if(iNode->isLeaf() || (iNode->getDepth()==node.getDepth()))
{ {
#ifdef DEBUG #ifdef DEBUG
std::cerr << "got inner node, already had it (late)" << std::endl; std::cerr << "got inner node, already had it (late)" << std::endl;
@@ -236,81 +200,46 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vector<unsigned cha
uint256 hash=iNode->getChildHash(branch); uint256 hash=iNode->getChildHash(branch);
if(!hash) return false; if(!hash) return false;
if(node.isLeaf()) SHAMapTreeNode::pointer newNode=boost::make_shared<SHAMapTreeNode>(node, rawNode, mSeq);
{ // leaf node if(hash!=newNode->getNodeHash()) // these aren't the droids we're looking for
SHAMapLeafNode::pointer leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(node, rawNode, mSeq));
if( (leaf->getNodeHash()!=hash) || (node!=(*leaf)) )
{
#ifdef DEBUG
std::cerr << "leaf fails consistency check" << std::endl;
#endif
return false;
}
mLeafByID[node]=leaf;
if(mDirtyLeafNodes) (*mDirtyLeafNodes)[node]=leaf;
// FIXME: This should check all sources
SHAMapInnerNode::pointer pNode=checkCacheNode(node.getParentNodeID());
if(!pNode)
{
assert(false);
return false;
}
for(int i=0; i<32; i++)
if(!checkCacheLeaf(pNode->getChildNodeID(i)))
return true;
pNode->setFullBelow();
while(!pNode->isRoot())
{
pNode=checkCacheNode(pNode->getParentNodeID());
if(!pNode)
{
assert(false);
return false;
}
for(int i=0; i<32; i++)
if(!checkCacheNode(pNode->getChildNodeID(i)))
return true;
pNode->setFullBelow();
}
return true;
}
SHAMapInnerNode::pointer newNode=SHAMapInnerNode::pointer(new SHAMapInnerNode(node, rawNode, mSeq));
if( (newNode->getNodeHash()!=hash) || (node!=newNode->getNodeID()) )
{
#ifdef DEBUG
std::cerr << "inner node fails consistency check" << std::endl;
std::cerr << " Built: " << newNode->getString() << " h=" << newNode->getNodeHash().GetHex() << std::endl;
std::cerr << "Expected: " << node.getString() << " h=" << hash.GetHex() << std::endl;
#endif
return false; return false;
}
mInnerNodeByID[node]=newNode; mTNByID[*newNode]=newNode;
if(mDirtyInnerNodes) (*mDirtyInnerNodes)[node]=newNode; if(!newNode->isLeaf())
#ifdef ST_DEBUG return true; // only a leaf can fill a branch
std::cerr << "Hooked: " << node.getString() << std::endl;
#endif // did this new leaf cause its parents to fill up
do
{
iNode=stack.top();
stack.pop();
assert(iNode->isInner());
for(int i=0; i<16; i++)
if(!iNode->isEmptyBranch(i))
{
SHAMapTreeNode::pointer nextNode=getNode(iNode->getChildNodeID(i), iNode->getChildHash(i), false);
if(!nextNode) return true;
if(nextNode->isInner() && !nextNode->isFullBelow()) return true;
}
iNode->setFullBelow();
} while(!stack.empty());
return true; return true;
} }
bool SHAMap::deepCompare(SHAMap& other) bool SHAMap::deepCompare(SHAMap& other)
{ // Intended for debug/test only { // Intended for debug/test only
std::stack<SHAMapInnerNode::pointer> stack; std::stack<SHAMapTreeNode::pointer> stack;
boost::recursive_mutex::scoped_lock sl(mLock); boost::recursive_mutex::scoped_lock sl(mLock);
stack.push(root); stack.push(root);
while(!stack.empty()) while(!stack.empty())
{ {
SHAMapInnerNode::pointer node=stack.top(); SHAMapTreeNode::pointer node=stack.top();
stack.pop(); stack.pop();
SHAMapInnerNode::pointer otherNode; SHAMapTreeNode::pointer otherNode;
if(node->isRoot()) otherNode=other.root; if(node->isRoot()) otherNode=other.root;
else otherNode=other.getInner(*node, node->getNodeHash(), false); else otherNode=other.getNode(*node, node->getNodeHash(), false);
if(!otherNode) if(!otherNode)
{ {
@@ -327,38 +256,31 @@ bool SHAMap::deepCompare(SHAMap& other)
std::cerr << "Comparing inner nodes " << node->getString() << std::endl; std::cerr << "Comparing inner nodes " << node->getString() << std::endl;
#endif #endif
for(int i=0; i<32; i++) if(node->getNodeHash() != otherNode->getNodeHash())
return false;
if(node->isLeaf())
{ {
if(node->isEmptyBranch(i)) if(!otherNode->isLeaf())
return false;
if(node->peekItem()->getTag()!=otherNode->peekItem()->getTag())
return false;
if(node->peekItem()->getData()!=otherNode->peekItem()->getData())
return false;
}
else if(node->isInner())
{
if(!otherNode->isInner())
return false;
for(int i=0; i<16; i++)
{ {
if(!otherNode->isEmptyBranch(i)) if(node->isEmptyBranch(i))
return false; {
} if(!otherNode->isEmptyBranch(i))
else
{
if(node->isChildLeaf())
{ //
SHAMapLeafNode::pointer leaf=getLeaf(node->getChildNodeID(i), node->getChildHash(i), false);
if(!leaf)
{
std::cerr << "unable to fetch leaf" << std::endl;
return false; return false;
}
SHAMapLeafNode::pointer otherLeaf=other.getLeaf(*leaf, leaf->getNodeHash(), false);
if(!otherLeaf)
{
std::cerr << "unable to fetch other leaf" << std::endl;
return false;
}
if(leaf->getNodeHash()!=otherLeaf->getNodeHash())
{
std::cerr << "leaf hash mismatch" << std::endl;
return false;
}
} }
else else
{ // do we have this inner node? {
SHAMapInnerNode::pointer next=getInner(node->getChildNodeID(i), node->getChildHash(i), false); SHAMapTreeNode::pointer next=getNode(node->getChildNodeID(i), node->getChildHash(i), false);
if(!next) if(!next)
{ {
std::cerr << "unable to fetch inner node" << std::endl; std::cerr << "unable to fetch inner node" << std::endl;
@@ -385,15 +307,16 @@ bool SHAMap::syncTest()
for(int i=0; i<items; i++) for(int i=0; i<items; i++)
{ {
Serializer s; Serializer s;
int dlen=rand()%30+4; int dlen=rand()%30+10;
for(int d=0; d<dlen; d++) for(int d=0; d<dlen; d++)
s.add32(rand()); s.add32(rand());
uint256 id=s.getSHA512Half(); uint160 id=s.getRIPEMD160();
source.addItem(SHAMapItem(id, s.peekData())); source.addItem(SHAMapItem(id, s.peekData()), false);
#ifdef ST_DEBUG #ifdef ST_DEBUG
std::cerr << "Item: " << id.GetHex() << std::endl; std::cerr << "Item: " << id.GetHex() << std::endl;
#endif #endif
} }
source.setImmutable(); source.setImmutable();
#ifdef DEBUG #ifdef DEBUG
@@ -443,7 +366,7 @@ bool SHAMap::syncTest()
hashes.clear(); hashes.clear();
// get the list of nodes we know we need // get the list of nodes we know we need
destination.getMissingNodes(nodeIDs, hashes, 1024); destination.getMissingNodes(nodeIDs, hashes, 512);
if(!nodeIDs.size()) break; if(!nodeIDs.size()) break;
#ifdef DEBUG #ifdef DEBUG
@@ -491,7 +414,7 @@ bool SHAMap::syncTest()
destination.clearSynching(); destination.clearSynching();
#ifdef DEBUG #ifdef DEBUG
std::cerr << "SYNCHING COMPLETE" << std::endl; std::cerr << "SYNCHING COMPLETE " << items << " items, " << nodes << " nodes" << std::endl;
#endif #endif
if(!source.deepCompare(destination)) if(!source.deepCompare(destination))

34
Serializer.cpp
View File

@@ -114,6 +114,40 @@ uint256 Serializer::get256(int offset) const
return ret; return ret;
} }
int Serializer::add1(unsigned char byte)
{
int ret=mData.size();
mData.push_back(byte);
return ret;
}
bool Serializer::get1(int& byte, int offset) const
{
if(offset>=mData.size()) return false;
byte=mData[offset];
return true;
}
bool Serializer::chop(int bytes)
{
if(bytes>mData.size()) return false;
mData.resize(mData.size()-bytes);
return true;
}
int Serializer::removeLastByte()
{
int size=mData.size()-1;
if(size<0)
{
assert(false);
return -1;
}
int ret=mData[size];
mData.resize(size);
return ret;
}
bool Serializer::getRaw(std::vector<unsigned char>& o, int offset, int length) const bool Serializer::getRaw(std::vector<unsigned char>& o, int offset, int length) const
{ {
if((offset+length)>mData.size()) return false; if((offset+length)>mData.size()) return false;

5
Serializer.h
View File

@@ -21,6 +21,7 @@ class Serializer
Serializer(const std::vector<unsigned char> &data) : mData(data) { ; } Serializer(const std::vector<unsigned char> &data) : mData(data) { ; }
// assemble functions // assemble functions
int add1(unsigned char byte);
int add16(uint16); int add16(uint16);
int add32(uint32); // ledger indexes, account sequence int add32(uint32); // ledger indexes, account sequence
int add64(uint64); // timestamps, amounts int add64(uint64); // timestamps, amounts
@@ -30,6 +31,8 @@ class Serializer
int addRaw(const void *ptr, int len); int addRaw(const void *ptr, int len);
// disassemble functions // disassemble functions
bool get1(int&, int offset) const;
bool get1(unsigned char&, int offset) const;
bool get16(uint16&, int offset) const; bool get16(uint16&, int offset) const;
bool get32(uint32&, int offset) const; bool get32(uint32&, int offset) const;
bool get64(uint64&, int offset) const; bool get64(uint64&, int offset) const;
@@ -53,6 +56,8 @@ class Serializer
std::vector<unsigned char> getData() const { return mData; } std::vector<unsigned char> getData() const { return mData; }
void secureErase() { memset(&(mData.front()), 0, mData.size()); erase(); } void secureErase() { memset(&(mData.front()), 0, mData.size()); erase(); }
void erase() { mData.clear(); } void erase() { mData.clear(); }
int removeLastByte();
bool chop(int num);
// signature functions // signature functions
bool checkSignature(int pubkeyOffset, int signatureOffset) const; bool checkSignature(int pubkeyOffset, int signatureOffset) const;