diff --git a/SHAMap.cpp b/SHAMap.cpp index 8628a55032..79e38fd76a 100644 --- a/SHAMap.cpp +++ b/SHAMap.cpp @@ -1,6 +1,9 @@ +#include + #include #include +#include #include "Serializer.h" #include "BitcoinUtil.h" @@ -8,205 +11,144 @@ SHAMap::SHAMap(uint32 seq) : mSeq(seq), mImmutable(false), mSynching(false) { - root=SHAMapInnerNode::pointer(new SHAMapInnerNode(SHAMapNode(SHAMapNode::rootDepth, uint256()), mSeq)); - mInnerNodeByID[*root]=root; + root=boost::make_shared(SHAMapNode(0, uint256()), mSeq); + root->makeInner(); + mTNByID[*root]=root; } -void SHAMap::dirtyUp(const uint256& id) +std::stack SHAMap::getStack(const uint256& id, bool include_nonmatching_leaf) +{ + // Walk the tree as far as possible to the specified identifier + // produce a stack of nodes along the way, with the terminal node at the top + std::stack stack; + SHAMapTreeNode::pointer node=root; + + while(!node->isLeaf()) + { + stack.push(node); + + int branch=node->selectBranch(id); + assert(branch>=0); + + uint256 hash=node->getChildHash(branch); + if(!hash) return stack; + + node=getNode(node->getChildNodeID(branch), node->getChildHash(branch), false); + if(!node) + { + if(mSynching) return stack; + throw SHAMapException(MissingNode); + } + } + + if(include_nonmatching_leaf || (node->peekItem()->getTag()==id)) + stack.push(node); + + return stack; +} + +void SHAMap::dirtyUp(std::stack& stack, const uint256& target, uint256 prevHash) { // walk the tree up from through the inner nodes to the root // update linking hashes and add nodes to dirty list assert(!mImmutable && !mSynching); - SHAMapLeafNode::pointer leaf=checkCacheLeaf(SHAMapNode(SHAMapNode::leafDepth, id)); - if(!leaf) throw SHAMapException(MissingNode); - uint256 hVal=leaf->getNodeHash(); - if(mDirtyLeafNodes) mDirtyLeafNodes->insert(std::make_pair(SHAMapNode(*leaf), leaf)); - if(!hVal) + while(!stack.empty()) { -#ifdef ST_DEBUG - std::cerr << " erasingL " << leaf->getString() << std::endl; -#endif - mLeafByID.erase(*leaf); - } + SHAMapTreeNode::pointer node=stack.top(); + stack.pop(); + assert(node->isInnerNode()); - for(int depth=SHAMapNode::leafDepth-1; depth>=0; depth--) - { // walk up the tree to the root updating nodes - SHAMapInnerNode::pointer node=checkCacheNode(SHAMapNode(depth, leaf->getNodeID())); - if(!node) throw SHAMapException(MissingNode); - if(!node->setChildHash(node->selectBranch(id), hVal)) + int branch=node->selectBranch(target); + assert(branch>=0); + + returnNode(node, true); + + if(!node->setChildHash(branch, prevHash)) { -#ifdef ST_DEBUG - std::cerr << " no change@ " << node->getString() << std::endl; -#endif + std::cerr << "dirtyUp terminates early" << std::endl; + assert(false); return; } #ifdef ST_DEBUG - std::cerr << "Dirty " << node->getString() << std::endl; + std::cerr << "dirtyUp sets branch " << branch << " to " << prevHash.GetHex() << std::endl; #endif - if(mDirtyInnerNodes) mDirtyInnerNodes->insert(std::make_pair(SHAMapNode(*node), node)); - hVal=node->getNodeHash(); - if(!hVal) - { -#ifdef ST_DEBUG - std::cerr << " erasingN " << node->getString() << std::endl; -#endif - mInnerNodeByID.erase(*node); - } + prevHash=node->getNodeHash(); + assert(!!prevHash); } } -SHAMapLeafNode::pointer SHAMap::checkCacheLeaf(const SHAMapNode& iNode) +SHAMapTreeNode::pointer SHAMap::checkCacheNode(const SHAMapNode& iNode) { - assert(iNode.isLeaf()); - boost::unordered_map::iterator it=mLeafByID.find(iNode); - if(it==mLeafByID.end()) return SHAMapLeafNode::pointer(); + boost::unordered_map::iterator it=mTNByID.find(iNode); + if(it==mTNByID.end()) return SHAMapTreeNode::pointer(); return it->second; } -SHAMapInnerNode::pointer SHAMap::checkCacheNode(const SHAMapNode& iNode) -{ - assert(!iNode.isLeaf()); - boost::unordered_map::iterator it=mInnerNodeByID.find(iNode); - if(it==mInnerNodeByID.end()) return SHAMapInnerNode::pointer(); - return it->second; -} +SHAMapTreeNode::pointer SHAMap::walkTo(const uint256& id, bool modify) +{ // walk down to the terminal node for this ID + SHAMapTreeNode::pointer inNode=root; -SHAMapLeafNode::pointer SHAMap::walkToLeaf(const uint256& id, bool create, bool modify) -{ // walk down to the leaf that would contain this ID - // is leaf node in cache - SHAMapLeafNode::pointer ln=checkCacheLeaf(SHAMapNode(SHAMapNode::leafDepth, id)); - if(ln) return returnLeaf(ln, modify); - - // walk tree to leaf - SHAMapInnerNode::pointer inNode=root; - - for(int i=0; iisLeaf()) { int branch=inNode->selectBranch(id); - if(branch<0) // somehow we got on the wrong branch - throw SHAMapException(InvalidNode); - if(inNode->isEmptyBranch(branch)) - { // no nodes below this one - if(!create) return SHAMapLeafNode::pointer(); - return createLeaf(*inNode, id); - } - if(inNode->isChildLeaf()) - { // child is leaf node - ln=getLeaf(inNode->getChildNodeID(branch), inNode->getChildHash(branch), modify); - if(ln==NULL) - { - if(!create) return SHAMapLeafNode::pointer(); - return createLeaf(*inNode, id); - } - } - else - { // child is another inner node - inNode=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), modify); - if(inNode==NULL) throw SHAMapException(InvalidNode); - } - } + if(inNode->isEmptyBranch(branch)) return inNode; + uint256 childHash=inNode->getChildHash(branch); + if(!childHash) return inNode; - assert(ln || !create); - return returnLeaf(ln, modify); + SHAMapTreeNode::pointer nextNode=getNode(inNode->getChildNodeID(branch), childHash, false); + if(!nextNode) throw SHAMapException(MissingNode); + inNode=nextNode; + } + if(modify) returnNode(inNode, true); + return inNode; } -SHAMapInnerNode::pointer SHAMap::walkTo(const SHAMapNode& id) -{ // walk down to this ID, as far as possible - SHAMapInnerNode::pointer inNode=root; - int i=0; - do +SHAMapTreeNode::pointer SHAMap::getNode(const SHAMapNode& id, const uint256& hash, bool modify) +{ // retrieve a node whose node hash is known + SHAMapTreeNode::pointer node=checkCacheNode(id); + if(node) { - int branch=inNode->selectBranch(id.getNodeID()); - if(branch<0) // somehow we got on the wrong branch - throw SHAMapException(InvalidNode); - if (inNode->isEmptyBranch(branch)) // we know no branches below this one + if(node->getNodeHash()!=hash) { #ifdef DEBUG - std::cerr << "Walk down empty branch" << std::endl; + std::cerr << "Attempt to get node, hash not in tree" << std::endl; + std::cerr << "ID: " << id.getString() << std::endl; + std::cerr << "TgtHash " << hash.GetHex() << std::endl; + std::cerr << "NodHash " << node->getNodeHash().GetHex() << std::endl; + dump(); #endif - return inNode; + throw SHAMapException(InvalidNode); } - if(inNode->isChildLeaf()) // this is the last inner node - return inNode; + returnNode(node, modify); + return node; + } - try - { - SHAMapInnerNode::pointer next=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), false); - if(!next) // we don't have the next node - { -#ifdef ST_DEBUG - std::cerr << "Unable to find node " << inNode->getChildNodeID(branch).getString() << std::endl; -#endif - return inNode; - } - assert(next->getDepth() == (inNode->getDepth()+1)); - inNode=next; - } - catch (const SHAMapException& x) - { - if(x!=SHAMapException(MissingNode)) throw(x); - return inNode; - } - assert(i++ < SHAMapNode::leafDepth); - } while(1); -} + std::vector nodeData; + if(!fetchNode(hash, nodeData)) return SHAMapTreeNode::pointer(); -SHAMapLeafNode::pointer SHAMap::getLeaf(const SHAMapNode& id, const uint256& hash, bool modify) -{ // retrieve a leaf whose node hash is known - assert(!!hash); - if(!id.isLeaf()) return SHAMapLeafNode::pointer(); - - SHAMapLeafNode::pointer leaf=checkCacheLeaf(id); - if(leaf) return returnLeaf(leaf, modify); - - std::vector leafData; - if(!fetchNode(hash, leafData)) return SHAMapLeafNode::pointer(); - leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(id, leafData, mSeq)); - if(leaf->getNodeHash()!=hash) throw SHAMapException(InvalidNode); - mLeafByID.insert(std::make_pair(id, leaf)); - return leaf; -} - -SHAMapInnerNode::pointer SHAMap::getInner(const SHAMapNode& id, const uint256& hash, bool modify) -{ // retrieve an inner node whose node hash is known - SHAMapInnerNode::pointer node=checkCacheNode(id); - if(node) return returnNode(node, modify); - - std::vector rawNode; - if(!fetchNode(hash, rawNode)) return SHAMapInnerNode::pointer(); - node=SHAMapInnerNode::pointer(new SHAMapInnerNode(id, rawNode, mSeq)); + node=boost::make_shared(id, nodeData, mSeq); if(node->getNodeHash()!=hash) throw SHAMapException(InvalidNode); - mInnerNodeByID.insert(std::make_pair(id, node)); - if(id.getDepth()==0) root=node; + if(!mTNByID.insert(std::make_pair(id, node)).second) + assert(false); return node; } -SHAMapLeafNode::pointer SHAMap::returnLeaf(SHAMapLeafNode::pointer leaf, bool modify) -{ // make sure the leaf is suitable for the intended operation (copy on write) - if(leaf && modify && (leaf->getSeq()!=mSeq)) - { - leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(*leaf, mSeq)); - mLeafByID[*leaf]=leaf; - if(mDirtyLeafNodes) mDirtyLeafNodes->insert(std::make_pair(SHAMapNode(*leaf), leaf)); - } - return leaf; -} - -SHAMapInnerNode::pointer SHAMap::returnNode(SHAMapInnerNode::pointer node, bool modify) +void SHAMap::returnNode(SHAMapTreeNode::pointer& node, bool modify) { // make sure the node is suitable for the intended operation (copy on write) + assert(node->isValid()); if(node && modify && (node->getSeq()!=mSeq)) { -#ifdef ST_DEBUG - std::cerr << "Node(" << node->getString() << ") bumpseq" << std::endl; +#ifdef DEBUG + std::cerr << "returnNode COW" << std::endl; #endif - node=SHAMapInnerNode::pointer(new SHAMapInnerNode(*node, mSeq)); - mInnerNodeByID.insert(std::make_pair(SHAMapNode(*node), node)); - if(mDirtyInnerNodes) mDirtyInnerNodes->insert(std::make_pair(SHAMapNode(*node), node)); + if(mDirtyNodes) (*mDirtyNodes)[*node]=node; + node=boost::make_shared(*node, mSeq); + assert(node->isValid()); + mTNByID[*node]=node; } - return node; } SHAMapItem::SHAMapItem(const uint256& tag, const std::vector& data) @@ -217,6 +159,57 @@ SHAMapItem::SHAMapItem(const uint160& tag, const std::vector& dat : mTag(tag.to256()), mData(data) { ; } +SHAMapItem::pointer SHAMap::firstBelow(SHAMapTreeNode::pointer node) +{ + // Return the first item below this node +#ifdef ST_DEBUG + std::cerr << "firstBelow(" << node->getString() << ")" << std::endl; +#endif + do + { // Walk down the tree + if(node->hasItem()) return node->peekItem(); + + bool foundNode=false; + for(int i=0; i<16; i++) + if(!node->isEmptyBranch(i)) + { +#ifdef ST_DEBUG + std::cerr << " FB: node " << node->getString() << std::endl; + std::cerr << " has non-empty branch " << i << " : " << + node->getChildNodeID(i).getString() << ", " << node->getChildHash(i).GetHex() << std::endl; +#endif + node=getNode(node->getChildNodeID(i), node->getChildHash(i), false); + if(!node) throw SHAMapException(MissingNode); + foundNode=true; + break; + } + if(!foundNode) return SHAMapItem::pointer(); + } while(1); +} + +SHAMapItem::pointer SHAMap::lastBelow(SHAMapTreeNode::pointer node) +{ +#ifdef DEBUG + std::cerr << "lastBelow(" << node->getString() << ")" << std::endl; +#endif + + do + { // Walk down the tree + if(node->hasItem()) return node->peekItem(); + + bool foundNode=false; + for(int i=16; i>=0; i++) + if(!node->isEmptyBranch(i)) + { + node=getNode(node->getChildNodeID(i), node->getChildHash(i), false); + if(!node) throw SHAMapException(MissingNode); + foundNode=true; + break; + } + if(!foundNode) return SHAMapItem::pointer(); + } while(1); +} + SHAMapItem::pointer SHAMap::peekFirstItem() { boost::recursive_mutex::scoped_lock sl(mLock); @@ -229,275 +222,242 @@ SHAMapItem::pointer SHAMap::peekLastItem() return lastBelow(root); } -SHAMapItem::pointer SHAMap::firstBelow(SHAMapInnerNode::pointer node) -{ -#ifdef DEBUG - std::cerr << "firstBelow(" << node->getString() << ")" << std::endl; -#endif - - int i; - while(!node->isChildLeaf()) - { - for(i=0; i<32; i++) - if(!node->isEmptyBranch(i)) - { - node=getInner(node->getChildNodeID(i), node->getChildHash(i), false); - if(!node) throw SHAMapException(MissingNode); - break; - } - - if(i==32) - { -#ifdef ST_DEBUG - std::cerr << " node:" << node->getString() << " has no children" << std::endl; -#endif - return SHAMapItem::pointer(); - } - } - - assert(node->isChildLeaf()); - for(int i=0; i<32; i++) - if(!node->isEmptyBranch(i)) - { - SHAMapLeafNode::pointer mLeaf=getLeaf(node->getChildNodeID(i), node->getChildHash(i), false); - if(!mLeaf) throw SHAMapException(MissingNode); - SHAMapItem::pointer item=mLeaf->firstItem(); - if(!item) throw SHAMapException(InvalidNode); - return item; - } - -#ifdef ST_DEBUG - std::cerr << " node:" << node->getString() << " has no children" << std::endl; -#endif - return SHAMapItem::pointer(); -} - -SHAMapItem::pointer SHAMap::lastBelow(SHAMapInnerNode::pointer node) -{ - boost::recursive_mutex::scoped_lock sl(mLock); - - const uint256 zero; - int i; - - while(!node->isChildLeaf()) - { - for(i=31; i>=0; i--) - { - uint256 cHash(node->getChildHash(i)); - if(cHash!=0) - { - node=getInner(node->getChildNodeID(i), cHash, false); - if(!node) return SHAMapItem::pointer(); - break; - } - } - if(i<0) return SHAMapItem::pointer(); - } - for(int i=31; i>=0; i--) - { - uint256 cHash=node->getChildHash(i); - if(cHash!=zero) - { - SHAMapLeafNode::pointer mLeaf=getLeaf(node->getChildNodeID(i), cHash, false); - if(!mLeaf) return SHAMapItem::pointer(); - return mLeaf->lastItem(); - } - } - return SHAMapItem::pointer(); -} - SHAMapItem::pointer SHAMap::peekNextItem(const uint256& id) { // Get a pointer to the next item in the tree after a given item - item must be in tree boost::recursive_mutex::scoped_lock sl(mLock); - SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false); - if(!leaf) + std::stack stack=getStack(id, true); + while(!stack.empty()) { -#ifdef DEBUG - std::cerr << "peekNextItem: current not found" << std::endl; -#endif - return SHAMapItem::pointer(); - } + SHAMapTreeNode::pointer node=stack.top(); + stack.pop(); - // is there another item in this leaf? (there almost never will be) - SHAMapItem::pointer next=leaf->nextItem(id); - if(next) return next; - - for(int depth=SHAMapNode::leafDepth-1; depth>=0; depth--) - { // walk up the tree until we find a node with a subsequent child - SHAMapInnerNode::pointer node=checkCacheNode(SHAMapNode(depth, id)); - if(!node) + if(node->isLeaf()) { -#ifdef DEBUG - std::cerr << "InnerNode missing: " << SHAMapNode(depth,id).getString() << std::endl; -#endif - throw SHAMapException(MissingNode); + if(node->peekItem()->getTag()>id) + return node->peekItem(); } -#ifdef ST_DEBUG - std::cerr << " UpTo " << node->getString() << std::endl; -#endif - for(int i=node->selectBranch(id)+1; i<32; i++) - if(!!node->getChildHash(i)) - { // node has a subsequent child - SHAMapNode nextNode(node->getChildNodeID(i)); - const uint256& nextHash(node->getChildHash(i)); - - if(nextNode.isLeaf()) - { // this is a terminal inner node - leaf=getLeaf(nextNode, nextHash, false); - if(!leaf) throw SHAMapException(MissingNode); - next=leaf->firstItem(); - if(!next) throw SHAMapException(InvalidNode); - return next; - } - - // the next item is the first item below this node - SHAMapInnerNode::pointer inner=getInner(nextNode, nextHash, false); - if(!inner) throw SHAMapException(MissingNode); - next=firstBelow(inner); - if(!next) throw SHAMapException(InvalidNode); - return next; + else for(int i=node->selectBranch(id)+1; i<16; i++) + if(!node->isEmptyBranch(i)) + { + node=getNode(node->getChildNodeID(i), node->getChildHash(i), false); + if(!node) throw SHAMapException(MissingNode); + SHAMapItem::pointer item=firstBelow(node); + if(!item) throw SHAMapException(MissingNode); + return item; } } -#ifdef ST_DEBUG - std::cerr << " peekNext off end" << std::endl; -#endif // must be last item return SHAMapItem::pointer(); } SHAMapItem::pointer SHAMap::peekPrevItem(const uint256& id) -{ +{ // Get a pointer to the previous item in the tree after a given item - item must be in tree boost::recursive_mutex::scoped_lock sl(mLock); - SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false); - if(!leaf) return SHAMapItem::pointer(); + std::stack stack=getStack(id, true); + while(!stack.empty()) + { + SHAMapTreeNode::pointer node=stack.top(); + stack.pop(); - // is there another item in this leaf? (there almost never will be) - SHAMapItem::pointer prev=leaf->prevItem(id); - if(prev) return prev; - - for(int depth=SHAMapNode::leafDepth-1; depth>=0; depth--) - { // walk up the tree until we find a node with a previous child - SHAMapInnerNode::pointer node=checkCacheNode(SHAMapNode(depth, id)); - if(!node) + if(node->isLeaf()) { -#ifdef DEBUG - std::cerr << "InnerNode missing: " << SHAMapNode(depth,id).getString() << std::endl; -#endif - throw SHAMapException(MissingNode); + if(node->peekItem()->getTag()peekItem(); } - for(int i=node->selectBranch(id)-1; i>=0; i--) - if(!!node->getChildHash(i)) - { // node has a subsequent child - SHAMapNode prevNode(node->getChildNodeID(i)); - const uint256& prevHash(node->getChildHash(i)); - - if(prevNode.isLeaf()) - { // this is a terminal inner node - leaf=getLeaf(prevNode, prevHash, false); - if(!leaf) throw SHAMapException(MissingNode); - prev=leaf->firstItem(); - if(!prev) throw SHAMapException(InvalidNode); - return prev; - } - - // the next item is the first item below this node - SHAMapInnerNode::pointer inner=getInner(prevNode, prevHash, false); - if(!inner) throw SHAMapException(MissingNode); - prev=lastBelow(inner); - if(!prev) throw SHAMapException(InvalidNode); - return prev; + else for(int i=node->selectBranch(id)-1; i>=0; i--) + if(!node->isEmptyBranch(i)) + { + node=getNode(node->getChildNodeID(i), node->getChildHash(i), false); + if(!node) throw SHAMapException(MissingNode); + SHAMapItem::pointer item=firstBelow(node); + if(!item) throw SHAMapException(MissingNode); + return item; } } - // must be last item return SHAMapItem::pointer(); } -SHAMapLeafNode::pointer SHAMap::createLeaf(const SHAMapInnerNode& lowestParent, const uint256& id) -{ // caller must call dirtyUp if they populate the leaf -#ifdef ST_DEBUG - std::cerr << "createLeaf(" << lowestParent.getString() << std::endl; - std::cerr << " for " << id.GetHex() << ")" << std::endl; -#endif - assert(!!id); - for(int depth=lowestParent.getDepth()+1; depthgetString() << std::endl; -#endif - return newLeaf; -} - SHAMapItem::pointer SHAMap::peekItem(const uint256& id) { boost::recursive_mutex::scoped_lock sl(mLock); - SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false); + SHAMapTreeNode::pointer leaf=walkTo(id, false); if(!leaf) return SHAMapItem::pointer(); - return leaf->findItem(id); + return leaf->peekItem(); } bool SHAMap::hasItem(const uint256& id) { // does the tree have an item with this ID boost::recursive_mutex::scoped_lock sl(mLock); - SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false); + SHAMapTreeNode::pointer leaf=walkTo(id, false); if(!leaf) return false; - return leaf->hasItem(id); + SHAMapItem::pointer item=leaf->peekItem(); + if(!item || item->getTag()!=id) return false; + return true; } bool SHAMap::delItem(const uint256& id) { // delete the item with this ID boost::recursive_mutex::scoped_lock sl(mLock); - SHAMapLeafNode::pointer leaf=walkToLeaf(id, false, false); - if(!leaf) return false; - if(!leaf->delItem(id)) return false; - dirtyUp(id); + + std::stack stack=getStack(id, false); + if(stack.empty()) throw SHAMapException(MissingNode); + + SHAMapTreeNode::pointer leaf=stack.top(); + stack.pop(); + if( !leaf || !leaf->hasItem() || (leaf->peekItem()->getTag()!=id) ) + return false; + SHAMapTreeNode::TNType type=leaf->getType(); + returnNode(leaf, true); + mTNByID.erase(*leaf); + + uint256 prevHash; + while(!stack.empty()) + { + SHAMapTreeNode::pointer node=stack.top(); + stack.pop(); + returnNode(node, true); + if(!node->setChildHash(node->selectBranch(id), prevHash)) + return true; + if(!prevHash && !node->isRoot()) + { // we may have made this a node with 1 or 0 children + int bc=node->getBranchCount(); + if(bc==0) + { + prevHash=uint256(); + mTNByID.erase(*node); + } + else if(bc==1) + { // pull up on the thread + SHAMapItem::pointer item=firstBelow(node); + assert(item); + node->setItem(item, type); + prevHash=node->getNodeHash(); + } + } + else prevHash=node->getNodeHash(); + } return true; } -bool SHAMap::addGiveItem(const SHAMapItem::pointer item) +bool SHAMap::addGiveItem(SHAMapItem::pointer item, bool isTransaction) { // add the specified item, does not update - boost::recursive_mutex::scoped_lock sl(mLock); - SHAMapLeafNode::pointer leaf=walkToLeaf(item->getTag(), true, true); - if(!leaf) - { - assert(false); - return false; - } - if(leaf->hasItem(item->getTag())) - { #ifdef ST_DEBUG - std::cerr << "leaf has item we're adding" << std::endl; + std::cerr << "aGI " << item->getTag().GetHex() << std::endl; #endif - return false; - } - if(!leaf->addUpdateItem(item, true)) + + uint256 tag=item->getTag(); + SHAMapTreeNode::TNType type=isTransaction ? SHAMapTreeNode::TRANSACTION : SHAMapTreeNode::ACCOUNT_STATE; + + boost::recursive_mutex::scoped_lock sl(mLock); + + std::stack stack=getStack(tag, true); + if(stack.empty()) throw SHAMapException(MissingNode); + + SHAMapTreeNode::pointer node=stack.top(); + stack.pop(); + + if( node->isLeaf() && (node->peekItem()->getTag()==tag) ) { - assert(false); + std::cerr << "addGiveItem ends on leaf with same tag" << std::endl; return false; } - dirtyUp(item->getTag()); + + uint256 prevHash; + returnNode(node, true); + + if(node->isInner()) + { // easy case, we end on an inner node +#ifdef ST_DEBUG + std::cerr << "aGI inner " << node->getString() << std::endl; +#endif + int branch=node->selectBranch(tag); + assert(node->isEmptyBranch(branch)); + SHAMapTreeNode::pointer newNode= + boost::make_shared(node->getChildNodeID(branch), item, type, mSeq); + if(!mTNByID.insert(std::make_pair(SHAMapNode(*newNode), newNode)).second) + { + std::cerr << "Node: " << node->getString() << std::endl; + std::cerr << "NewNode: " << newNode->getString() << std::endl; + assert(false); + throw SHAMapException(InvalidNode); + } + node->setChildHash(branch, newNode->getNodeHash()); + } + else + { // this is a leaf node that has to be made an inner node holding two items +#ifdef ST_DEBUG + std::cerr << "aGI leaf " << node->getString() << std::endl; +#endif + SHAMapItem::pointer otherItem=node->peekItem(); + assert(otherItem && (tag!=otherItem->getTag()) ); + + node->makeInner(); + + int b1, b2; + + while( (b1=node->selectBranch(tag)) == (b2=node->selectBranch(otherItem->getTag())) ) + { // we need a new inner node, since both go on same branch at this level +#ifdef ST_DEBUG + std::cerr << "need new inner node at " << node->getDepth() << std::endl; +#endif + SHAMapTreeNode::pointer newNode=boost::make_shared(node->getChildNodeID(b1), mSeq); + newNode->makeInner(); + if(!mTNByID.insert(std::make_pair(SHAMapNode(*newNode), newNode)).second) + assert(false); + stack.push(node); + node=newNode; + } + + // we can add the two leaf nodes here + assert(node->isInner()); + SHAMapTreeNode::pointer newNode= + boost::make_shared(node->getChildNodeID(b1), item, type, mSeq); + assert(newNode->isValid() && newNode->isLeaf()); + if(!mTNByID.insert(std::make_pair(SHAMapNode(*newNode), newNode)).second) + assert(false); + node->setChildHash(b1, newNode->getNodeHash()); // OPTIMIZEME hash op not needed + + newNode=boost::make_shared(node->getChildNodeID(b2), otherItem, type, mSeq); + assert(newNode->isValid() && newNode->isLeaf()); + if(!mTNByID.insert(std::make_pair(SHAMapNode(*newNode), newNode)).second) + assert(false); + node->setChildHash(b2, newNode->getNodeHash()); + } + + prevHash=node->getNodeHash(); + assert(!!prevHash); + dirtyUp(stack, tag, prevHash); return true; } -bool SHAMap::addItem(const SHAMapItem& i) +bool SHAMap::addItem(const SHAMapItem& i, bool isTransaction) { - return addGiveItem(SHAMapItem::pointer(new SHAMapItem(i))); + return addGiveItem(boost::make_shared(i), isTransaction); } -bool SHAMap::updateGiveItem(SHAMapItem::pointer item) -{ +bool SHAMap::updateGiveItem(SHAMapItem::pointer item, bool isTransaction) +{ // can't change the tag but can change the hash + uint256 tag=item->getTag(); + boost::recursive_mutex::scoped_lock sl(mLock); - SHAMapLeafNode::pointer leaf=walkToLeaf(item->getTag(), true, true); - if(!leaf) return false; - if(!leaf->addUpdateItem(item, true)) return false; - dirtyUp(item->getTag()); + + std::stack stack=getStack(tag, true); + if(stack.empty()) throw SHAMapException(MissingNode); + + SHAMapTreeNode::pointer node=stack.top(); + stack.pop(); + + if(!node->isLeaf() || (node->peekItem()->getTag()==tag) ) + return false; + + returnNode(node, true); + if(!node->setItem(item, isTransaction ? SHAMapTreeNode::TRANSACTION : SHAMapTreeNode::ACCOUNT_STATE)) + return true; + + dirtyUp(stack, tag, node->getNodeHash()); return true; } @@ -519,28 +479,15 @@ int SHAMap::flushDirty(int maxNodes, HashedObjectType t, uint32 seq) int flushed=0; Serializer s; - if(mDirtyLeafNodes) + if(mDirtyNodes) { - while(!mDirtyLeafNodes->empty()) + while(!mDirtyNodes->empty()) { - SHAMapLeafNode::pointer& dln=mDirtyLeafNodes->begin()->second; - s.erase(); - dln->addRaw(s); - HashedObject::store(t, seq, s.peekData(), s.getSHA512Half()); - mDirtyLeafNodes->erase(mDirtyLeafNodes->begin()); - if(flushed++>=maxNodes) return flushed; - } - } - - if(mDirtyInnerNodes) - { - while(!mDirtyInnerNodes->empty()) - { - SHAMapInnerNode::pointer& din=mDirtyInnerNodes->begin()->second; + SHAMapTreeNode::pointer& din=mDirtyNodes->begin()->second; s.erase(); din->addRaw(s); HashedObject::store(t, seq, s.peekData(), s.getSHA512Half()); - mDirtyInnerNodes->erase(mDirtyInnerNodes->begin()); + mDirtyNodes->erase(mDirtyNodes->begin()); if(flushed++>=maxNodes) return flushed; } } @@ -548,48 +495,30 @@ int SHAMap::flushDirty(int maxNodes, HashedObjectType t, uint32 seq) return flushed; } -SHAMapInnerNode::pointer SHAMap::getInnerNode(const SHAMapNode& node) +SHAMapTreeNode::pointer SHAMap::getNode(const SHAMapNode& nodeID) { boost::recursive_mutex::scoped_lock sl(mLock); - SHAMapInnerNode::pointer inNode=root; - for(int i=0; igetDepth()==node.getDepth()) - { - if((*inNode)!=node) return SHAMapInnerNode::pointer(); - return inNode; - } - int branch=inNode->selectBranch(node.getNodeID()); - if( (branch<0) || (inNode->isEmptyBranch(branch)) ) return SHAMapInnerNode::pointer(); - inNode=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), false); - if(!inNode) return inNode; - } - if(inNode->getDepth()==node.getDepth()) - { - if((*inNode)!=node) return SHAMapInnerNode::pointer(); - return inNode; - } - return SHAMapInnerNode::pointer(); -} -SHAMapLeafNode::pointer SHAMap::getLeafNode(const SHAMapNode& leaf) -{ - boost::recursive_mutex::scoped_lock sl(mLock); - SHAMapInnerNode::pointer inNode=root; - for(int i=0; inNode->getDepth()selectBranch(leaf.getNodeID()); - if( (branch<0) || (inNode->isEmptyBranch(branch)) ) return SHAMapLeafNode::pointer(); - inNode=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), false); - if(!inNode) return SHAMapLeafNode::pointer(); + int branch=node->selectBranch(nodeID.getNodeID()); + assert(branch>=0); + if( (branch<0) || (node->isEmptyBranch(branch)) ) + return SHAMapTreeNode::pointer(); + + node=getNode(node->getChildNodeID(branch), node->getChildHash(branch), false); + if(!node) throw SHAMapException(MissingNode); } - int branch=inNode->selectBranch(leaf.getNodeID()); - if( (branch<0) || (inNode->isEmptyBranch(branch)) ) return SHAMapLeafNode::pointer(); - return getLeaf(inNode->getChildNodeID(branch), inNode->getChildHash(branch), false); + return node; } void SHAMap::dump() { +#if 0 std::cerr << "SHAMap::dump" << std::endl; SHAMapItem::pointer i=peekFirstItem(); while(i) @@ -598,12 +527,23 @@ void SHAMap::dump() i=peekNextItem(i->getTag()); } std::cerr << "SHAMap::dump done" << std::endl; +#endif + + std::cerr << " MAP Contains" << std::endl; + boost::recursive_mutex::scoped_lock sl(mLock); + for(boost::unordered_map::iterator it=mTNByID.begin(); + it!=mTNByID.end(); ++it) + { + std::cerr << it->second->getString() << std::endl; + } + } static std::vectorIntToVUC(int i) { std::vector vuc; - vuc.push_back((unsigned char) i); + for(int i=0; i<32; i++) + vuc.push_back((unsigned char) i); return vuc; } @@ -619,19 +559,29 @@ bool SHAMap::TestSHAMap() SHAMap sMap; SHAMapItem i1(h1, IntToVUC(1)), i2(h2, IntToVUC(2)), i3(h3, IntToVUC(3)), i4(h4, IntToVUC(4)), i5(h5, IntToVUC(5)); - sMap.addItem(i2); - sMap.addItem(i1); + if(!sMap.addItem(i2, true)) + { + assert(false); + return false; + } + if(!sMap.addItem(i1, true)) + { + assert(false); + return false; + } - SHAMapItem::pointer i=sMap.peekFirstItem(); + SHAMapItem::pointer i; + + i=sMap.peekFirstItem(); assert(!!i && (*i==i1)); i=sMap.peekNextItem(i->getTag()); assert(!!i && (*i==i2)); i=sMap.peekNextItem(i->getTag()); assert(!i); - sMap.addItem(i4); + sMap.addItem(i4, true); sMap.delItem(i2.getTag()); - sMap.addItem(i3); + sMap.addItem(i3, true); i=sMap.peekFirstItem(); assert(!!i && (*i==i1)); @@ -642,9 +592,8 @@ bool SHAMap::TestSHAMap() i=sMap.peekNextItem(i->getTag()); assert(!i); - if(!syncTest()); + if(!syncTest()) return false; - return true; } diff --git a/SHAMap.h b/SHAMap.h index 8cbafe31d6..e5e0030c85 100644 --- a/SHAMap.h +++ b/SHAMap.h @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include @@ -18,7 +18,6 @@ class SHAMap; // A tree-like map of SHA256 hashes -// 21 levels, consisting of a root, 19 interior node levels, and leaves // The trees are designed for rapid synchronization and compression of differences @@ -28,29 +27,27 @@ public: typedef boost::shared_ptr pointer; private: - static uint256 smMasks[21]; // AND with hash to get node id + static uint256 smMasks[64]; // AND with hash to get node id uint256 mNodeID; int mDepth; public: - // 0 is root, 20 is leaf static const int rootDepth=0; - static const int leafDepth=20; SHAMapNode() : mDepth(0) { ; } SHAMapNode(int depth, const uint256& hash); int getDepth() const { return mDepth; } const uint256& getNodeID() const { return mNodeID; } - bool isRoot() const { return mDepth==0; } - bool isLeaf() const { return mDepth==leafDepth; } - bool isChildLeaf() const { return mDepth==(leafDepth-1); } - bool isInner() const { return !isRoot() && !isLeaf(); } virtual bool isPopulated() const { return false; } - SHAMapNode getParentNodeID() const { return SHAMapNode(mDepth-1, mNodeID); } + SHAMapNode getParentNodeID() const + { + assert(mDepth); + return SHAMapNode(mDepth-1, mNodeID); + } SHAMapNode getChildNodeID(int m) const; int selectBranch(const uint256& hash) const; @@ -62,6 +59,7 @@ public: bool operator!=(const uint256&) const; bool operator<=(const SHAMapNode&) const; bool operator>=(const SHAMapNode&) const; + bool isRoot() const { return mDepth==0; } virtual std::string getString() const; void dump() const; @@ -100,6 +98,8 @@ public: const uint256& getTag() const { return mTag; } std::vector getData() const { return mData; } const std::vector& peekData() const { return mData; } + void addRaw(Serializer &s) { s.addRaw(mData); } + void addRaw(std::vector& s) { s.insert(s.end(), mData.begin(), mData.end()); } void updateData(const std::vector& data) { mData=data; } @@ -118,93 +118,74 @@ public: virtual void dump(); }; -class SHAMapLeafNode : public SHAMapNode +class SHAMapTreeNode : public SHAMapNode { friend class SHAMap; public: - typedef boost::shared_ptr pointer; + typedef boost::shared_ptr pointer; + + enum TNType + { + ERROR =0, + INNER =1, + TRANSACTION =2, + ACCOUNT_STATE =3 + }; private: uint256 mHash; - std::list mItems; + uint256 mHashes[16]; + SHAMapItem::pointer mItem; uint32 mSeq; + TNType mType; + bool mFullBelow; bool updateHash(); - SHAMapLeafNode(const SHAMapLeafNode&); // no implementation - SHAMapLeafNode& operator=(const SHAMapLeafNode&); // no implementation - -protected: - bool addUpdateItem(SHAMapItem::pointer, bool doHash); - bool delItem(const SHAMapItem::pointer i) { return delItem(i->getTag()); } - bool delItem(const uint256& tag); + SHAMapTreeNode(const SHAMapTreeNode&); // no implementation + SHAMapTreeNode& operator=(const SHAMapTreeNode&); // no implementation public: - SHAMapLeafNode(const SHAMapNode& nodeID, uint32 seq); - SHAMapLeafNode(const SHAMapLeafNode& node, uint32 seq); - SHAMapLeafNode(const SHAMapNode& id, const std::vector& contents, uint32 seq); + SHAMapTreeNode(const SHAMapNode& nodeID, uint32 seq); // empty node + SHAMapTreeNode(const SHAMapTreeNode& node, uint32 seq); // copy node from older tree + SHAMapTreeNode(const SHAMapNode& nodeID, SHAMapItem::pointer item, TNType type, uint32 seq); + // raw node functions + SHAMapTreeNode(const SHAMapNode& id, const std::vector& contents, uint32 seq); // raw node void addRaw(Serializer &); virtual bool isPopulated() const { return true; } + // node functions uint32 getSeq() const { return mSeq; } void setSeq(uint32 s) { mSeq=s; } - const uint256& getNodeHash() const { return mHash; } - bool isEmpty() const { return mItems.empty(); } - int getItemCount() const { return mItems.size(); } + TNType getType() const { return mType; } - bool hasItem(const uint256& item) const; - SHAMapItem::pointer findItem(const uint256& tag); - SHAMapItem::pointer firstItem(); - SHAMapItem::pointer lastItem(); - SHAMapItem::pointer nextItem(const uint256& tag); - SHAMapItem::pointer prevItem(const uint256& tag); + // type functions + bool isLeaf() const { return mType==TRANSACTION || mType==ACCOUNT_STATE; } + bool isInner() const { return mType==INNER; } + bool isValid() const { return mType!=ERROR; } + bool isTransaction() const { return mType!=TRANSACTION; } + bool isAccountState() const { return mType!=ACCOUNT_STATE; } - virtual void dump(); -}; - - -class SHAMapInnerNode : public SHAMapNode -{ - friend class SHAMap; - -public: - typedef boost::shared_ptr pointer; -private: - uint256 mHash; - uint256 mHashes[32]; - uint32 mSeq; - bool mFullBelow; // we have all nodes below this node - - bool updateHash(); - - SHAMapInnerNode(const SHAMapInnerNode&); // no implementation - SHAMapInnerNode& operator=(const SHAMapInnerNode&); // no implementation - -protected: + // inner node functions + bool isInnerNode() const { return !mItem; } bool setChildHash(int m, const uint256& hash); + const uint256& getChildHash(int m) const; + bool isEmptyBranch(int m) const { return !mHashes[m]; } + int getBranchCount() const; + void makeInner(); -public: - SHAMapInnerNode(const SHAMapNode& id, uint32 seq); - SHAMapInnerNode(const SHAMapInnerNode& node, uint32 seq); - SHAMapInnerNode(const SHAMapNode& id, const std::vector& contents, uint32 seq); - - void addRaw(Serializer&); - - uint32 getSeq() const { return mSeq; } - void setSeq(uint32 s) { mSeq=s; } - - virtual bool isPopulated() const { return true; } + // item node function + bool hasItem() const { return !!mItem; } + SHAMapItem::pointer peekItem() { return mItem; } + bool setItem(SHAMapItem::pointer& i, TNType type); + // sync functions bool isFullBelow(void) const { return mFullBelow; } void setFullBelow(void) { mFullBelow=true; } - bool isEmptyBranch(int m) const { return !mHashes[m]; } - const uint256& getNodeHash() const { return mHash; } - const uint256& getChildHash(int m) const; - bool isEmpty() const; virtual void dump(); virtual std::string getString() const; @@ -225,32 +206,27 @@ public: private: uint32 mSeq; mutable boost::recursive_mutex mLock; - boost::unordered_map mLeafByID; - boost::unordered_map mInnerNodeByID; + boost::unordered_map mTNByID; - boost::shared_ptr > mDirtyLeafNodes; - boost::shared_ptr > mDirtyInnerNodes; + boost::shared_ptr > mDirtyNodes; - SHAMapInnerNode::pointer root; + SHAMapTreeNode::pointer root; bool mImmutable, mSynching; protected: - void dirtyUp(const uint256& id); - SHAMapLeafNode::pointer createLeaf(const SHAMapInnerNode& lowestParent, const uint256& id); - SHAMapLeafNode::pointer checkCacheLeaf(const SHAMapNode&); - SHAMapLeafNode::pointer walkToLeaf(const uint256& id, bool create, bool modify); - SHAMapLeafNode::pointer getLeaf(const SHAMapNode& id, const uint256& hash, bool modify); - SHAMapLeafNode::pointer returnLeaf(SHAMapLeafNode::pointer leaf, bool modify); + void dirtyUp(std::stack& stack, const uint256& target, uint256 prevHash); + std::stack getStack(const uint256& id, bool include_nonmatching_leaf); + SHAMapTreeNode::pointer walkTo(const uint256& id, bool modify); + SHAMapTreeNode::pointer checkCacheNode(const SHAMapNode&); + void returnNode(SHAMapTreeNode::pointer&, bool modify); - SHAMapInnerNode::pointer checkCacheNode(const SHAMapNode&); - SHAMapInnerNode::pointer getInner(const SHAMapNode& id, const uint256& hash, bool modify); - SHAMapInnerNode::pointer returnNode(SHAMapInnerNode::pointer node, bool modify); - SHAMapInnerNode::pointer walkTo(const SHAMapNode& id); + SHAMapTreeNode::pointer getNode(const SHAMapNode& id); + SHAMapTreeNode::pointer getNode(const SHAMapNode& id, const uint256& hash, bool modify); - SHAMapItem::pointer firstBelow(SHAMapInnerNode::pointer); - SHAMapItem::pointer lastBelow(SHAMapInnerNode::pointer); + SHAMapItem::pointer firstBelow(SHAMapTreeNode::pointer); + SHAMapItem::pointer lastBelow(SHAMapTreeNode::pointer); public: @@ -260,32 +236,20 @@ public: // hold the map stable across operations ScopedLock Lock() const { return ScopedLock(mLock); } - // inner node access functions - bool hasInnerNode(const SHAMapNode& id); - bool giveInnerNode(SHAMapInnerNode::pointer); - SHAMapInnerNode::pointer getInnerNode(const SHAMapNode&); - - // leaf node access functions - bool hasLeafNode(const SHAMapNode& id); - bool giveLeafNode(SHAMapLeafNode::pointer); - SHAMapLeafNode::pointer getLeafNode(const SHAMapNode&); - - // generic node functions - std::vector getRawNode(const SHAMapNode& id); - bool addRawNode(const SHAMapNode& nodeID, std::vector rawNode); + bool hasNode(const SHAMapNode& id); // normal hash access functions bool hasItem(const uint256& id); bool delItem(const uint256& id); - bool addItem(const SHAMapItem& i); - bool updateItem(const SHAMapItem& i); + bool addItem(const SHAMapItem& i, bool isTransaction); + bool updateItem(const SHAMapItem& i, bool isTransaction); SHAMapItem getItem(const uint256& id); uint256 getHash() const { return root->getNodeHash(); } uint256 getHash() { return root->getNodeHash(); } // save a copy if you have a temporary anyway - bool updateGiveItem(SHAMapItem::pointer); - bool addGiveItem(SHAMapItem::pointer); + bool updateGiveItem(SHAMapItem::pointer, bool isTransaction); + bool addGiveItem(SHAMapItem::pointer, bool isTransaction); // save a copy if you only need a temporary SHAMapItem::pointer peekItem(const uint256& id); diff --git a/SHAMapDiff.cpp b/SHAMapDiff.cpp index ccc9f0fc50..aadcba919d 100644 --- a/SHAMapDiff.cpp +++ b/SHAMapDiff.cpp @@ -20,11 +20,15 @@ class SHAMapDiffNode mNodeID(id), mOurHash(ourHash), mOtherHash(otherHash) { ; } }; + bool SHAMap::compare(SHAMap::pointer otherMap, SHAMapDiff& differences, int maxCount) { // compare two hash trees, add up to maxCount differences to the difference table // return value: true=complete table of differences given, false=too many differences // throws on corrupt tables or missing nodes +#if 0 +// FIXME: Temporarily disabled + std::stack nodeStack; // track nodes we've pushed nodeStack.push(SHAMapDiffNode(SHAMapNode(), getHash(), otherMap->getHash())); @@ -143,5 +147,8 @@ bool SHAMap::compare(SHAMap::pointer otherMap, SHAMapDiff& differences, int maxC } } } + +#endif + return true; } diff --git a/SHAMapNodes.cpp b/SHAMapNodes.cpp index f31ad364a2..2bdd7d2856 100644 --- a/SHAMapNodes.cpp +++ b/SHAMapNodes.cpp @@ -3,12 +3,14 @@ #include #include +#include #include #include "Serializer.h" #include "BitcoinUtil.h" #include "SHAMap.h" + std::string SHAMapNode::getString() const { std::string ret="NodeID("; @@ -19,7 +21,7 @@ std::string SHAMapNode::getString() const return ret; } -uint256 SHAMapNode::smMasks[21]; +uint256 SHAMapNode::smMasks[64]; bool SHAMapNode::operator<(const SHAMapNode &s) const { @@ -72,52 +74,57 @@ bool SHAMapNode::operator!=(const uint256 &s) const void SHAMapNode::ClassInit() { // set up the depth masks uint256 selector; - for(int i=0; i<=leafDepth; i++) + for(int i=0; i<64; i+=2) { smMasks[i]=selector; - *(selector.begin()+i)=0x1F; + *(selector.begin()+(i/2))=0x0F; + smMasks[i+1]=selector; + *(selector.begin()+(i/2))=0xFF; } } uint256 SHAMapNode::getNodeID(int depth, const uint256& hash) { - assert(depth>=0 && depth<=leafDepth); + assert(depth>=0 && depth<64); return hash & smMasks[depth]; } SHAMapNode::SHAMapNode(int depth, const uint256 &hash) : mDepth(depth) { // canonicalize the hash to a node ID for this depth - assert(depth>=0 && depth<=leafDepth); + assert(depth>=0 && depth<64); mNodeID = getNodeID(depth, hash); } SHAMapNode SHAMapNode::getChildNodeID(int m) const { // This can be optimized to avoid the << if needed - assert(!isLeaf()); - assert((m>=0) && (m<32)); + assert((m>=0) && (m<16)); uint256 branch=m; - branch<<=mDepth*8; + branch<<=mDepth*4; return SHAMapNode(mDepth+1, mNodeID | branch); } -int SHAMapNode::selectBranch(const uint256 &hash) const -{ - if(isLeaf()) // no nodes under this node +int SHAMapNode::selectBranch(const uint256& hash) const +{ // Which branch would contain the specified hash + if(mDepth==63) { assert(false); return -1; } if((hash&smMasks[mDepth])!=mNodeID) { + std::cerr << "selectBranch(" << getString() << std::endl; + std::cerr << " " << hash.GetHex() << " off branch" << std::endl; assert(false); return -1; // does not go under this node } - uint256 selector(hash & smMasks[mDepth+1]); - int branch=*(selector.begin()+mDepth); - assert(branch>=0 && branch<32); + int branch=*(hash.begin()+(mDepth/2)); + if(mDepth%2) branch>>=4; + else branch&=0xf; + + assert(branch>=0 && branch<16); return branch; } @@ -126,259 +133,211 @@ void SHAMapNode::dump() const std::cerr << getString() << std::endl; } -SHAMapLeafNode::SHAMapLeafNode(const SHAMapNode& nodeID, uint32 seq) : SHAMapNode(nodeID), mHash(0), mSeq(seq) +SHAMapTreeNode::SHAMapTreeNode(const SHAMapNode& nodeID, uint32 seq) : SHAMapNode(nodeID), mHash(0), mSeq(seq), + mType(ERROR), mFullBelow(false) { - assert(nodeID.isLeaf()); } -SHAMapLeafNode::SHAMapLeafNode(const SHAMapLeafNode& node, uint32 seq) : SHAMapNode(node), - mHash(node.mHash), mItems(node.mItems), mSeq(seq) +SHAMapTreeNode::SHAMapTreeNode(const SHAMapTreeNode& node, uint32 seq) : SHAMapNode(node), + mHash(node.mHash), mItem(node.mItem), mSeq(seq), mType(node.mType), mFullBelow(false) { - assert(node.isLeaf()); + if(node.mItem) + mItem=boost::make_shared(*node.mItem); + else + memcpy(mHashes, node.mHashes, sizeof(mHashes)); } -SHAMapLeafNode::SHAMapLeafNode(const SHAMapNode& id, const std::vector& rawLeaf, uint32 seq) - : SHAMapNode(id), mSeq(seq) +SHAMapTreeNode::SHAMapTreeNode(const SHAMapNode& node, SHAMapItem::pointer item, TNType type, uint32 seq) : + SHAMapNode(node), mItem(item), mSeq(seq), mType(type), mFullBelow(true) { - Serializer s(rawLeaf); - int pos=0; - while(poss.getLength())) throw SHAMapException(InvalidNode); - addUpdateItem(SHAMapItem::pointer(new SHAMapItem(id, s.getRaw(pos, len))), false); - pos+=len; - } + assert(item->peekData().size()>=32); updateHash(); } -void SHAMapLeafNode::addRaw(Serializer &s) +SHAMapTreeNode::SHAMapTreeNode(const SHAMapNode& id, const std::vector& rawNode, uint32 seq) + : SHAMapNode(id), mSeq(seq), mType(ERROR), mFullBelow(false) { - BOOST_FOREACH(SHAMapItem::pointer& nodeItem, mItems) - { - s.add256(nodeItem->getTag()); - s.add16(nodeItem->peekData().size()); - s.addRaw(nodeItem->peekData()); + Serializer s(rawNode); + + int type=s.removeLastByte(); + int len=s.getLength(); + if( (type<0) || (type>3) || (len<33) ) throw SHAMapException(InvalidNode); + + if(type==0) + { // transaction + mItem=boost::make_shared(s.getSHA512Half(), s.peekData()); + mType=TRANSACTION; } -} - -bool SHAMapLeafNode::hasItem(const uint256& item) const -{ - BOOST_FOREACH(const SHAMapItem::pointer& nodeItem, mItems) - if(nodeItem->getTag()==item) return true; - return false; -} - -bool SHAMapLeafNode::addUpdateItem(SHAMapItem::pointer item, bool doHash) -{ // The node will almost never have more than one item in it -#ifdef ST_DEBUG - std::cerr << "Leaf(" << getString() << ")" << std::endl; - std::cerr << " addi(" << item->getTag().GetHex() << std::endl; -#endif - std::list::iterator it; - for(it=mItems.begin(); it!=mItems.end(); ++it) - { - SHAMapItem &nodeItem=**it; - if(nodeItem.getTag()==item->getTag()) - { - if(nodeItem.peekData()==item->peekData()) - return false; // no change - nodeItem.updateData(item->peekData()); - if(doHash) return updateHash(); - return true; - } - if(nodeItem.getTag()>item->getTag()) - { - mItems.insert(it, item); - if(doHash) return updateHash(); - return true; - } + else if(type==1) + { // account state + uint160 u; + s.get160(u, len-20); + s.chop(20); + if(!u) throw SHAMapException(InvalidNode); + mItem=boost::make_shared(u, s.peekData()); + mType=ACCOUNT_STATE; } - mItems.push_back(item); - - if(doHash) return updateHash(); - return true; -} - -bool SHAMapLeafNode::delItem(const uint256& tag) -{ - std::list::iterator it; - for(it=mItems.begin(); it!=mItems.end(); ++it) - { - if((*it)->getTag()==tag) - { - mItems.erase(it); - return updateHash(); - } + else if(type==2) + { // full inner + if(len!=512) throw SHAMapException(InvalidNode); + for(int i=0; i<16; i++) + s.get256(mHashes[i], i*32); + mType=INNER; } - return false; -} - -SHAMapItem::pointer SHAMapLeafNode::findItem(const uint256& tag) -{ - BOOST_FOREACH(SHAMapItem::pointer& it, mItems) - if(it->getTag() == tag) return it; - return SHAMapItem::pointer(); -} - -SHAMapItem::pointer SHAMapLeafNode::firstItem() -{ - if(mItems.empty()) return SHAMapItem::pointer(); - return *(mItems.begin()); -} - -SHAMapItem::pointer SHAMapLeafNode::nextItem(const uint256& tag) -{ -#ifdef ST_DEBUG - std::cerr << "LeafNode::nextItem(" << tag.GetHex() << std::endl; - BOOST_FOREACH(SHAMapItem::pointer& it, mItems) - std::cerr << " item(" << it->getTag().GetHex() << std::endl; -#endif - std::list::iterator it; - for(it=mItems.begin(); it!=mItems.end(); ++it) - { - if((*it)->getTag()==tag) + else if(type==3) + { // compressed inner + for(int i=0; i<(len/33); i++) { - if(++it==mItems.end()) return SHAMapItem::pointer(); - return *it; + int pos; + s.get1(pos, 32+(i*33)); + if( (pos<0) || (pos>=16)) throw SHAMapException(InvalidNode); + s.get256(mHashes[pos], i*33); } + mType=INNER; } -#ifdef DEBUG - std::cerr << "nextItem(!found)" << std::endl; -#endif - return SHAMapItem::pointer(); + + updateHash(); } -SHAMapItem::pointer SHAMapLeafNode::prevItem(const uint256& tag) +void SHAMapTreeNode::addRaw(Serializer &s) { - std::list::reverse_iterator it; - for(it=mItems.rbegin(); it!=mItems.rend(); ++it) + if(mType==ERROR) throw SHAMapException(InvalidNode); + + if(mType==TRANSACTION) { - if((*it)->getTag()==tag) - { - ++it; - if(it==mItems.rend()) return SHAMapItem::pointer(); - return *it; - } + mItem->addRaw(s); + s.add1(0); + assert(s.getLength()>32); + return; } - return SHAMapItem::pointer(); + + if(mType==ACCOUNT_STATE) + { + mItem->addRaw(s); + assert(s.getLength()>20); + s.add160(mItem->getTag().to160()); + s.add1(1); + return; + } + + if(getBranchCount()<5) + { // compressed node + for(int i=0; i<16; i++) + if(!!mHashes[i]) + { + s.add256(mHashes[i]); + s.add1(i); + } + s.add1(3); + return; + } + + for(int i=0; i<16; i++) + s.add256(mHashes[i]); + s.add1(2); } -SHAMapItem::pointer SHAMapLeafNode::lastItem() -{ - if(mItems.empty()) return SHAMapItem::pointer(); - return *(mItems.rbegin()); -} - -bool SHAMapLeafNode::updateHash() +bool SHAMapTreeNode::updateHash() { uint256 nh; - if(!mItems.empty()) + + if(mType==INNER) + { + bool empty=true; + for(int i=0; i<16; i++) + if(!!mHashes[i]) + { + empty=false; + break; + } + if(!empty) + { + uint256 j[2]; + SHA512(reinterpret_cast(mHashes), sizeof(mHashes), (unsigned char *) j); + nh=j[0]; + assert(!!nh); + } + } + else if(mType==ACCOUNT_STATE) { Serializer s; - BOOST_FOREACH(const SHAMapItem::pointer &mi, mItems) - s.addRaw(mi->peekData()); + mItem->addRaw(s); + s.add160(mItem->getTag().to160()); nh=s.getSHA512Half(); } + else if(mType==TRANSACTION) + nh=mItem->getTag(); + else assert(false); + if(nh==mHash) return false; mHash=nh; return true; } -void SHAMapLeafNode::dump() +bool SHAMapTreeNode::setItem(SHAMapItem::pointer& i, TNType type) { - std::cerr << "SHAMapLeafNode(" << getNodeID().GetHex() << ")" << std::endl; - std::cerr << " " << mItems.size() << " items" << std::endl; + uint256 hash=getNodeHash(); + mType=type; + mItem=i; + return getNodeHash()==hash; } -SHAMapInnerNode::SHAMapInnerNode(const SHAMapNode& id, uint32 seq) : SHAMapNode(id), mSeq(seq), mFullBelow(false) -{ // can be root - assert(id.getDepth()& contents, uint32 seq) - : SHAMapNode(id), mSeq(seq), mFullBelow(false) +int SHAMapTreeNode::getBranchCount() const { - assert(!id.isLeaf()); - assert(contents.size()==32*256/8); - Serializer s(contents); - for(int i=0; i<32; i++) - mHashes[i]=s.get256(i*32); - updateHash(); + int ret=0; + for(int i=0; i<16; i++) + if(!mHashes[i]) ret++; + return ret; } -SHAMapInnerNode::SHAMapInnerNode(const SHAMapInnerNode& node, uint32 seq) : SHAMapNode(node), mHash(node.mHash), - mSeq(seq), mFullBelow(false) +void SHAMapTreeNode::makeInner() { - assert(!node.isLeaf()); - memcpy(mHashes, node.mHashes, sizeof(mHashes)); - updateHash(); + mItem=SHAMapItem::pointer(); + memset(mHashes, 0, sizeof(mHashes)); + mType=INNER; + mHash.zero(); } -std::string SHAMapInnerNode::getString() const +void SHAMapTreeNode::dump() +{ + std::cerr << "SHAMapTreeNode(" << getNodeID().GetHex() << ")" << std::endl; +} + +std::string SHAMapTreeNode::getString() const { std::string ret="NodeID("; ret+=boost::lexical_cast(getDepth()); ret+=","; ret+=getNodeID().GetHex(); ret+=")"; - for(int i=0; i<32; i++) - if(!isEmptyBranch(i)) - { - ret+=",b"; - ret+=boost::lexical_cast(i); - } + if(isInner()) + { + for(int i=0; i<16; i++) + if(!isEmptyBranch(i)) + { + ret+=",b"; + ret+=boost::lexical_cast(i); + } + } + if(isLeaf()) + { + ret+=",leaf"; + } return ret; } -void SHAMapInnerNode::addRaw(Serializer &s) +bool SHAMapTreeNode::setChildHash(int m, const uint256 &hash) { - for(int i=0; i<32; i++) - s.add256(mHashes[i]); -} -bool SHAMapInnerNode::setChildHash(int m, const uint256 &hash) -{ - assert( (m>=0) && (m<32) ); + assert( (m>=0) && (m<16) ); + assert(mType==INNER); if(mHashes[m]==hash) return false; mHashes[m]=hash; return updateHash(); } -const uint256& SHAMapInnerNode::getChildHash(int m) const +const uint256& SHAMapTreeNode::getChildHash(int m) const { - assert( (m>=0) && (m<32) ); + assert( (m>=0) && (m<16) && (mType==INNER) ); return mHashes[m]; } - -bool SHAMapInnerNode::isEmpty() const -{ - for(int i=0; i<32; i++) - if(mHashes[i]!=0) return false; - return true; -} - -bool SHAMapInnerNode::updateHash() -{ - uint256 j[2]; - if(!isEmpty()) - SHA512(reinterpret_cast(mHashes), sizeof(mHashes), (unsigned char *) j); - if(mHash==j[0]) return false; - mHash=j[0]; - return true; -} - -void SHAMapInnerNode::dump() -{ - std::cerr << "SHAMapInnerNode(" << getDepth() << ", " << getNodeID().GetHex() << ")" << std::endl; - - int children=0; - for(int i=0; i<32; i++) - if(!!mHashes[i]) children++; - - std::cerr << " " << children << " child(ren)" << std::endl; -} diff --git a/SHAMapSync.cpp b/SHAMapSync.cpp index 9b7960f423..e6643ee111 100644 --- a/SHAMapSync.cpp +++ b/SHAMapSync.cpp @@ -1,6 +1,8 @@ #include +#include + #include #include "SHAMap.h" @@ -8,6 +10,8 @@ void SHAMap::getMissingNodes(std::vector& nodeIDs, std::vector& hashes, int max) { boost::recursive_mutex::scoped_lock sl(mLock); + + assert(root->isValid()); if(root->isFullBelow()) { @@ -17,48 +21,44 @@ void SHAMap::getMissingNodes(std::vector& nodeIDs, std::vector stack; + if(!root->isInner()) + { + std::cerr << "synching empty tree" << std::endl; + return; + } + + std::stack stack; stack.push(root); while( (max>0) && (!stack.empty()) ) { - SHAMapInnerNode::pointer node=stack.top(); + SHAMapTreeNode::pointer node=stack.top(); stack.pop(); #ifdef GMN_DEBUG std::cerr << "gMN: popped " << node->getString() << std::endl; #endif - for(int i=0; i<32; i++) - { + for(int i=0; i<16; i++) if(!node->isEmptyBranch(i)) { -#ifdef GNMN_DEBUG +#ifdef GMN_DEBUG std::cerr << "gMN: " << node->getString() << " has non-empty branch " << i << std::endl; #endif - bool missing=false; - SHAMapNode childNID=node->getChildNodeID(i); - if(node->isChildLeaf()) - { // do we have this leaf node? - if(!getLeaf(childNID, node->getChildHash(i), false)) missing=true; - } - else + SHAMapTreeNode::pointer desc=getNode(node->getChildNodeID(i), node->getChildHash(i), false); + if(desc) { - SHAMapInnerNode::pointer desc=getInner(childNID, node->getChildHash(i), false); - if(!desc) - missing=true; - else if(!desc->isFullBelow()) + if(desc->isInner() && !desc->isFullBelow()) stack.push(desc); } - if(missing && max-->0) + else if(max-- > 0) { #ifdef GMN_DEBUG std::cerr << "gMN: need " << node->getChildNodeID(i).getString() << std::endl; #endif - nodeIDs.push_back(childNID); + nodeIDs.push_back(node->getChildNodeID(i)); } } - } } } @@ -66,21 +66,11 @@ bool SHAMap::getNodeFat(const SHAMapNode& wanted, std::vector& nodeI std::list >& rawNodes) { boost::recursive_mutex::scoped_lock sl(mLock); - if(wanted.isLeaf()) - { // no fat way to get a leaf - SHAMapLeafNode::pointer leaf=getLeafNode(wanted); - if(!leaf) return false; - nodeIDs.push_back(*leaf); - Serializer s; - leaf->addRaw(s); - rawNodes.push_back(s.peekData()); - return true; - } - SHAMapInnerNode::pointer node=getInnerNode(wanted); + SHAMapTreeNode::pointer node=getNode(wanted); if(!node) { - assert(false); + assert(false); // Remove for release, this can happen if we get a bogus request return false; } @@ -92,46 +82,21 @@ bool SHAMap::getNodeFat(const SHAMapNode& wanted, std::vector& nodeI if(wanted.isRoot()) // don't get a fat root return true; - bool ret=true; - for(int i=0; i<32; i++) - { + for(int i=0; i<16; i++) if(!node->isEmptyBranch(i)) { - if(node->isChildLeaf()) - { - SHAMapLeafNode::pointer leaf=getLeaf(node->getChildNodeID(i), node->getChildHash(i), false); - if(!leaf) - { - assert(false); - ret=false; - } - else - { - nodeIDs.push_back(*leaf); - Serializer s; - leaf->addRaw(s); - rawNodes.push_back(s.peekData()); - } - } - else - { - SHAMapInnerNode::pointer ino=getInner(node->getChildNodeID(i), node->getChildHash(i), false); - if(!ino) - { - assert(false); - ret=false; - } - else - { - nodeIDs.push_back(*ino); - Serializer s; - ino->addRaw(s); - rawNodes.push_back(s.peekData()); - } + SHAMapTreeNode::pointer nextNode=getNode(node->getChildNodeID(i), node->getChildHash(i), false); + assert(nextNode); + if(nextNode) + { + nodeIDs.push_back(*nextNode); + Serializer s; + nextNode->addRaw(s); + rawNodes.push_back(s.peekData()); } } - } - return ret; + + return true; } bool SHAMap::addRootNode(const std::vector& rootNode) @@ -139,7 +104,7 @@ bool SHAMap::addRootNode(const std::vector& rootNode) boost::recursive_mutex::scoped_lock sl(mLock); // we already have a root node - if(root->getNodeHash()!=0) + if(!!root->getNodeHash()) { #ifdef DEBUG std::cerr << "got root node, already have one" << std::endl; @@ -147,17 +112,19 @@ bool SHAMap::addRootNode(const std::vector& rootNode) return true; } - SHAMapInnerNode::pointer node=SHAMapInnerNode::pointer(new SHAMapInnerNode(SHAMapNode(), rootNode, 0)); + SHAMapTreeNode::pointer node=boost::make_shared(SHAMapNode(), rootNode, 0); if(!node) return false; #ifdef DEBUG node->dump(); #endif + returnNode(root, true); + root=node; - mInnerNodeByID[*node]=node; - if(mDirtyInnerNodes) (*mDirtyInnerNodes)[*node]=node; + mTNByID[*root]=root; if(!root->getNodeHash()) root->setFullBelow(); + return true; } @@ -166,7 +133,7 @@ bool SHAMap::addRootNode(const uint256& hash, const std::vector& boost::recursive_mutex::scoped_lock sl(mLock); // we already have a root node - if(root->getNodeHash()!=0) + if(!!root->getNodeHash()) { #ifdef DEBUG std::cerr << "got root node, already have one" << std::endl; @@ -175,14 +142,15 @@ bool SHAMap::addRootNode(const uint256& hash, const std::vector& return true; } - SHAMapInnerNode::pointer node=SHAMapInnerNode::pointer(new SHAMapInnerNode(SHAMapNode(), rootNode, 0)); + SHAMapTreeNode::pointer node=boost::make_shared(SHAMapNode(), rootNode, 0); if(!node) return false; if(node->getNodeHash()!=hash) return false; + returnNode(root, true); root=node; - mInnerNodeByID[*node]=node; - if(mDirtyInnerNodes) (*mDirtyInnerNodes)[*node]=node; + mTNByID[*root]=root; if(!root->getNodeHash()) root->setFullBelow(); + return true; } @@ -193,23 +161,19 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vector stack=getStack(node.getNodeID(), true); + if(stack.empty()) return false; + + SHAMapTreeNode::pointer iNode=stack.top(); if(!iNode) { // we should always have a root assert(false); return true; } - if(iNode->getDepth()==node.getDepth()) + if(iNode->isLeaf() || (iNode->getDepth()==node.getDepth())) { #ifdef DEBUG std::cerr << "got inner node, already had it (late)" << std::endl; @@ -236,81 +200,46 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vectorgetChildHash(branch); if(!hash) return false; - if(node.isLeaf()) - { // leaf node - SHAMapLeafNode::pointer leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(node, rawNode, mSeq)); - if( (leaf->getNodeHash()!=hash) || (node!=(*leaf)) ) - { -#ifdef DEBUG - std::cerr << "leaf fails consistency check" << std::endl; -#endif - return false; - } - mLeafByID[node]=leaf; - if(mDirtyLeafNodes) (*mDirtyLeafNodes)[node]=leaf; - - // FIXME: This should check all sources - SHAMapInnerNode::pointer pNode=checkCacheNode(node.getParentNodeID()); - if(!pNode) - { - assert(false); - return false; - } - - for(int i=0; i<32; i++) - if(!checkCacheLeaf(pNode->getChildNodeID(i))) - return true; - pNode->setFullBelow(); - - while(!pNode->isRoot()) - { - pNode=checkCacheNode(pNode->getParentNodeID()); - if(!pNode) - { - assert(false); - return false; - } - for(int i=0; i<32; i++) - if(!checkCacheNode(pNode->getChildNodeID(i))) - return true; - pNode->setFullBelow(); - } - - return true; - } - - SHAMapInnerNode::pointer newNode=SHAMapInnerNode::pointer(new SHAMapInnerNode(node, rawNode, mSeq)); - if( (newNode->getNodeHash()!=hash) || (node!=newNode->getNodeID()) ) - { -#ifdef DEBUG - std::cerr << "inner node fails consistency check" << std::endl; - std::cerr << " Built: " << newNode->getString() << " h=" << newNode->getNodeHash().GetHex() << std::endl; - std::cerr << "Expected: " << node.getString() << " h=" << hash.GetHex() << std::endl; -#endif + SHAMapTreeNode::pointer newNode=boost::make_shared(node, rawNode, mSeq); + if(hash!=newNode->getNodeHash()) // these aren't the droids we're looking for return false; - } - mInnerNodeByID[node]=newNode; - if(mDirtyInnerNodes) (*mDirtyInnerNodes)[node]=newNode; -#ifdef ST_DEBUG - std::cerr << "Hooked: " << node.getString() << std::endl; -#endif + + mTNByID[*newNode]=newNode; + if(!newNode->isLeaf()) + return true; // only a leaf can fill a branch + + // did this new leaf cause its parents to fill up + do + { + iNode=stack.top(); + stack.pop(); + assert(iNode->isInner()); + for(int i=0; i<16; i++) + if(!iNode->isEmptyBranch(i)) + { + SHAMapTreeNode::pointer nextNode=getNode(iNode->getChildNodeID(i), iNode->getChildHash(i), false); + if(!nextNode) return true; + if(nextNode->isInner() && !nextNode->isFullBelow()) return true; + } + iNode->setFullBelow(); + } while(!stack.empty()); return true; } bool SHAMap::deepCompare(SHAMap& other) { // Intended for debug/test only - std::stack stack; + std::stack stack; boost::recursive_mutex::scoped_lock sl(mLock); stack.push(root); while(!stack.empty()) { - SHAMapInnerNode::pointer node=stack.top(); + SHAMapTreeNode::pointer node=stack.top(); stack.pop(); - SHAMapInnerNode::pointer otherNode; + SHAMapTreeNode::pointer otherNode; if(node->isRoot()) otherNode=other.root; - else otherNode=other.getInner(*node, node->getNodeHash(), false); + else otherNode=other.getNode(*node, node->getNodeHash(), false); if(!otherNode) { @@ -327,38 +256,31 @@ bool SHAMap::deepCompare(SHAMap& other) std::cerr << "Comparing inner nodes " << node->getString() << std::endl; #endif - for(int i=0; i<32; i++) + if(node->getNodeHash() != otherNode->getNodeHash()) + return false; + if(node->isLeaf()) { - if(node->isEmptyBranch(i)) + if(!otherNode->isLeaf()) + return false; + if(node->peekItem()->getTag()!=otherNode->peekItem()->getTag()) + return false; + if(node->peekItem()->getData()!=otherNode->peekItem()->getData()) + return false; + } + else if(node->isInner()) + { + if(!otherNode->isInner()) + return false; + for(int i=0; i<16; i++) { - if(!otherNode->isEmptyBranch(i)) - return false; - } - else - { - if(node->isChildLeaf()) - { // - SHAMapLeafNode::pointer leaf=getLeaf(node->getChildNodeID(i), node->getChildHash(i), false); - if(!leaf) - { - std::cerr << "unable to fetch leaf" << std::endl; + if(node->isEmptyBranch(i)) + { + if(!otherNode->isEmptyBranch(i)) return false; - } - SHAMapLeafNode::pointer otherLeaf=other.getLeaf(*leaf, leaf->getNodeHash(), false); - if(!otherLeaf) - { - std::cerr << "unable to fetch other leaf" << std::endl; - return false; - } - if(leaf->getNodeHash()!=otherLeaf->getNodeHash()) - { - std::cerr << "leaf hash mismatch" << std::endl; - return false; - } } else - { // do we have this inner node? - SHAMapInnerNode::pointer next=getInner(node->getChildNodeID(i), node->getChildHash(i), false); + { + SHAMapTreeNode::pointer next=getNode(node->getChildNodeID(i), node->getChildHash(i), false); if(!next) { std::cerr << "unable to fetch inner node" << std::endl; @@ -385,15 +307,16 @@ bool SHAMap::syncTest() for(int i=0; i=mData.size()) return false; + byte=mData[offset]; + return true; +} + +bool Serializer::chop(int bytes) +{ + if(bytes>mData.size()) return false; + mData.resize(mData.size()-bytes); + return true; +} + +int Serializer::removeLastByte() +{ + int size=mData.size()-1; + if(size<0) + { + assert(false); + return -1; + } + int ret=mData[size]; + mData.resize(size); + return ret; +} + bool Serializer::getRaw(std::vector& o, int offset, int length) const { if((offset+length)>mData.size()) return false; diff --git a/Serializer.h b/Serializer.h index 2c0cc0fd71..e9621b288e 100644 --- a/Serializer.h +++ b/Serializer.h @@ -21,6 +21,7 @@ class Serializer Serializer(const std::vector &data) : mData(data) { ; } // assemble functions + int add1(unsigned char byte); int add16(uint16); int add32(uint32); // ledger indexes, account sequence int add64(uint64); // timestamps, amounts @@ -30,6 +31,8 @@ class Serializer int addRaw(const void *ptr, int len); // disassemble functions + bool get1(int&, int offset) const; + bool get1(unsigned char&, int offset) const; bool get16(uint16&, int offset) const; bool get32(uint32&, int offset) const; bool get64(uint64&, int offset) const; @@ -53,6 +56,8 @@ class Serializer std::vector getData() const { return mData; } void secureErase() { memset(&(mData.front()), 0, mData.size()); erase(); } void erase() { mData.clear(); } + int removeLastByte(); + bool chop(int num); // signature functions bool checkSignature(int pubkeyOffset, int signatureOffset) const;