From 5a52d6fec36d6a5135160c0f1e5b9f459a5e06f2 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 30 Jan 2012 20:56:38 -0800 Subject: [PATCH] More work on the ledger/SHAMap sync code. --- Makefile | 4 +- SHAMap.cpp | 42 ++++++++++++++++-- SHAMap.h | 12 +++--- SHAMapNodes.cpp | 6 +-- SHAMapSync.cpp | 112 ++++++++++++++++++++++++++++++++++++++++++++++++ newcoin.proto | 2 +- 6 files changed, 164 insertions(+), 14 deletions(-) create mode 100644 SHAMapSync.cpp diff --git a/Makefile b/Makefile index 2cbea7760e..dd05707875 100644 --- a/Makefile +++ b/Makefile @@ -67,8 +67,8 @@ HEADERS = \ SRCS= keystore.cpp BitcoinUtil.cpp \ main.cpp Hanko.cpp Transaction.cpp SHAMap.cpp SHAMapNodes.cpp Serializer.cpp Ledger.cpp \ - AccountState.cpp Wallet.cpp NewcoinAddress.cpp Config.cpp PackedMessage.cpp \ - Application.cpp TimingService.cpp KnownNodeList.cpp ConnectionPool.cpp Peer.cpp \ + AccountState.cpp Wallet.cpp NewcoinAddress.cpp Config.cpp PackedMessage.cpp SHAMapSync.cpp \ + Application.cpp TimingService.cpp KnownNodeList.cpp ConnectionPool.cpp Peer.cpp LedgerAcquire.cpp \ PeerDoor.cpp RPCDoor.cpp RPCServer.cpp rpc.cpp Conversion.cpp RequestParser.cpp HashedObject.cpp \ UniqueNodeList.cpp PubKeyCache.cpp SHAMapDiff.cpp DeterministicKeys.cpp LedgerMaster.cpp \ LedgerHistory.cpp NetworkOPs.cpp CallRPC.cpp DBInit.cpp LocalTransaction.cpp TransactionMaster.cpp diff --git a/SHAMap.cpp b/SHAMap.cpp index a0edff26a1..650c71c18e 100644 --- a/SHAMap.cpp +++ b/SHAMap.cpp @@ -1,10 +1,11 @@ -#include "Serializer.h" -#include "BitcoinUtil.h" -#include "SHAMap.h" #include #include +#include "Serializer.h" +#include "BitcoinUtil.h" +#include "SHAMap.h" + SHAMap::SHAMap(uint32 seq) : mSeq(seq) { root=SHAMapInnerNode::pointer(new SHAMapInnerNode(SHAMapNode(SHAMapNode::rootDepth, uint256()), mSeq)); @@ -512,6 +513,41 @@ int SHAMap::flushDirty(int maxNodes, HashedObjectType t, uint32 seq) return flushed; } +SHAMapInnerNode::pointer SHAMap::getInnerNode(const SHAMapNode& node) +{ + boost::recursive_mutex::scoped_lock sl(mLock); + SHAMapInnerNode::pointer inNode=root; + for(int i=0; iselectBranch(node.getNodeID()); + if( (branch<0) || (inNode->isEmptyBranch(branch)) ) return SHAMapInnerNode::pointer(); + inNode=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), false); + if(!inNode) return inNode; + if(inNode->getDepth()==node.getDepth()) + { + if((*inNode)!=node) return SHAMapInnerNode::pointer(); + return inNode; + } + } + return SHAMapInnerNode::pointer(); +} + +SHAMapLeafNode::pointer SHAMap::getLeafNode(const SHAMapNode& leaf) +{ + boost::recursive_mutex::scoped_lock sl(mLock); + SHAMapInnerNode::pointer inNode=root; + for(int i=0; inNode->getDepth()selectBranch(leaf.getNodeID()); + if( (branch<0) || (inNode->isEmptyBranch(branch)) ) return SHAMapLeafNode::pointer(); + inNode=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), false); + if(!inNode) return SHAMapLeafNode::pointer(); + } + int branch=inNode->selectBranch(leaf.getNodeID()); + if( (branch<0) || (inNode->isEmptyBranch(branch)) ) return SHAMapLeafNode::pointer(); + return getLeaf(inNode->getChildNodeID(branch), inNode->getChildHash(branch), false); +} + void SHAMap::dump() { std::cerr << "SHAMap::dump" << std::endl; diff --git a/SHAMap.h b/SHAMap.h index 47eb3982d7..d6b929bb75 100644 --- a/SHAMap.h +++ b/SHAMap.h @@ -167,6 +167,7 @@ private: uint256 mHash; uint256 mHashes[32]; uint32 mSeq; + bool mFullBelow; // we have all nodes below this node bool updateHash(); @@ -188,6 +189,8 @@ public: virtual bool isPopulated() const { return true; } + bool isFullBelow(void) const { return mFullBelow; } + void setFullBelow(void) { mFullBelow=true; } bool isEmptyBranch(int m) const { return !mHashes[m]; } const uint256& getNodeHash() const { return mHash; } const uint256& getChildHash(int m) const; @@ -282,11 +285,10 @@ public: SHAMapItem::pointer peekNextItem(const uint160& u) { return peekNextItem(u.to256()); } // comparison/sync functions - void getMissingNodes(std::vector& nodeHashes, int max); - void getMissingObjects(std::vector& objectHashes, int max); - bool getNodeFat(const SHAMapNode& node, std::vector& nodeHashes, int max); - bool getNodeFat(const uint256& hash, std::vector& nodeHashes, int max); - bool addKnownNode(const std::vector& rawNode); + void getMissingNodes(std::vector& nodeIDs, std::vector& hashes, int max); + bool getNodeFat(const SHAMapNode& node, std::vector& nodeIDs, + std::list >& rawNode); + bool addKnownNode(const SHAMapNode& nodeID, const std::vector& rawNode); // caution: otherMap must be accessed only by this function // return value: true=successfully completed, false=too different diff --git a/SHAMapNodes.cpp b/SHAMapNodes.cpp index a8ed3e6dc6..9adf87e1f9 100644 --- a/SHAMapNodes.cpp +++ b/SHAMapNodes.cpp @@ -282,13 +282,13 @@ void SHAMapLeafNode::dump() std::cerr << " " << mItems.size() << " items" << std::endl; } -SHAMapInnerNode::SHAMapInnerNode(const SHAMapNode& id, uint32 seq) : SHAMapNode(id), mSeq(seq) +SHAMapInnerNode::SHAMapInnerNode(const SHAMapNode& id, uint32 seq) : SHAMapNode(id), mSeq(seq), mFullBelow(false) { // can be root assert(id.getDepth()& contents, uint32 seq) - : SHAMapNode(id), mSeq(seq) + : SHAMapNode(id), mSeq(seq), mFullBelow(false) { assert(!id.isLeaf()); assert(contents.size()==32*256/8); @@ -298,7 +298,7 @@ SHAMapInnerNode::SHAMapInnerNode(const SHAMapNode& id, const std::vector + +#include "SHAMap.h" + +void SHAMap::getMissingNodes(std::vector& nodeIDs, std::vector& hashes, int max) +{ + boost::recursive_mutex::scoped_lock sl(mLock); + + if(root->isFullBelow()) return; + + std::stack stack; + stack.push(root); + while( (max>0) && (!stack.empty()) ) + { + SHAMapInnerNode::pointer node=stack.top(); + stack.pop(); + bool all_leaves=true; + for(int i=0; i<32; i++) + { + if(!node->isEmptyBranch(i)) + { + if(node->isChildLeaf()) + { // do we have this leaf node? + SHAMapLeafNode::pointer leaf=getLeaf(node->getChildNodeID(i), node->getChildHash(i), false); + if(!leaf) + { + if(max-->0) + { + nodeIDs.push_back(node->getChildNodeID(i)); + hashes.push_back(node->getChildHash(i)); + } + all_leaves=false; + } + } + else + { // do we have this inner node? + SHAMapInnerNode::pointer desc=getInner(node->getChildNodeID(i), node->getChildHash(i), false); + if(!desc) + { + if(max-->0) + { + nodeIDs.push_back(node->getChildNodeID(i)); + hashes.push_back(node->getChildHash(i)); + } + all_leaves=false; + } + else if(!desc->isFullBelow()) stack.push(desc); + } + } + } + if(all_leaves) node->setFullBelow(); + } +} + +bool SHAMap::getNodeFat(const SHAMapNode& wanted, std::vector& nodeIDs, + std::list >& rawNodes) +{ + boost::recursive_mutex::scoped_lock sl(mLock); + if(wanted.isLeaf()) + { // no fat way to get a leaf + SHAMapLeafNode::pointer leaf=getLeafNode(wanted); + if(!leaf) return false; + nodeIDs.push_back(*leaf); + Serializer s; + leaf->addRaw(s); + rawNodes.push_back(s.peekData()); + return true; + } + + SHAMapInnerNode::pointer node=getInnerNode(wanted); + if(!node) return false; + + bool ret=true; + for(int i=0; i<32; i++) + { + if(!node->isEmptyBranch(i)) + { + if(node->isChildLeaf()) + { + SHAMapLeafNode::pointer leaf=getLeaf(node->getChildNodeID(i), node->getChildHash(i), false); + if(!leaf) ret=false; + else + { + nodeIDs.push_back(*leaf); + Serializer s; + leaf->addRaw(s); + rawNodes.push_back(s.peekData()); + } + } + else + { + SHAMapInnerNode::pointer ino=getInner(node->getChildNodeID(i), node->getChildHash(i), false); + if(!ino) ret=false; + else + { + nodeIDs.push_back(*ino); + Serializer s; + ino->addRaw(s); + rawNodes.push_back(s.peekData()); + } + } + } + } + return ret; +} + +bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vector& rawNode) +{ + // WRITEME + return true; +} diff --git a/newcoin.proto b/newcoin.proto index 296cb9d8d1..7a5144c9ca 100644 --- a/newcoin.proto +++ b/newcoin.proto @@ -190,7 +190,7 @@ enum TMLedgerInfoType { message TMGetLedger { required bytes ledgerHash = 1; required TMLedgerInfoType type = 2; - repeated LedgerNodes nodes = 3; + repeated bytes nodes = 3; } message TMLedgerData {