More work on the ledger/SHAMap sync code.

Makefile

@@ -67,8 +67,8 @@ HEADERS = \
 
 SRCS= keystore.cpp BitcoinUtil.cpp \
  main.cpp Hanko.cpp Transaction.cpp SHAMap.cpp SHAMapNodes.cpp Serializer.cpp Ledger.cpp \
- AccountState.cpp Wallet.cpp NewcoinAddress.cpp Config.cpp PackedMessage.cpp \
+ AccountState.cpp Wallet.cpp NewcoinAddress.cpp Config.cpp PackedMessage.cpp SHAMapSync.cpp \
- Application.cpp TimingService.cpp KnownNodeList.cpp ConnectionPool.cpp Peer.cpp \
+ Application.cpp TimingService.cpp KnownNodeList.cpp ConnectionPool.cpp Peer.cpp LedgerAcquire.cpp \
  PeerDoor.cpp RPCDoor.cpp RPCServer.cpp rpc.cpp Conversion.cpp RequestParser.cpp HashedObject.cpp \
  UniqueNodeList.cpp PubKeyCache.cpp SHAMapDiff.cpp DeterministicKeys.cpp LedgerMaster.cpp \
  LedgerHistory.cpp NetworkOPs.cpp CallRPC.cpp DBInit.cpp LocalTransaction.cpp TransactionMaster.cpp

SHAMap.cpp

@@ -1,10 +1,11 @@
-#include "Serializer.h"
-#include "BitcoinUtil.h"
-#include "SHAMap.h"
 
 #include <boost/foreach.hpp>
 #include <boost/lexical_cast.hpp>
 
+#include "Serializer.h"
+#include "BitcoinUtil.h"
+#include "SHAMap.h"
+
 SHAMap::SHAMap(uint32 seq) : mSeq(seq)
 {
 	root=SHAMapInnerNode::pointer(new SHAMapInnerNode(SHAMapNode(SHAMapNode::rootDepth, uint256()), mSeq));
@@ -512,6 +513,41 @@ int SHAMap::flushDirty(int maxNodes, HashedObjectType t, uint32 seq)
 	return flushed;
 }
 
+SHAMapInnerNode::pointer SHAMap::getInnerNode(const SHAMapNode& node)
+{
+	boost::recursive_mutex::scoped_lock sl(mLock);
+	SHAMapInnerNode::pointer inNode=root;
+	for(int i=0; i<SHAMapNode::leafDepth; i++)
+	{
+		int branch=inNode->selectBranch(node.getNodeID());
+		if( (branch<0) || (inNode->isEmptyBranch(branch)) ) return SHAMapInnerNode::pointer();
+		inNode=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), false);
+		if(!inNode) return inNode;
+		if(inNode->getDepth()==node.getDepth())
+		{
+			if((*inNode)!=node) return SHAMapInnerNode::pointer();
+			return inNode;
+		}
+	}
+	return SHAMapInnerNode::pointer();
+}
+
+SHAMapLeafNode::pointer SHAMap::getLeafNode(const SHAMapNode& leaf)
+{
+	boost::recursive_mutex::scoped_lock sl(mLock);
+	SHAMapInnerNode::pointer inNode=root;
+	for(int i=0; inNode->getDepth()<SHAMapNode::leafDepth; i++)
+	{
+		int branch=inNode->selectBranch(leaf.getNodeID());
+		if( (branch<0) || (inNode->isEmptyBranch(branch)) ) return SHAMapLeafNode::pointer();
+		inNode=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), false);
+		if(!inNode) return SHAMapLeafNode::pointer();
+	}
+	int branch=inNode->selectBranch(leaf.getNodeID());
+	if( (branch<0) || (inNode->isEmptyBranch(branch)) ) return SHAMapLeafNode::pointer();
+	return getLeaf(inNode->getChildNodeID(branch), inNode->getChildHash(branch), false);
+}
+
 void SHAMap::dump()
 {
 	std::cerr << "SHAMap::dump" << std::endl;

SHAMap.h

@@ -167,6 +167,7 @@ private:
 	uint256		mHash;
 	uint256		mHashes[32];
 	uint32		mSeq;
+	bool		mFullBelow;	// we have all nodes below this node
 
 	bool updateHash();
 
@@ -188,6 +189,8 @@ public:
 
 	virtual bool isPopulated() const { return true; }
 
+	bool isFullBelow(void) const		{ return mFullBelow; }
+	void setFullBelow(void)				{ mFullBelow=true; }
 	bool isEmptyBranch(int m) const		{ return !mHashes[m]; }
 	const uint256& getNodeHash() const  { return mHash; }
 	const uint256& getChildHash(int m) const;
@@ -282,11 +285,10 @@ public:
 	SHAMapItem::pointer peekNextItem(const uint160& u) { return peekNextItem(u.to256()); }
 
 	// comparison/sync functions
-	void getMissingNodes(std::vector<SHAMapNode>& nodeHashes, int max);
+	void getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint256>& hashes, int max);
-	void getMissingObjects(std::vector<uint256>& objectHashes, int max);
+	bool getNodeFat(const SHAMapNode& node, std::vector<SHAMapNode>& nodeIDs,
-	bool getNodeFat(const SHAMapNode& node, std::vector<uint256>& nodeHashes, int max);
+	 std::list<std::vector<unsigned char> >& rawNode);
-	bool getNodeFat(const uint256& hash, std::vector<uint256>& nodeHashes, int max);
+	bool addKnownNode(const SHAMapNode& nodeID, const std::vector<unsigned char>& rawNode);
-	bool addKnownNode(const std::vector<unsigned char>& rawNode);
 
 	// caution: otherMap must be accessed only by this function
 	// return value: true=successfully completed, false=too different

SHAMapNodes.cpp

@@ -282,13 +282,13 @@ void SHAMapLeafNode::dump()
 	std::cerr << "  " << mItems.size() << " items" << std::endl;
 }
 
-SHAMapInnerNode::SHAMapInnerNode(const SHAMapNode& id, uint32 seq) : SHAMapNode(id), mSeq(seq)
+SHAMapInnerNode::SHAMapInnerNode(const SHAMapNode& id, uint32 seq) : SHAMapNode(id), mSeq(seq), mFullBelow(false)
 { // can be root
 	assert(id.getDepth()<SHAMapNode::leafDepth);
 }
 
 SHAMapInnerNode::SHAMapInnerNode(const SHAMapNode& id, const std::vector<unsigned char>& contents, uint32 seq)
-	: SHAMapNode(id), mSeq(seq)
+	: SHAMapNode(id), mSeq(seq), mFullBelow(false)
 {
 	assert(!id.isLeaf());
 	assert(contents.size()==32*256/8);
@@ -298,7 +298,7 @@ SHAMapInnerNode::SHAMapInnerNode(const SHAMapNode& id, const std::vector<unsigne
 }
 
 SHAMapInnerNode::SHAMapInnerNode(const SHAMapInnerNode& node, uint32 seq) : SHAMapNode(node), mHash(node.mHash),
-		mSeq(seq)
+		mSeq(seq), mFullBelow(false)
 {
 	assert(!node.isLeaf());
 	memcpy(mHashes, node.mHashes, sizeof(mHashes));

SHAMapSync.cpp

@@ -0,0 +1,112 @@
+
+#include <stack>
+
+#include "SHAMap.h"
+
+void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint256>& hashes, int max)
+{
+	boost::recursive_mutex::scoped_lock sl(mLock);
+	
+	if(root->isFullBelow()) return;
+
+	std::stack<SHAMapInnerNode::pointer> stack;
+	stack.push(root);
+	while( (max>0) && (!stack.empty()) )
+	{
+		SHAMapInnerNode::pointer node=stack.top();
+		stack.pop();
+		bool all_leaves=true;
+		for(int i=0; i<32; i++)
+		{
+			if(!node->isEmptyBranch(i))
+			{
+				if(node->isChildLeaf())
+				{ // do we have this leaf node?
+					SHAMapLeafNode::pointer leaf=getLeaf(node->getChildNodeID(i), node->getChildHash(i), false);
+					if(!leaf)
+					{
+						if(max-->0)
+						{
+							nodeIDs.push_back(node->getChildNodeID(i));
+							hashes.push_back(node->getChildHash(i));
+						}
+						all_leaves=false;
+					}
+				}
+				else
+				{ // do we have this inner node?
+					SHAMapInnerNode::pointer desc=getInner(node->getChildNodeID(i), node->getChildHash(i), false);
+					if(!desc)
+					{
+						if(max-->0)
+						{
+							nodeIDs.push_back(node->getChildNodeID(i));
+							hashes.push_back(node->getChildHash(i));
+						}
+						all_leaves=false;
+					}
+					else if(!desc->isFullBelow()) stack.push(desc);
+				}
+			}
+		}
+		if(all_leaves) node->setFullBelow();
+	}
+}
+
+bool SHAMap::getNodeFat(const SHAMapNode& wanted, std::vector<SHAMapNode>& nodeIDs,
+	std::list<std::vector<unsigned char> >& rawNodes)
+{
+	boost::recursive_mutex::scoped_lock sl(mLock);
+	if(wanted.isLeaf())
+	{ // no fat way to get a leaf
+		SHAMapLeafNode::pointer leaf=getLeafNode(wanted);
+		if(!leaf) return false;
+		nodeIDs.push_back(*leaf);
+		Serializer s;
+		leaf->addRaw(s);
+		rawNodes.push_back(s.peekData());
+		return true;
+	}
+
+	SHAMapInnerNode::pointer node=getInnerNode(wanted);
+	if(!node) return false;
+
+	bool ret=true;
+	for(int i=0; i<32; i++)
+	{
+		if(!node->isEmptyBranch(i))
+		{
+		 	if(node->isChildLeaf())
+		 	{
+		 		SHAMapLeafNode::pointer leaf=getLeaf(node->getChildNodeID(i), node->getChildHash(i), false);
+		 		if(!leaf) ret=false;
+		 		else
+		 		{
+		 			nodeIDs.push_back(*leaf);
+		 			Serializer s;
+		 			leaf->addRaw(s);
+		 			rawNodes.push_back(s.peekData());
+		 		}
+		 	}
+		 	else
+		 	{
+		 		SHAMapInnerNode::pointer ino=getInner(node->getChildNodeID(i), node->getChildHash(i), false);
+		 		if(!ino) ret=false;
+		 		else
+		 		{
+		 			nodeIDs.push_back(*ino);
+		 			Serializer s;
+		 			ino->addRaw(s);
+		 			rawNodes.push_back(s.peekData());
+		 		}
+		 	}
+		}
+	}
+	return ret;
+}
+
+bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vector<unsigned char>& rawNode)
+{
+	// WRITEME
+	return true;
+}

newcoin.proto

@@ -190,7 +190,7 @@ enum TMLedgerInfoType {
 message TMGetLedger {
 	required bytes ledgerHash		= 1;
 	required TMLedgerInfoType type	= 2;
-	repeated LedgerNodes nodes		= 3;
+	repeated bytes nodes			= 3;
 }
 
 message TMLedgerData {