Complete the SHAMapSync unit test. The test currently fails. Debug in progress

SHAMap.cpp

@@ -6,7 +6,7 @@
 #include "BitcoinUtil.h"
 #include "SHAMap.h"
 
-SHAMap::SHAMap(uint32 seq) : mSeq(seq)
+SHAMap::SHAMap(uint32 seq) : mSeq(seq), mImmutable(false), mSynching(false)
 {
 	root=SHAMapInnerNode::pointer(new SHAMapInnerNode(SHAMapNode(SHAMapNode::rootDepth, uint256()), mSeq));
 	mInnerNodeByID[*root]=root;
@@ -15,9 +15,11 @@ SHAMap::SHAMap(uint32 seq) : mSeq(seq)
 void SHAMap::dirtyUp(const uint256& id)
 { // walk the tree up from through the inner nodes to the root
   // update linking hashes and add nodes to dirty list
+
 #ifdef DEBUG
 	std::cerr << "dirtyUp(" << id.GetHex() << ")" << std::endl;
 #endif
+	assert(!mImmutable && !mSynching);
 	SHAMapLeafNode::pointer leaf=checkCacheLeaf(SHAMapNode(SHAMapNode::leafDepth, id));
 	if(!leaf) throw SHAMapException(MissingNode);
 
@@ -547,15 +549,20 @@ SHAMapInnerNode::pointer SHAMap::getInnerNode(const SHAMapNode& node)
 	SHAMapInnerNode::pointer inNode=root;
 	for(int i=0; i<SHAMapNode::leafDepth; i++)
 	{
-		int branch=inNode->selectBranch(node.getNodeID());
-		if( (branch<0) || (inNode->isEmptyBranch(branch)) ) return SHAMapInnerNode::pointer();
-		inNode=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), false);
-		if(!inNode) return inNode;
 		if(inNode->getDepth()==node.getDepth())
 		{
 			if((*inNode)!=node) return SHAMapInnerNode::pointer();
 			return inNode;
 		}
+		int branch=inNode->selectBranch(node.getNodeID());
+		if( (branch<0) || (inNode->isEmptyBranch(branch)) ) return SHAMapInnerNode::pointer();
+		inNode=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), false);
+		if(!inNode) return inNode;
+	}
+	if(inNode->getDepth()==node.getDepth())
+	{
+		if((*inNode)!=node) return SHAMapInnerNode::pointer();
+		return inNode;
 	}
 	return SHAMapInnerNode::pointer();
 }
@@ -597,39 +604,42 @@ static std::vector<unsigned char>IntToVUC(int i)
 
 bool SHAMap::TestSHAMap()
 { // h3 and h4 differ only in the leaf, same terminal node (level 19)
- uint256 h1, h2, h3, h4, h5;
+	uint256 h1, h2, h3, h4, h5;
- h1.SetHex("092891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7");
+	h1.SetHex("092891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7");
- h2.SetHex("436ccbac3347baa1f1e53baeef1f43334da88f1f6d70d963b833afd6dfa289fe");
+	h2.SetHex("436ccbac3347baa1f1e53baeef1f43334da88f1f6d70d963b833afd6dfa289fe");
- h3.SetHex("b92891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
+	h3.SetHex("b92891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
- h4.SetHex("b92891fe4ef6cee585fdc6fda2e09eb4d386363158ec3321b8123e5a772c6ca8");
+	h4.SetHex("b92891fe4ef6cee585fdc6fda2e09eb4d386363158ec3321b8123e5a772c6ca8");
- h5.SetHex("a92891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7");
+	h5.SetHex("a92891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7");
 
- SHAMap sMap;
+	SHAMap sMap;
- SHAMapItem i1(h1, IntToVUC(1)), i2(h2, IntToVUC(2)), i3(h3, IntToVUC(3)), i4(h4, IntToVUC(4)), i5(h5, IntToVUC(5));
+	SHAMapItem i1(h1, IntToVUC(1)), i2(h2, IntToVUC(2)), i3(h3, IntToVUC(3)), i4(h4, IntToVUC(4)), i5(h5, IntToVUC(5));
 
- sMap.addItem(i2);
+	sMap.addItem(i2);
- sMap.addItem(i1);
+	sMap.addItem(i1);
 
- SHAMapItem::pointer i=sMap.peekFirstItem();
+	SHAMapItem::pointer i=sMap.peekFirstItem();
- assert(!!i && (*i==i1));
+	assert(!!i && (*i==i1));
- i=sMap.peekNextItem(i->getTag());
+	i=sMap.peekNextItem(i->getTag());
- assert(!!i && (*i==i2));
+	assert(!!i && (*i==i2));
- i=sMap.peekNextItem(i->getTag());
+	i=sMap.peekNextItem(i->getTag());
- assert(!i);
+	assert(!i);
 
- sMap.addItem(i4);
+	sMap.addItem(i4);
- sMap.delItem(i2.getTag());
+	sMap.delItem(i2.getTag());
- sMap.addItem(i3);
+	sMap.addItem(i3);
- 
- i=sMap.peekFirstItem();
- assert(!!i && (*i==i1));
- i=sMap.peekNextItem(i->getTag());
- assert(!!i && (*i==i3));
- i=sMap.peekNextItem(i->getTag());
- assert(!!i && (*i==i4));
- i=sMap.peekNextItem(i->getTag());
- assert(!i);
 
- return true;
+	i=sMap.peekFirstItem();
+	assert(!!i && (*i==i1));
+	i=sMap.peekNextItem(i->getTag());
+	assert(!!i && (*i==i3));
+	i=sMap.peekNextItem(i->getTag());
+	assert(!!i && (*i==i4));
+	i=sMap.peekNextItem(i->getTag());
+	assert(!i);
+
+	if(!syncTest());
+		 return false;
+
+	return true;
 }
 

SHAMap.h

@@ -225,6 +225,8 @@ private:
 
 	SHAMapInnerNode::pointer root;
 
+	bool mImmutable, mSynching;
+
 protected:
 	void dirtyUp(const uint256& id);
 
@@ -294,8 +296,15 @@ public:
 	bool getNodeFat(const SHAMapNode& node, std::vector<SHAMapNode>& nodeIDs,
 	 std::list<std::vector<unsigned char> >& rawNode);
 	bool addRootNode(const uint256& hash, const std::vector<unsigned char>& rootNode);
+	bool addRootNode(const std::vector<unsigned char>& rootNode);
 	bool addKnownNode(const SHAMapNode& nodeID, const std::vector<unsigned char>& rawNode);
 
+	// status functions
+	void setImmutable(void) { mImmutable=true; }
+	void clearImmutable(void) { mImmutable=false; }
+	void setSynching(void) { mSynching=true; }
+	void clearSynching(void) { mSynching=false; }
+
 	// caution: otherMap must be accessed only by this function
 	// return value: true=successfully completed, false=too different
 	bool compare(SHAMap::pointer otherMap, SHAMapDiff& differences, int maxCount);
@@ -311,6 +320,7 @@ public:
 	bool operator==(const SHAMap& s) { return getHash()==s.getHash(); }
 
 	static bool TestSHAMap();
+	static bool syncTest();
 	bool deepCompare(SHAMap& other);
 	virtual void dump();
 };

SHAMapSync.cpp

@@ -7,7 +7,13 @@ void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint2
 {
 	boost::recursive_mutex::scoped_lock sl(mLock);
 	
-	if(root->isFullBelow()) return;
+	if(root->isFullBelow())
+	{
+#ifdef DEBUG
+		std::cerr << "getMissingNodes: root is full below" << std::endl;
+#endif
+		return;
+	}
 
 	std::stack<SHAMapInnerNode::pointer> stack;
 	stack.push(root);
@@ -20,6 +26,9 @@ void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint2
 		{
 			if(!node->isEmptyBranch(i))
 			{
+#ifdef DEBUG
+				std::cerr << "gMN: " << node->getString() << " has non-empty branch " << i << std::endl;
+#endif
 				if(node->isChildLeaf())
 				{ // do we have this leaf node?
 					SHAMapLeafNode::pointer leaf=getLeaf(node->getChildNodeID(i), node->getChildHash(i), false);
@@ -27,6 +36,9 @@ void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint2
 					{
 						if(max-->0)
 						{
+#ifdef DEBUG
+							std::cerr << "gMN: need leaf " << i << std::endl;
+#endif
 							nodeIDs.push_back(node->getChildNodeID(i));
 							hashes.push_back(node->getChildHash(i));
 						}
@@ -45,7 +57,13 @@ void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint2
 						}
 						all_leaves=false;
 					}
-					else if(!desc->isFullBelow()) stack.push(desc);
+					else if(!desc->isFullBelow())
+					{
+#ifdef DEBUG
+						std::cerr << "gMN: iterate " << desc->getString() << std::endl;
+#endif
+						stack.push(desc);
+					}
 				}
 			}
 		}
@@ -105,6 +123,28 @@ bool SHAMap::getNodeFat(const SHAMapNode& wanted, std::vector<SHAMapNode>& nodeI
 	return ret;
 }
 
+bool SHAMap::addRootNode(const std::vector<unsigned char>& rootNode)
+{
+	boost::recursive_mutex::scoped_lock sl(mLock);
+
+	// we already have a root node
+	if(root->getNodeHash()!=0)
+	{
+#ifdef DEBUG
+		std::cerr << "got root node, already have one" << std::endl;
+#endif
+		return true;
+	}
+
+	SHAMapInnerNode::pointer node=SHAMapInnerNode::pointer(new SHAMapInnerNode(SHAMapNode(), rootNode, 0));
+	if(!node) return false;
+
+	root=node;
+	mInnerNodeByID[*node]=node;
+	if(mDirtyInnerNodes) (*mDirtyInnerNodes)[*node]=node;
+	return true;
+}
+
 bool SHAMap::addRootNode(const uint256& hash, const std::vector<unsigned char>& rootNode)
 {
 	boost::recursive_mutex::scoped_lock sl(mLock);
@@ -208,9 +248,9 @@ bool SHAMap::deepCompare(SHAMap& other)
 { // Intended for debug/test only
 	std::stack<SHAMapInnerNode::pointer> stack;
 	boost::recursive_mutex::scoped_lock sl(mLock);
-	SHAMapInnerNode::pointer node=root;
 
-	while(node)
+	stack.push(root);
+	while(!stack.empty())
 	{
 		SHAMapInnerNode::pointer node=stack.top();
 		stack.pop();
@@ -278,3 +318,112 @@ bool SHAMap::deepCompare(SHAMap& other)
 	}
 	return true;
 }
+
+bool SHAMap::syncTest()
+{
+	SHAMap source, destination;
+
+	// add random data to the source map
+	int items=10+rand()%400;
+	for(int i=0; i<items; i++)
+	{
+		Serializer s;
+		int dlen=rand()%30+4;
+		for(int d=0; d<dlen; d++)
+			s.add32(rand());
+		source.addItem(SHAMapItem(s.getSHA512Half(), s.peekData()));		
+	}
+	source.setImmutable();
+
+	std::vector<SHAMapNode> nodeIDs, gotNodeIDs;
+	std::list<std::vector<unsigned char> > gotNodes;
+	std::vector<uint256> hashes;
+
+	std::vector<SHAMapNode>::iterator nodeIDIterator;
+	std::list<std::vector<unsigned char> >::iterator rawNodeIterator;
+
+	int passes=0;
+	int nodes=0;
+
+	destination.setSynching();
+
+	if(!source.getNodeFat(SHAMapNode(), nodeIDs, gotNodes))
+	{
+		std::cerr << "GetNodeFat(root) fails" << std::endl;
+		assert(false);
+		return false;
+	}
+	if(gotNodes.size()!=1)
+	{
+		std::cerr << "Didn't get root node" << std::endl;
+		assert(false);
+		return false;
+	}
+	if(!destination.addRootNode(*gotNodes.begin()))
+	{
+		std::cerr << "AddRootNode fails" << std::endl;
+		assert(false);
+		return false;
+	}
+	nodeIDs.clear();
+	gotNodes.clear();
+
+	do
+	{
+		passes++;
+		hashes.clear();
+
+		// get the list of nodes we know we need
+		source.getMissingNodes(nodeIDs, hashes, 128);
+		if(!nodeIDs.size()) break;
+		
+		// get as many nodes as possible based on this information
+		for(nodeIDIterator=nodeIDs.begin(); nodeIDIterator!=nodeIDs.end(); ++nodeIDIterator)
+			if(!source.getNodeFat(*nodeIDIterator, nodeIDs, gotNodes))
+			{
+				std::cerr << "GetNodeFat fails" << std::endl;
+				assert(false);
+				return false;
+			}
+		nodeIDs.clear();
+		hashes.clear();
+
+		if(!gotNodeIDs.size())
+		{
+			std::cerr << "No nodes gotten" << std::endl;
+			assert(false);
+			return false;
+		}
+
+		for(nodeIDIterator=gotNodeIDs.begin(), rawNodeIterator=gotNodes.begin();
+				nodeIDIterator!=gotNodeIDs.end(); ++nodeIDIterator, ++rawNodeIterator)
+		{
+			nodes++;
+			if(!destination.addKnownNode(*nodeIDIterator, *rawNodeIterator))
+			{
+				std::cerr << "AddKnownNode fails" << std::endl;
+				assert(false);
+				return false;
+			}
+		}
+		nodeIDs.clear();
+		gotNodes.clear();
+
+
+	} while(1);
+	destination.clearSynching();
+
+	if(!source.deepCompare(destination))
+	{
+		std::cerr << "DeepCompare fails" << std::endl;
+		assert(false);
+		return false;
+	}
+
+#ifdef DEBUG
+	std::cerr << "SHAMapSync test passed: " << items << " items, " <<
+		passes << " passes, " << nodes << " nodes" << std::endl;
+#endif
+	
+	return true;
+}