Major optimization in the way we track SHAMaps.

SHAMap.cpp

@@ -67,7 +67,7 @@ SHAMapLeafNode::pointer SHAMap::checkCacheLeaf(const SHAMapNode& iNode)
 SHAMapInnerNode::pointer SHAMap::checkCacheNode(const SHAMapNode& iNode)
 {
 	assert(!iNode.isLeaf());
-	std::map<SHAMapNode, SHAMapInnerNode::pointer>::iterator it=mInnerNodeByID.find(iNode);
+	boost::unordered_map<SHAMapNode, SHAMapInnerNode::pointer>::iterator it=mInnerNodeByID.find(iNode);
 	if(it==mInnerNodeByID.end()) return SHAMapInnerNode::pointer();
 	return it->second;
 }
@@ -89,10 +89,6 @@ SHAMapLeafNode::pointer SHAMap::walkToLeaf(const uint256& id, bool create, bool
 			throw SHAMapException(InvalidNode);
 		if(inNode->isEmptyBranch(branch))
 		{ // no nodes below this one
-#ifdef DEBUG
-			std::cerr << "No nodes below level " << i << ", branch " << branch << std::endl;
-			std::cerr << "  terminal node is " << inNode->getString() << std::endl;
-#endif
 			if(!create) return SHAMapLeafNode::pointer();
 			return createLeaf(*inNode, id);
 		}
@@ -140,7 +136,7 @@ SHAMapInnerNode::pointer SHAMap::walkTo(const SHAMapNode& id)
 			SHAMapInnerNode::pointer next=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), false);
 			if(!next) // we don't have the next node
 			{
-#ifdef DEBUG
+#ifdef ST_DEBUG
 				std::cerr << "Unable to find node " << inNode->getChildNodeID(branch).getString() << std::endl;
 #endif
 				return inNode;
@@ -166,7 +162,7 @@ SHAMapLeafNode::pointer SHAMap::getLeaf(const SHAMapNode& id, const uint256& has
 	if(leaf) return returnLeaf(leaf, modify);
 
 	std::vector<unsigned char> leafData;
-	if(!fetchNode(hash, leafData)) throw SHAMapException(MissingNode);
+	if(!fetchNode(hash, leafData)) return SHAMapLeafNode::pointer();
 	leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(id, leafData, mSeq));
 	if(leaf->getNodeHash()!=hash) throw SHAMapException(InvalidNode);
 	mLeafByID.insert(std::make_pair(id, leaf));
@@ -179,7 +175,7 @@ SHAMapInnerNode::pointer SHAMap::getInner(const SHAMapNode& id, const uint256& h
 	if(node) return returnNode(node, modify);
 	
 	std::vector<unsigned char> rawNode;
-	if(!fetchNode(hash, rawNode)) throw SHAMapException(MissingNode);
+	if(!fetchNode(hash, rawNode)) return SHAMapInnerNode::pointer();
 	node=SHAMapInnerNode::pointer(new SHAMapInnerNode(id, rawNode, mSeq));
 	if(node->getNodeHash()!=hash) throw SHAMapException(InvalidNode);
 
@@ -421,7 +417,7 @@ SHAMapItem::pointer SHAMap::peekPrevItem(const uint256& id)
 
 SHAMapLeafNode::pointer SHAMap::createLeaf(const SHAMapInnerNode& lowestParent, const uint256& id)
 { // caller must call dirtyUp if they populate the leaf
-#ifdef DEBUG
+#ifdef ST_DEBUG
 	std::cerr << "createLeaf(" << lowestParent.getString() << std::endl;
 	std::cerr << "  for " << id.GetHex() << ")" << std::endl;
 #endif
@@ -433,7 +429,7 @@ SHAMapLeafNode::pointer SHAMap::createLeaf(const SHAMapInnerNode& lowestParent,
 	}
 	SHAMapLeafNode::pointer newLeaf(new SHAMapLeafNode(SHAMapNode(SHAMapNode::leafDepth, id), mSeq));
 	mLeafByID[*newLeaf]=newLeaf;
-#ifdef DEBUG
+#ifdef ST_DEBUG
 	std::cerr << "made leaf " << newLeaf->getString() << std::endl;
 #endif
 	return newLeaf;

SHAMap.h

@@ -7,6 +7,7 @@
 
 #include <boost/shared_ptr.hpp>
 #include <boost/enable_shared_from_this.hpp>
+#include <boost/unordered/unordered_map.hpp>
 
 #include "types.h"
 #include "uint256.h"
@@ -49,9 +50,9 @@ public:
 	bool isInner() const 			{ return !isRoot() && !isLeaf(); }
 	virtual bool isPopulated() const { return false; }
 
-	SHAMapNode getParentNodeID()	{ return SHAMapNode(mDepth-1, mNodeID); }
+	SHAMapNode getParentNodeID()	const	{ return SHAMapNode(mDepth-1, mNodeID); }
-	SHAMapNode getChildNodeID(int m);
+	SHAMapNode getChildNodeID(int m) const;
-	int selectBranch(const uint256& hash);
+	int selectBranch(const uint256& hash) const;
 
 	bool operator<(const SHAMapNode&) const;
 	bool operator>(const SHAMapNode&) const;
@@ -62,8 +63,10 @@ public:
 	bool operator<=(const SHAMapNode&) const;
 	bool operator>=(const SHAMapNode&) const;
 
+	std::size_t getHash() const;
+
 	virtual std::string getString() const;
-	void dump();
+	void dump() const;
 
 	static void ClassInit();
 	static uint256 getNodeID(int depth, const uint256& hash);
@@ -164,7 +167,6 @@ class SHAMapInnerNode : public SHAMapNode
 
 public:
 	typedef boost::shared_ptr<SHAMapInnerNode> pointer;
-
 private:
 	uint256		mHash;
 	uint256		mHashes[32];
@@ -208,6 +210,14 @@ enum SHAMapException
 	InvalidNode=2
 };
 
+class hash_SMN : std::unary_function<SHAMapNode, std::size_t>
+{
+public:
+	std::size_t operator() (const SHAMapNode& mn) const
+	{
+		return mn.getHash();
+	}
+};
 
 class SHAMap
 {
@@ -219,7 +229,9 @@ private:
 	uint32 mSeq;
 	mutable boost::recursive_mutex mLock;
 	std::map<SHAMapNode, SHAMapLeafNode::pointer> mLeafByID;
-	std::map<SHAMapNode, SHAMapInnerNode::pointer> mInnerNodeByID;
+
+	boost::unordered_map<SHAMapNode, SHAMapInnerNode::pointer, hash_SMN> mInnerNodeByID;
+
 	boost::shared_ptr<std::map<SHAMapNode, SHAMapLeafNode::pointer> > mDirtyLeafNodes;
  	boost::shared_ptr<std::map<SHAMapNode, SHAMapInnerNode::pointer> > mDirtyInnerNodes;
 

SHAMapNodes.cpp

@@ -81,13 +81,24 @@ uint256 SHAMapNode::getNodeID(int depth, const uint256& hash)
 	return hash & smMasks[depth];
 }
 
+std::size_t SHAMapNode::getHash() const
+{
+	std::size_t ret=mDepth;
+	for(int i=0; i<5; i++)
+	{
+		ret*=2654435761U;
+		ret^=mNodeID.PeekAt(i);
+	}
+	return ret;
+}
+
 SHAMapNode::SHAMapNode(int depth, const uint256 &hash) : mDepth(depth)
 { // canonicalize the hash to a node ID for this depth
 	assert(depth>=0 && depth<=leafDepth);
 	mNodeID = getNodeID(depth, hash);
 }
 
-SHAMapNode SHAMapNode::getChildNodeID(int m)
+SHAMapNode SHAMapNode::getChildNodeID(int m) const
 {
 	assert(!isLeaf());
 	assert((m>=0) && (m<32));
@@ -99,7 +110,7 @@ SHAMapNode SHAMapNode::getChildNodeID(int m)
 	return ret;
 }
 
-int SHAMapNode::selectBranch(const uint256 &hash)
+int SHAMapNode::selectBranch(const uint256 &hash) const
 {
 	if(isLeaf())	// no nodes under this node
 	{
@@ -118,7 +129,7 @@ int SHAMapNode::selectBranch(const uint256 &hash)
 	return branch;
 }
 
-void SHAMapNode::dump()
+void SHAMapNode::dump() const
 {
 	std::cerr << getString() << std::endl;
 }
@@ -172,7 +183,7 @@ bool SHAMapLeafNode::hasItem(const uint256& item) const
 
 bool SHAMapLeafNode::addUpdateItem(SHAMapItem::pointer item, bool doHash)
 { // The node will almost never have more than one item in it
-#ifdef DEBUG
+#ifdef ST_DEBUG
 	std::cerr << "Leaf(" << getString() << ")" << std::endl;
 	std::cerr << "  addi(" << item->getTag().GetHex() << std::endl;
 #endif

SHAMapSync.cpp

@@ -1,6 +1,8 @@
 
 #include <stack>
 
+#include <openssl/rand.h>
+
 #include "SHAMap.h"
 
 void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint256>& hashes, int max)
@@ -9,7 +11,7 @@ void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint2
 	
 	if(root->isFullBelow())
 	{
-#ifdef DEBUG
+#ifdef GMN_DEBUG
 		std::cerr << "getMissingNodes: root is full below" << std::endl;
 #endif
 		return;
@@ -23,7 +25,7 @@ void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint2
 		SHAMapInnerNode::pointer node=stack.top();
 		stack.pop();
 
-#ifdef DEBUG
+#ifdef GMN_DEBUG
 		std::cerr << "gMN: popped " << node->getString() << std::endl;
 #endif
 
@@ -31,35 +33,29 @@ void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint2
 		{
 			if(!node->isEmptyBranch(i))
 			{
-#ifdef DEBUG
+#ifdef GNMN_DEBUG
 				std::cerr << "gMN: " << node->getString() << " has non-empty branch " << i << std::endl;
 #endif
-				try
+				bool missing=false;
-				{
+				SHAMapNode childNID=node->getChildNodeID(i);
-					if(node->isChildLeaf())
+				if(node->isChildLeaf())
-					{ // do we have this leaf node?
+				{ // do we have this leaf node?
-						SHAMapLeafNode::pointer leaf=getLeaf(node->getChildNodeID(i), node->getChildHash(i), false);
+					if(!getLeaf(childNID, node->getChildHash(i), false)) missing=true;
-						assert(leaf);
-					}
-					else
-					{
-						SHAMapInnerNode::pointer desc=getInner(node->getChildNodeID(i), node->getChildHash(i), false);
-						assert(desc);
-						if(!desc->isFullBelow())
-							stack.push(desc);
-					}
 				}
-				catch(const SHAMapException &x)
+				else
 				{
-					if(x!=SHAMapException(MissingNode)) throw(x);
+					SHAMapInnerNode::pointer desc=getInner(childNID, node->getChildHash(i), false);
-					if(max-->0)
+					if(!desc)
-					{
+						missing=true;
-#ifdef DEBUG
+					else if(!desc->isFullBelow())
-						std::cerr << "gMN: need leaf " << node->getChildNodeID(i).getString() << std::endl;
+						stack.push(desc);
+				}
+				if(missing && max-->0)
+				{
+#ifdef GMN_DEBUG
+					std::cerr << "gMN: need " << node->getChildNodeID(i).getString() << std::endl;
 #endif
-						nodeIDs.push_back(node->getChildNodeID(i));
+					nodeIDs.push_back(childNID);
-						hashes.push_back(node->getChildHash(i));
-					}
 				}
 			}
 		}
@@ -253,7 +249,32 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vector<unsigned cha
 		mLeafByID[node]=leaf;
 		if(mDirtyLeafNodes) (*mDirtyLeafNodes)[node]=leaf;
 
-		// WRITEME: See if our parent(s) are full below
+		// FIXME: This should check all sources
+		SHAMapInnerNode::pointer pNode=checkCacheNode(node.getParentNodeID());
+		if(!pNode)
+		{
+			assert(false);
+			return false;
+		}
+		
+		for(int i=0; i<32; i++)
+			if(!checkCacheLeaf(pNode->getChildNodeID(i)))
+				return true;
+		pNode->setFullBelow();
+
+		while(!pNode->isRoot())
+		{
+			pNode=checkCacheNode(pNode->getParentNodeID());
+			if(!pNode)
+			{
+				assert(false);
+				return false;
+			}
+			for(int i=0; i<32; i++)
+				if(!checkCacheNode(pNode->getChildNodeID(i)))
+					return true;
+			pNode->setFullBelow();
+		}
 
 		return true;
 	}
@@ -270,7 +291,7 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vector<unsigned cha
 	}
 	mInnerNodeByID[node]=newNode;
 	if(mDirtyInnerNodes) (*mDirtyInnerNodes)[node]=newNode;
-#ifdef DEBUG
+#ifdef ST_DEBUG
 	std::cerr << "Hooked: " << node.getString() << std::endl;
 #endif
 	return true;
@@ -302,7 +323,7 @@ bool SHAMap::deepCompare(SHAMap& other)
 			return false;
 		}
 
-#ifdef DEBUG
+#ifdef DC_DEBUG
 		std::cerr << "Comparing inner nodes " << node->getString() << std::endl;
 #endif
 
@@ -353,10 +374,14 @@ bool SHAMap::deepCompare(SHAMap& other)
 
 bool SHAMap::syncTest()
 {
+	unsigned int seed;
+	RAND_pseudo_bytes(reinterpret_cast<unsigned char *>(&seed), sizeof(seed));
+	srand(seed);
+
 	SHAMap source, destination;
 
 	// add random data to the source map
-	int items=3; // 10+rand()%400;
+	int items=10+rand()%4000;
 	for(int i=0; i<items; i++)
 	{
 		Serializer s;
@@ -365,7 +390,7 @@ bool SHAMap::syncTest()
 			s.add32(rand());
 		uint256 id=s.getSHA512Half();
 		source.addItem(SHAMapItem(id, s.peekData()));
-#ifdef DEBUG
+#ifdef ST_DEBUG
 		std::cerr << "Item: " << id.GetHex() << std::endl;
 #endif
 	}
@@ -418,7 +443,7 @@ bool SHAMap::syncTest()
 		hashes.clear();
 
 		// get the list of nodes we know we need
-		destination.getMissingNodes(nodeIDs, hashes, 128);
+		destination.getMissingNodes(nodeIDs, hashes, 512);
 		if(!nodeIDs.size()) break;
 
 #ifdef DEBUG

Serializer.cpp

@@ -152,10 +152,10 @@ uint256 Serializer::getSHA512Half(int size) const
 
 uint256 Serializer::getSHA512Half(const std::vector<unsigned char>& data, int size)
 {
-	char buf[64];
+	uint256 j[2];
 	if((size<0)||(size>data.size())) size=data.size();
-	SHA512(&(data.front()), size, (unsigned char *) buf);
+	SHA512(&(data.front()), size, (unsigned char *) j);
-	return * reinterpret_cast<uint256*>(&buf[0]);
+	return j[0];
 }
 
 bool Serializer::checkSignature(int pubkeyOffset, int signatureOffset) const

uint256.h

@@ -290,7 +290,10 @@ public:
 		return (!(a == b));
 	}
 
-
+	unsigned int PeekAt(int j) const
+	{
+		return pn[j];
+	}
 
 	std::string GetHex() const
 	{