Fix the retrieve ledger hash problem for real. Make partial ledger operations work.

Set base code for how thin servers will operate on partial ledgers.
2025-12-06 17:27:55 +00:00 · 2012-06-26 02:01:13 -07:00
parent e4d8bda4a6
commit fa48a1fb09
4 changed files with 122 additions and 78 deletions
--- a/src/SHAMap.cpp
+++ b/src/SHAMap.cpp
@@ -59,7 +59,7 @@ SHAMap::pointer SHAMap::snapShot(bool isMutable)
 	return ret;
 }

-std::stack<SHAMapTreeNode::pointer> SHAMap::getStack(const uint256& id, bool include_nonmatching_leaf)
+std::stack<SHAMapTreeNode::pointer> SHAMap::getStack(const uint256& id, bool include_nonmatching_leaf, bool partialOk)
 {
 	// Walk the tree as far as possible to the specified identifier
 	// produce a stack of nodes along the way, with the terminal node at the top
@@ -73,13 +73,18 @@ std::stack<SHAMapTreeNode::pointer> SHAMap::getStack(const uint256& id, bool inc
 		assert(branch >= 0);

 		uint256 hash = node->getChildHash(branch);
-		if (hash.isZero()) return stack;
+		if (hash.isZero())
+			return stack;

-		node = getNode(node->getChildNodeID(branch), hash, false);
-		if (!node)
+		try
 		{
-			if (isSynching()) return stack;
-			throw std::runtime_error("missing node");
+			node = getNode(node->getChildNodeID(branch), hash, false);
+		}
+		catch (SHAMapMissingNode& mn)
+		{
+			if (partialOk)
+				return stack;
+			throw mn;
 		}
 	}

@@ -135,15 +140,19 @@ SHAMapTreeNode::pointer SHAMap::walkTo(const uint256& id, bool modify)
 	while (!inNode->isLeaf())
 	{
 		int branch = inNode->selectBranch(id);
-		if (inNode->isEmptyBranch(branch)) return inNode;
+		if (inNode->isEmptyBranch(branch))
+			return inNode;
 		uint256 childHash = inNode->getChildHash(branch);

 		SHAMapTreeNode::pointer nextNode = getNode(inNode->getChildNodeID(branch), childHash, false);
-		if (!nextNode) throw std::runtime_error("missing node");
+		if (!nextNode)
+			throw SHAMapMissingNode(inNode->getChildNodeID(branch), childHash);
 		inNode = nextNode;
 	}
-	if (inNode->getTag() != id) return SHAMapTreeNode::pointer();
-	if (modify) returnNode(inNode, true);
+	if (inNode->getTag() != id)
+		return SHAMapTreeNode::pointer();
+	if (modify)
+		returnNode(inNode, true);
 	return inNode;
 }

@@ -156,7 +165,8 @@ SHAMapTreeNode* SHAMap::walkToPointer(const uint256& id)
 		const uint256& nextHash = inNode->getChildHash(branch);
 		if (!nextHash) return NULL;
 		inNode = getNodePointer(inNode->getChildNodeID(branch), nextHash);
-		if (!inNode) throw std::runtime_error("missing node");
+		if (!inNode)
+			throw SHAMapMissingNode(inNode->getChildNodeID(branch), nextHash);
 	}
 	return (inNode->getTag() == id) ? inNode : NULL;
 }
@@ -166,27 +176,22 @@ SHAMapTreeNode::pointer SHAMap::getNode(const SHAMapNode& id, const uint256& has
 	SHAMapTreeNode::pointer node = checkCacheNode(id);
 	if (node)
 	{
+#ifdef DEBUG
 		if (node->getNodeHash() != hash)
 		{
-#ifdef DEBUG
 			std::cerr << "Attempt to get node, hash not in tree" << std::endl;
 			std::cerr << "ID: " << id.getString() << std::endl;
 			std::cerr << "TgtHash " << hash.GetHex() << std::endl;
 			std::cerr << "NodHash " << node->getNodeHash().GetHex() << std::endl;
 			dump();
-#endif
 			throw std::runtime_error("invalid node");
 		}
+#endif
 		returnNode(node, modify);
 		return node;
 	}

-	node = fetchNode(id, hash);
-	if (!node) return node;
-
-	if (node->getNodeHash() != hash)
-		throw std::runtime_error("invalid node hash");
-
+	node = fetchNodeExternal(id, hash);
 	if (!mTNByID.insert(std::make_pair(id, node)).second)
 		assert(false);
 	return node;
@@ -198,12 +203,7 @@ SHAMapTreeNode* SHAMap::getNodePointer(const SHAMapNode& id, const uint256& hash
 	if (it != mTNByID.end())
 		return &*it->second;

-	SHAMapTreeNode::pointer node = fetchNode(id, hash);
-	if (!node) return NULL;
-
-	if (node->getNodeHash() != hash)
-		throw std::runtime_error("invalid node fetched");
-
+	SHAMapTreeNode::pointer node = fetchNodeExternal(id, hash);
 	if (!mTNByID.insert(std::make_pair(id, node)).second)
 		assert(false);
 	return &*node;
@@ -311,26 +311,26 @@ SHAMapItem::pointer SHAMap::onlyBelow(SHAMapTreeNode* node)

 void SHAMap::eraseChildren(SHAMapTreeNode::pointer node)
 { // this node has only one item below it, erase its children
-	bool erase=false;
-	while(node->isInner())
+	bool erase = false;
+	while (node->isInner())
 	{
-		for(int i=0; i<16; i++)
-			if(!node->isEmptyBranch(i))
+		for (int i = 0; i < 16; ++i)
+			if (!node->isEmptyBranch(i))
 			{
 				SHAMapTreeNode::pointer nextNode = getNode(node->getChildNodeID(i), node->getChildHash(i), false);
-				if(erase)
+				if (erase)
 				{
 					returnNode(node, true);
-					if(mTNByID.erase(*node))
-					assert(false);
+					if (mTNByID.erase(*node))
+						assert(false);
 				}
-				erase=true;
-				node=nextNode;
+				erase = true;
+				node = nextNode;
 				break;
 			}
 	}
 	returnNode(node, true);
-	if(mTNByID.erase(*node) == 0)
+	if (mTNByID.erase(*node) == 0)
 		assert(false);
 	return;
 }
@@ -351,7 +351,7 @@ SHAMapItem::pointer SHAMap::peekNextItem(const uint256& id)
 { // Get a pointer to the next item in the tree after a given item - item must be in tree
 	boost::recursive_mutex::scoped_lock sl(mLock);

-	std::stack<SHAMapTreeNode::pointer> stack = getStack(id, true);
+	std::stack<SHAMapTreeNode::pointer> stack = getStack(id, true, false);
 	while(!stack.empty())
 	{
 		SHAMapTreeNode::pointer node = stack.top();
@@ -366,7 +366,6 @@ SHAMapItem::pointer SHAMap::peekNextItem(const uint256& id)
 			if(!node->isEmptyBranch(i))
 			{
 				node = getNode(node->getChildNodeID(i), node->getChildHash(i), false);
-				if (!node) throw std::runtime_error("missing node");
 				SHAMapItem::pointer item = firstBelow(&*node);
 				if (!item) throw std::runtime_error("missing node");
 				return item;
@@ -380,7 +379,7 @@ SHAMapItem::pointer SHAMap::peekPrevItem(const uint256& id)
 { // Get a pointer to the previous item in the tree after a given item - item must be in tree
 	boost::recursive_mutex::scoped_lock sl(mLock);

-	std::stack<SHAMapTreeNode::pointer> stack = getStack(id, true);
+	std::stack<SHAMapTreeNode::pointer> stack = getStack(id, true, false);
 	while (!stack.empty())
 	{
 		SHAMapTreeNode::pointer node = stack.top();
@@ -395,7 +394,6 @@ SHAMapItem::pointer SHAMap::peekPrevItem(const uint256& id)
 				if(!node->isEmptyBranch(i))
 				{
 					node = getNode(node->getChildNodeID(i), node->getChildHash(i), false);
-					if(!node) throw std::runtime_error("missing node");
 					SHAMapItem::pointer item = firstBelow(&*node);
 					if (!item) throw std::runtime_error("missing node");
 					return item;
@@ -426,8 +424,9 @@ bool SHAMap::delItem(const uint256& id)
 	boost::recursive_mutex::scoped_lock sl(mLock);
 	assert(mState != Immutable);

-	std::stack<SHAMapTreeNode::pointer> stack=getStack(id, true);
-	if(stack.empty()) throw std::runtime_error("missing node");
+	std::stack<SHAMapTreeNode::pointer> stack = getStack(id, true, false);
+	if(stack.empty())
+		throw std::runtime_error("missing node");

 	SHAMapTreeNode::pointer leaf=stack.top();
 	stack.pop();
@@ -501,8 +500,9 @@ bool SHAMap::addGiveItem(SHAMapItem::pointer item, bool isTransaction)
 	boost::recursive_mutex::scoped_lock sl(mLock);
 	assert(mState != Immutable);

-	std::stack<SHAMapTreeNode::pointer> stack = getStack(tag, true);
-	if (stack.empty()) throw std::runtime_error("missing node");
+	std::stack<SHAMapTreeNode::pointer> stack = getStack(tag, true, false);
+	if (stack.empty())
+		throw std::runtime_error("missing node");

 	SHAMapTreeNode::pointer node = stack.top();
 	stack.pop();
@@ -592,7 +592,7 @@ bool SHAMap::updateGiveItem(SHAMapItem::pointer item, bool isTransaction)
 	boost::recursive_mutex::scoped_lock sl(mLock);
 	assert(mState != Immutable);

-	std::stack<SHAMapTreeNode::pointer> stack = getStack(tag, true);
+	std::stack<SHAMapTreeNode::pointer> stack = getStack(tag, true, false);
 	if (stack.empty()) throw std::runtime_error("missing node");

 	SHAMapTreeNode::pointer node = stack.top();
@@ -620,15 +620,21 @@ void SHAMapItem::dump()
 	std::cerr << "SHAMapItem(" << mTag.GetHex() << ") " << mData.size() << "bytes" << std::endl;
 }

-SHAMapTreeNode::pointer SHAMap::fetchNode(const SHAMapNode& id, const uint256& hash)
+SHAMapTreeNode::pointer SHAMap::fetchNodeExternal(const SHAMapNode& id, const uint256& hash)
 {
-	if (!theApp->running()) return SHAMapTreeNode::pointer();
+	if (!theApp->running())
+		throw SHAMapMissingNode(id, hash);

 	HashedObject::pointer obj(theApp->getHashedObjectStore().retrieve(hash));
-	if(!obj) return SHAMapTreeNode::pointer();
+	if(!obj)
+		throw SHAMapMissingNode(id, hash);
 	assert(Serializer::getSHA512Half(obj->getData()) == hash);

-	return boost::make_shared<SHAMapTreeNode>(id, obj->getData(), mSeq, STN_ARF_PREFIXED);
+	SHAMapTreeNode::pointer ret =  boost::make_shared<SHAMapTreeNode>(id, obj->getData(), mSeq, STN_ARF_PREFIXED);
+#ifdef DEBUG
+	assert((ret->getNodeHash() == hash) && (id == *ret));
+#endif
+	return ret;
 }

 void SHAMap::armDirty()
--- a/src/SHAMap.h
+++ b/src/SHAMap.h
@@ -229,6 +229,22 @@ public:
 	{ return false; }
 };

+class SHAMapMissingNode : public std::runtime_error
+{
+protected:
+	SHAMapNode mNodeID;
+	uint256 mNodeHash;
+
+public:
+	SHAMapMissingNode(const SHAMapNode& nodeID, const uint256& nodeHash) :
+		std::runtime_error(nodeID.getString()), mNodeID(nodeID), mNodeHash(nodeHash)
+	{ ; }
+	virtual ~SHAMapMissingNode() throw()
+	{ ; }
+	const SHAMapNode& getNodeID() const	{ return mNodeID; }
+	const uint256& getNodeHash() const	{ return mNodeHash; }
+};
+
 class SHAMap
 {
 public:
@@ -249,7 +265,7 @@ private:
 protected:

 	void dirtyUp(std::stack<SHAMapTreeNode::pointer>& stack, const uint256& target, uint256 prevHash);
-	std::stack<SHAMapTreeNode::pointer> getStack(const uint256& id, bool include_nonmatching_leaf);
+	std::stack<SHAMapTreeNode::pointer> getStack(const uint256& id, bool include_nonmatching_leaf, bool partialOk);
 	SHAMapTreeNode::pointer walkTo(const uint256& id, bool modify);
 	SHAMapTreeNode* walkToPointer(const uint256& id);
 	SHAMapTreeNode::pointer checkCacheNode(const SHAMapNode&);
@@ -333,7 +349,7 @@ public:
 	uint32 getSeq()				{ return mSeq; }

 	// overloads for backed maps
-	boost::shared_ptr<SHAMapTreeNode> fetchNode(const SHAMapNode& id, const uint256& hash);
+	boost::shared_ptr<SHAMapTreeNode> fetchNodeExternal(const SHAMapNode& id, const uint256& hash);

 	bool operator==(const SHAMap& s) { return getHash() == s.getHash(); }

--- a/src/SHAMapSync.cpp
+++ b/src/SHAMapSync.cpp
@@ -46,32 +46,39 @@ void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint2
 			{
 				SHAMapNode childID = node->getChildNodeID(branch);
 				const uint256& childHash = node->getChildHash(branch);
-				SHAMapTreeNode::pointer d = getNode(childID, childHash, false);
-				if ((!d) && (filter != NULL))
+				SHAMapTreeNode::pointer d;
+				try
 				{
-					std::vector<unsigned char> nodeData;
-					if (filter->haveNode(childID, childHash, nodeData))
+					d = getNode(childID, childHash, false);
+				}
+				catch (SHAMapMissingNode& mn)
+				{ // node is not in the map
+					if (filter != NULL)
 					{
-						d = boost::make_shared<SHAMapTreeNode>(childID, nodeData, mSeq, STN_ARF_WIRE);
-						if (childHash != d->getNodeHash())
+						std::vector<unsigned char> nodeData;
+						if (filter->haveNode(childID, childHash, nodeData))
 						{
-							Log(lsERROR) << "Wrong hash from cached object";
-							d = SHAMapTreeNode::pointer();
-						}
-						else
-						{
-							Log(lsTRACE) << "Got sync node from cache: " << d->getString();
-							mTNByID[*d] = d;
+							d = boost::make_shared<SHAMapTreeNode>(childID, nodeData, mSeq, STN_ARF_WIRE);
+							if (childHash != d->getNodeHash())
+							{
+								Log(lsERROR) << "Wrong hash from cached object";
+								d = SHAMapTreeNode::pointer();
+							}
+							else
+							{
+								Log(lsTRACE) << "Got sync node from cache: " << d->getString();
+								mTNByID[*d] = d;
+							}
 						}
 					}
 				}
 				if (!d)
-				{
+				{ // we need this node
 					nodeIDs.push_back(node->getChildNodeID(branch));
 					if (--max <= 0)
 						return;
 				}
-				else if (d->isInner() && !d->isFullBelow())
+				else if (d->isInner() && !d->isFullBelow()) // we might need children of this node
 					stack.push(d);
 			}
 		}
@@ -159,8 +166,10 @@ bool SHAMap::addRootNode(const uint256& hash, const std::vector<unsigned char>&
 	}

 	SHAMapTreeNode::pointer node = boost::make_shared<SHAMapTreeNode>(SHAMapNode(), rootNode, 0, STN_ARF_WIRE);
-	if (!node) return false;
-	if (node->getNodeHash() != hash) return false;
+	if (!node)
+		return false;
+	if (node->getNodeHash() != hash)
+		return false;

 	returnNode(root, true);
 	root = node;
@@ -178,14 +187,17 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vector<unsigned cha
 	SHAMapSyncFilter* filter)
 { // return value: true=okay, false=error
 	assert(!node.isRoot());
-	if (!isSynching()) return false;
+	if (!isSynching())
+		return false;

 	boost::recursive_mutex::scoped_lock sl(mLock);

-	if (checkCacheNode(node)) return true;
+	if (checkCacheNode(node))
+		return true;

-	std::stack<SHAMapTreeNode::pointer> stack = getStack(node.getNodeID(), true);
-	if (stack.empty()) return false;
+	std::stack<SHAMapTreeNode::pointer> stack = getStack(node.getNodeID(), true, true);
+	if (stack.empty())
+		return false;

 	SHAMapTreeNode::pointer iNode = stack.top();
 	if (!iNode)
@@ -221,7 +233,8 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vector<unsigned cha
 	if (hash != newNode->getNodeHash()) // these aren't the droids we're looking for
 		return false;

-	if (filter) filter->gotNode(node, hash, rawNode, newNode->isLeaf());
+	if (filter)
+		filter->gotNode(node, hash, rawNode, newNode->isLeaf());

 	mTNByID[*newNode] = newNode;
 	if (!newNode->isLeaf())
@@ -236,9 +249,16 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vector<unsigned cha
 		for (int i = 0; i < 16; ++i)
 			if (!iNode->isEmptyBranch(i))
 			{
-				SHAMapTreeNode::pointer nextNode = getNode(iNode->getChildNodeID(i), iNode->getChildHash(i), false);
-				if (!nextNode) return true;
-				if (nextNode->isInner() && !nextNode->isFullBelow()) return true;
+				try
+				{
+					SHAMapTreeNode::pointer nextNode = getNode(iNode->getChildNodeID(i), iNode->getChildHash(i), false);
+					if (nextNode->isInner() && !nextNode->isFullBelow())
+						return true;
+				}
+				catch (SHAMapMissingNode)
+				{
+					return true;
+				}
 			}
 		iNode->setFullBelow();
 	} while (!stack.empty());
@@ -359,7 +379,7 @@ static bool confuseMap(SHAMap &map, int count)
 std::list<std::vector<unsigned char> > SHAMap::getTrustedPath(const uint256& index)
 {
 	boost::recursive_mutex::scoped_lock sl(mLock);
-	std::stack<SHAMapTreeNode::pointer> stack = SHAMap::getStack(index, false);
+	std::stack<SHAMapTreeNode::pointer> stack = SHAMap::getStack(index, false, false);

 	if (stack.empty() || !stack.top()->isLeaf())
 		throw std::runtime_error("requested leaf not present");
@@ -449,11 +469,13 @@ BOOST_AUTO_TEST_CASE( SHAMapSync_test )
 		
 		// get as many nodes as possible based on this information
 		for (nodeIDIterator = nodeIDs.begin(); nodeIDIterator != nodeIDs.end(); ++nodeIDIterator)
+		{
 			if (!source.getNodeFat(*nodeIDIterator, gotNodeIDs, gotNodes, (rand() % 2) == 0))
 			{
 				Log(lsFATAL) << "GetNodeFat fails";
 				BOOST_FAIL("GetNodeFat");
 			}
+		}
 		assert(gotNodeIDs.size() == gotNodes.size());
 		nodeIDs.clear();
 		hashes.clear();
--- a/src/SHAMapSync.h
+++ b/src/SHAMapSync.h
@@ -40,7 +40,7 @@ public:
 		theApp->getHashedObjectStore().store(ACCOUNT_NODE, mLedgerSeq, nodeData, nodeHash);
 	}
 	virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector<unsigned char>& nodeData)
-	{ // fetchNode already tried
+	{ // fetchNodeExternal already tried
 		return false;
 	}
 };
@@ -61,7 +61,7 @@ public:
 		theApp->getHashedObjectStore().store(isLeaf ? TRANSACTION : TRANSACTION_NODE, mLedgerSeq, nodeData, nodeHash);
 	}
 	virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector<unsigned char>& nodeData)
-	{ // fetchNode already tried
+	{ // fetchNodeExternal already tried
 		return false;
 	}
 };