Build enough information into the leaf nodes so that they can be parsed

without knowledge of the objects they contain. This will allow new
transaction or account formats to be added (possibly with larger data sizes)
without breaking the ability to parse the hash trees. It also simplifies
the operation-specific tree code (since it doesn't have to parse the raw
leaf data).

BinaryFormats.txt

@@ -53,14 +53,9 @@ Fields:
 The transaction ID is the first 256-bits of the SHA512 hash of the 145 byte
 signed transaction.
 
-3) Transaction (ledger format)
-
-Fields:
-1) Transaction in signed format
-2) 8-byte fees held, unsigned BE integer
 
 
-5) Ledger (signed format)
+3) Ledger (signed format)
 
 Fields:
 1) 4-byte ledger index, unsigned BE integer
@@ -75,7 +70,7 @@ Fields:
  Proposed: Prefix (0x4C475000) of 120 byte fields 1-8
 
 
-6) Account status (ledger format)
+4) Account status (ledger format)
 
 Fields:
 1) 20-byte Account ID
@@ -84,7 +79,7 @@ Fields:
 
 
 
-7) Non-Leaf Tree Node
+5) Non-Leaf Tree Node
 
 Contains 32 hashes, each 20-bytes. They correspond to hashes of the nodes
 for the 32 possible values of the *first* 5 bits of the *next* byte of the
@@ -92,7 +87,7 @@ hash. By convention, an empty node has a hash of zero.
 
 
 
-8) Leaf Node
+6) Leaf Node
 
 Contains every item in this node, sorted in order of increasing tags
 (Elements that start with a zero byte come first.) In practice, this
@@ -104,8 +99,18 @@ By convention, an empty leaf node has a hash of all zero bytes.
 For a transaction, the tag is the transaction ID (hash of the transaction
 data). For an account state, the tag is the 20-byte account ID.
 
+Transaction Leaf Node Fields:
+1) 32-byte Transaction ID (LE)
+2) 2-byte Length, 153 (145+8)
+3) 145-byte Transaction in signed format
+4) 8-byte fees held, unsigned BE integer
 
-8) Contact block
+Account State Leaf Node Fields:
+1) 32-byte Account ID (20-byte ID zero-extended, in LE format)
+2) 2-byte length (32)
+3) 32-byte account status in ledger format
+
+7) Contact block
 
 These are used to pass node contact information around the network and are
 signed so nodes can prove they are operational.

SHAMap.cpp

@@ -88,7 +88,7 @@ SHAMapLeafNode::pointer SHAMap::walkToLeaf(const uint256& id, bool create, bool
 	return returnLeaf(ln, modify);
 }
 
-SHAMapLeafNode::pointer SHAMap::getLeaf(const SHAMapNode &id, const uint256& hash, bool modify)
+SHAMapLeafNode::pointer SHAMap::getLeaf(const SHAMapNode& id, const uint256& hash, bool modify)
 { // retrieve a leaf whose node hash is known
 	assert(!!hash);
 	if(!id.isLeaf()) return SHAMapLeafNode::pointer();
@@ -96,27 +96,22 @@ SHAMapLeafNode::pointer SHAMap::getLeaf(const SHAMapNode &id, const uint256& has
 	SHAMapLeafNode::pointer leaf=mLeafByID[id];			// is the leaf in memory
 	if(leaf) return returnLeaf(leaf, modify);
 
-	std::vector<SHAMapItem::pointer> leafData;			// is it in backing store
-	if(!fetchLeafNode(hash, id, leafData))
- 		throw SHAMapException(MissingNode);
-
-	leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(id, mSeq));
-	BOOST_FOREACH(SHAMapItem::pointer& item, leafData)
-		leaf->addUpdateItem(item);
-	leaf->updateHash();
+	std::vector<unsigned char> leafData;
+	if(!fetchLeafNode(hash, id, leafData)) throw SHAMapException(MissingNode);
+	leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(id, leafData, mSeq));
 	if(leaf->getNodeHash()!=hash) throw SHAMapException(InvalidNode);
+
 	mLeafByID[id]=leaf;
 	return leaf;
 }
 
-SHAMapInnerNode::pointer SHAMap::getInner(const SHAMapNode &id, const uint256& hash, bool modify)
+SHAMapInnerNode::pointer SHAMap::getInner(const SHAMapNode& id, const uint256& hash, bool modify)
 { // retrieve an inner node whose node hash is known
 	SHAMapInnerNode::pointer node=mInnerNodeByID[id];
 	if(node) return returnNode(node, modify);
 	
 	std::vector<unsigned char> rawNode;
 	if(!fetchInnerNode(hash, id, rawNode)) throw SHAMapException(MissingNode);
-
 	node=SHAMapInnerNode::pointer(new SHAMapInnerNode(id, rawNode, mSeq));
 	if(node->getNodeHash()!=hash) throw SHAMapException(InvalidNode);
 
@@ -421,7 +416,7 @@ bool SHAMap::fetchInnerNode(const uint256&, const SHAMapNode&, std::vector<unsig
 	return false;
 }
 
-bool SHAMap::fetchLeafNode(const uint256&, const SHAMapNode&, std::vector<SHAMapItem::pointer>&)
+bool SHAMap::fetchLeafNode(const uint256&, const SHAMapNode&, std::vector<unsigned char>&)
 {
 	return false;
 }

SHAMap.h

@@ -127,6 +127,9 @@ protected:
 public:
 	SHAMapLeafNode(const SHAMapNode& nodeID, uint32 seq);
 	SHAMapLeafNode(const SHAMapLeafNode& node, uint32 seq);
+	SHAMapLeafNode(const SHAMapNode& id, const std::vector<unsigned char>& contents, uint32 seq);
+
+	void addRaw(Serializer &);
 
 	virtual bool isPopulated(void) const { return true; }
 
@@ -173,6 +176,8 @@ public:
 	SHAMapInnerNode(const SHAMapInnerNode& node, uint32 seq);
 	SHAMapInnerNode(const SHAMapNode& id, const std::vector<unsigned char>& contents, uint32 seq);
 
+	void addRaw(Serializer&);
+
 	uint32 getSeq(void) const { return mSeq; }
 	void setSeq(uint32 s) { mSeq=s; }
 
@@ -209,10 +214,10 @@ private:
 	SHAMapInnerNode::pointer root;
 
 protected:
-	void dirtyUp(const uint256 &id);
+	void dirtyUp(const uint256& id);
 
 	SHAMapLeafNode::pointer createLeaf(const SHAMapInnerNode& lowestParent, const uint256& id);
-	SHAMapLeafNode::pointer checkCacheLeaf(const SHAMapNode &);
+	SHAMapLeafNode::pointer checkCacheLeaf(const SHAMapNode&);
 	SHAMapLeafNode::pointer walkToLeaf(const uint256& id, bool create, bool modify);
 
 	SHAMapLeafNode::pointer getLeaf(const SHAMapNode& id, const uint256& hash, bool modify);
@@ -234,12 +239,12 @@ public:
 	// inner node access functions
 	bool hasInnerNode(const SHAMapNode& id);
 	bool giveInnerNode(SHAMapInnerNode::pointer);
-	SHAMapInnerNode::pointer getInnerNode(const SHAMapNode &);
+	SHAMapInnerNode::pointer getInnerNode(const SHAMapNode&);
 
 	// leaf node access functions
 	bool hasLeafNode(const SHAMapNode& id);
 	bool giveLeafNode(SHAMapLeafNode::pointer);
-	SHAMapLeafNode::pointer getLeafNode(const SHAMapNode &);
+	SHAMapLeafNode::pointer getLeafNode(const SHAMapNode&);
 
 	// generic node functions
 	std::vector<unsigned char> getRawNode(const SHAMapNode& id);
@@ -279,7 +284,7 @@ public:
 
 	// overloads for backed maps
 	virtual bool fetchInnerNode(const uint256& hash, const SHAMapNode& id, std::vector<unsigned char>& rawNode);
-	virtual bool fetchLeafNode(const uint256& hash, const SHAMapNode& id, std::vector<SHAMapItem::pointer>& nodeData);
+	virtual bool fetchLeafNode(const uint256& hash, const SHAMapNode& id, std::vector<unsigned char>& rawNode);
 	virtual bool writeInnerNode(const uint256& hash, const SHAMapNode& id, const std::vector<unsigned char>& rawNode);
 	virtual bool writeLeafNode(const uint256& hash, const SHAMapNode& id, const std::vector<unsigned char>& rawNode);
 

SHAMapNodes.cpp

@@ -118,9 +118,38 @@ SHAMapLeafNode::SHAMapLeafNode(const SHAMapLeafNode& node, uint32 seq) : SHAMapN
 	assert(node.isLeaf());
 }
 
+SHAMapLeafNode::SHAMapLeafNode(const SHAMapNode& id, const std::vector<unsigned char>& rawLeaf, uint32 seq)
+	: SHAMapNode(id), mSeq(seq)
+{
+	Serializer s(rawLeaf);
+	int pos=0;
+	while(pos<s.getLength())
+	{
+		uint256 id=s.get256(pos);
+		pos+=32;
+		uint16 len;
+		if(!s.get16(len, pos)) throw SHAMapException(InvalidNode);
+		pos+=2;
+		if(!id || !len || ((pos+len)>s.getLength())) throw SHAMapException(InvalidNode);
+		addUpdateItem(SHAMapItem::pointer(new SHAMapItem(id, s.getRaw(pos, len))));
+		pos+=len;
+	}
+	updateHash();
+}
+
+void SHAMapLeafNode::addRaw(Serializer &s)
+{
+	BOOST_FOREACH(SHAMapItem::pointer& nodeItem, mItems)
+	{
+		s.add256(nodeItem->getTag());
+		s.add16(nodeItem->peekData().size());
+		s.addRaw(nodeItem->peekData());
+	}
+}
+
 bool SHAMapLeafNode::hasItem(const uint256& item) const
 {
-	BOOST_FOREACH(SHAMapItem::pointer nodeItem, mItems)
+	BOOST_FOREACH(const SHAMapItem::pointer& nodeItem, mItems)
 		if(nodeItem->getTag()==item) return true;
 	return false;
 }
@@ -254,6 +283,11 @@ SHAMapInnerNode::SHAMapInnerNode(const SHAMapInnerNode& node, uint32 seq) : SHAM
 	memcpy(mHashes, node.mHashes, sizeof(mHashes));
 }
 
+void SHAMapInnerNode::addRaw(Serializer &s)
+{
+	for(int i=0; i<32; i++)
+		s.add256(mHashes[i]);
+}
 bool SHAMapInnerNode::setChildHash(int m, const uint256 &hash)
 {
 	assert( (m>=0) && (m<32) );

Serializer.cpp

@@ -4,6 +4,17 @@
 #include <openssl/ripemd.h>
 #include <openssl/sha.h>
 
+int Serializer::add16(uint16 i)
+{
+	int ret=mData.size();
+	for(int j=0; j<sizeof(i); j++)
+	{
+		mData.push_back((unsigned char) (i&0xff));
+		i>>=8;
+	}
+	return ret;
+}
+
 int Serializer::add32(uint32 i)
 {
 	int ret=mData.size();
@@ -47,8 +58,21 @@ int Serializer::addRaw(const std::vector<unsigned char> &vector)
 	return ret;
 }
 
+bool Serializer::get16(uint16& o, int offset) const
+{
+	o=0;
+	if((offset+sizeof(o))>mData.size()) return false;
+	for(int i=0, o=0; i<sizeof(o); i++)
+	{
+		o<<=8;
+		o|=mData.at(offset++);
+	}
+	return true;
+}
+
 bool Serializer::get32(uint32& o, int offset) const
 {
+	o=0;
 	if((offset+sizeof(o))>mData.size()) return false;
 	for(int i=0, o=0; i<sizeof(o); i++)
 	{
@@ -60,6 +84,7 @@ bool Serializer::get32(uint32& o, int offset) const
 
 bool Serializer::get64(uint64& o, int offset) const
 {
+	o=0;
 	if((offset+sizeof(o))>mData.size()) return false;
 	for(int i=0, o=0; i<sizeof(o); i++)
 	{

Serializer.h

@@ -21,6 +21,7 @@ class Serializer
 	Serializer(const std::vector<unsigned char> &data) : mData(data) { ; }
 
 	// assemble functions
+	int add16(uint16);
 	int add32(uint32);				// ledger indexes, account sequence
 	int add64(uint64);				// timestamps, amounts
 	int add160(const uint160&);	// account names, hankos
@@ -28,6 +29,7 @@ class Serializer
 	int addRaw(const std::vector<unsigned char> &vector);
 
 	// disassemble functions
+	bool get16(uint16&, int offset) const;
 	bool get32(uint32&, int offset) const;
 	bool get64(uint64&, int offset) const;
 	bool get160(uint160&, int offset) const;