From ebb9a9c25590e8c6083c6654973d74f7667a06fd Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 28 Nov 2011 13:59:34 -0800 Subject: [PATCH] Build enough information into the leaf nodes so that they can be parsed without knowledge of the objects they contain. This will allow new transaction or account formats to be added (possibly with larger data sizes) without breaking the ability to parse the hash trees. It also simplifies the operation-specific tree code (since it doesn't have to parse the raw leaf data). --- BinaryFormats.txt | 25 +++++++++++++++---------- SHAMap.cpp | 19 +++++++------------ SHAMap.h | 15 ++++++++++----- SHAMapNodes.cpp | 36 +++++++++++++++++++++++++++++++++++- Serializer.cpp | 25 +++++++++++++++++++++++++ Serializer.h | 2 ++ 6 files changed, 94 insertions(+), 28 deletions(-) diff --git a/BinaryFormats.txt b/BinaryFormats.txt index 7848d9cac..e7d3905d2 100644 --- a/BinaryFormats.txt +++ b/BinaryFormats.txt @@ -53,14 +53,9 @@ Fields: The transaction ID is the first 256-bits of the SHA512 hash of the 145 byte signed transaction. -3) Transaction (ledger format) - -Fields: -1) Transaction in signed format -2) 8-byte fees held, unsigned BE integer -5) Ledger (signed format) +3) Ledger (signed format) Fields: 1) 4-byte ledger index, unsigned BE integer @@ -75,7 +70,7 @@ Fields: Proposed: Prefix (0x4C475000) of 120 byte fields 1-8 -6) Account status (ledger format) +4) Account status (ledger format) Fields: 1) 20-byte Account ID @@ -84,7 +79,7 @@ Fields: -7) Non-Leaf Tree Node +5) Non-Leaf Tree Node Contains 32 hashes, each 20-bytes. They correspond to hashes of the nodes for the 32 possible values of the *first* 5 bits of the *next* byte of the @@ -92,7 +87,7 @@ hash. By convention, an empty node has a hash of zero. -8) Leaf Node +6) Leaf Node Contains every item in this node, sorted in order of increasing tags (Elements that start with a zero byte come first.) In practice, this @@ -104,8 +99,18 @@ By convention, an empty leaf node has a hash of all zero bytes. For a transaction, the tag is the transaction ID (hash of the transaction data). For an account state, the tag is the 20-byte account ID. +Transaction Leaf Node Fields: +1) 32-byte Transaction ID (LE) +2) 2-byte Length, 153 (145+8) +3) 145-byte Transaction in signed format +4) 8-byte fees held, unsigned BE integer -8) Contact block +Account State Leaf Node Fields: +1) 32-byte Account ID (20-byte ID zero-extended, in LE format) +2) 2-byte length (32) +3) 32-byte account status in ledger format + +7) Contact block These are used to pass node contact information around the network and are signed so nodes can prove they are operational. diff --git a/SHAMap.cpp b/SHAMap.cpp index 7b37a04c3..d7c471243 100644 --- a/SHAMap.cpp +++ b/SHAMap.cpp @@ -88,7 +88,7 @@ SHAMapLeafNode::pointer SHAMap::walkToLeaf(const uint256& id, bool create, bool return returnLeaf(ln, modify); } -SHAMapLeafNode::pointer SHAMap::getLeaf(const SHAMapNode &id, const uint256& hash, bool modify) +SHAMapLeafNode::pointer SHAMap::getLeaf(const SHAMapNode& id, const uint256& hash, bool modify) { // retrieve a leaf whose node hash is known assert(!!hash); if(!id.isLeaf()) return SHAMapLeafNode::pointer(); @@ -96,27 +96,22 @@ SHAMapLeafNode::pointer SHAMap::getLeaf(const SHAMapNode &id, const uint256& has SHAMapLeafNode::pointer leaf=mLeafByID[id]; // is the leaf in memory if(leaf) return returnLeaf(leaf, modify); - std::vector leafData; // is it in backing store - if(!fetchLeafNode(hash, id, leafData)) - throw SHAMapException(MissingNode); - - leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(id, mSeq)); - BOOST_FOREACH(SHAMapItem::pointer& item, leafData) - leaf->addUpdateItem(item); - leaf->updateHash(); + std::vector leafData; + if(!fetchLeafNode(hash, id, leafData)) throw SHAMapException(MissingNode); + leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(id, leafData, mSeq)); if(leaf->getNodeHash()!=hash) throw SHAMapException(InvalidNode); + mLeafByID[id]=leaf; return leaf; } -SHAMapInnerNode::pointer SHAMap::getInner(const SHAMapNode &id, const uint256& hash, bool modify) +SHAMapInnerNode::pointer SHAMap::getInner(const SHAMapNode& id, const uint256& hash, bool modify) { // retrieve an inner node whose node hash is known SHAMapInnerNode::pointer node=mInnerNodeByID[id]; if(node) return returnNode(node, modify); std::vector rawNode; if(!fetchInnerNode(hash, id, rawNode)) throw SHAMapException(MissingNode); - node=SHAMapInnerNode::pointer(new SHAMapInnerNode(id, rawNode, mSeq)); if(node->getNodeHash()!=hash) throw SHAMapException(InvalidNode); @@ -421,7 +416,7 @@ bool SHAMap::fetchInnerNode(const uint256&, const SHAMapNode&, std::vector&) +bool SHAMap::fetchLeafNode(const uint256&, const SHAMapNode&, std::vector&) { return false; } diff --git a/SHAMap.h b/SHAMap.h index 17b0088c6..e5e5ad788 100644 --- a/SHAMap.h +++ b/SHAMap.h @@ -127,6 +127,9 @@ protected: public: SHAMapLeafNode(const SHAMapNode& nodeID, uint32 seq); SHAMapLeafNode(const SHAMapLeafNode& node, uint32 seq); + SHAMapLeafNode(const SHAMapNode& id, const std::vector& contents, uint32 seq); + + void addRaw(Serializer &); virtual bool isPopulated(void) const { return true; } @@ -173,6 +176,8 @@ public: SHAMapInnerNode(const SHAMapInnerNode& node, uint32 seq); SHAMapInnerNode(const SHAMapNode& id, const std::vector& contents, uint32 seq); + void addRaw(Serializer&); + uint32 getSeq(void) const { return mSeq; } void setSeq(uint32 s) { mSeq=s; } @@ -209,10 +214,10 @@ private: SHAMapInnerNode::pointer root; protected: - void dirtyUp(const uint256 &id); + void dirtyUp(const uint256& id); SHAMapLeafNode::pointer createLeaf(const SHAMapInnerNode& lowestParent, const uint256& id); - SHAMapLeafNode::pointer checkCacheLeaf(const SHAMapNode &); + SHAMapLeafNode::pointer checkCacheLeaf(const SHAMapNode&); SHAMapLeafNode::pointer walkToLeaf(const uint256& id, bool create, bool modify); SHAMapLeafNode::pointer getLeaf(const SHAMapNode& id, const uint256& hash, bool modify); @@ -234,12 +239,12 @@ public: // inner node access functions bool hasInnerNode(const SHAMapNode& id); bool giveInnerNode(SHAMapInnerNode::pointer); - SHAMapInnerNode::pointer getInnerNode(const SHAMapNode &); + SHAMapInnerNode::pointer getInnerNode(const SHAMapNode&); // leaf node access functions bool hasLeafNode(const SHAMapNode& id); bool giveLeafNode(SHAMapLeafNode::pointer); - SHAMapLeafNode::pointer getLeafNode(const SHAMapNode &); + SHAMapLeafNode::pointer getLeafNode(const SHAMapNode&); // generic node functions std::vector getRawNode(const SHAMapNode& id); @@ -279,7 +284,7 @@ public: // overloads for backed maps virtual bool fetchInnerNode(const uint256& hash, const SHAMapNode& id, std::vector& rawNode); - virtual bool fetchLeafNode(const uint256& hash, const SHAMapNode& id, std::vector& nodeData); + virtual bool fetchLeafNode(const uint256& hash, const SHAMapNode& id, std::vector& rawNode); virtual bool writeInnerNode(const uint256& hash, const SHAMapNode& id, const std::vector& rawNode); virtual bool writeLeafNode(const uint256& hash, const SHAMapNode& id, const std::vector& rawNode); diff --git a/SHAMapNodes.cpp b/SHAMapNodes.cpp index 5f0806338..7808fa004 100644 --- a/SHAMapNodes.cpp +++ b/SHAMapNodes.cpp @@ -118,9 +118,38 @@ SHAMapLeafNode::SHAMapLeafNode(const SHAMapLeafNode& node, uint32 seq) : SHAMapN assert(node.isLeaf()); } +SHAMapLeafNode::SHAMapLeafNode(const SHAMapNode& id, const std::vector& rawLeaf, uint32 seq) + : SHAMapNode(id), mSeq(seq) +{ + Serializer s(rawLeaf); + int pos=0; + while(poss.getLength())) throw SHAMapException(InvalidNode); + addUpdateItem(SHAMapItem::pointer(new SHAMapItem(id, s.getRaw(pos, len)))); + pos+=len; + } + updateHash(); +} + +void SHAMapLeafNode::addRaw(Serializer &s) +{ + BOOST_FOREACH(SHAMapItem::pointer& nodeItem, mItems) + { + s.add256(nodeItem->getTag()); + s.add16(nodeItem->peekData().size()); + s.addRaw(nodeItem->peekData()); + } +} + bool SHAMapLeafNode::hasItem(const uint256& item) const { - BOOST_FOREACH(SHAMapItem::pointer nodeItem, mItems) + BOOST_FOREACH(const SHAMapItem::pointer& nodeItem, mItems) if(nodeItem->getTag()==item) return true; return false; } @@ -254,6 +283,11 @@ SHAMapInnerNode::SHAMapInnerNode(const SHAMapInnerNode& node, uint32 seq) : SHAM memcpy(mHashes, node.mHashes, sizeof(mHashes)); } +void SHAMapInnerNode::addRaw(Serializer &s) +{ + for(int i=0; i<32; i++) + s.add256(mHashes[i]); +} bool SHAMapInnerNode::setChildHash(int m, const uint256 &hash) { assert( (m>=0) && (m<32) ); diff --git a/Serializer.cpp b/Serializer.cpp index d592ef276..f2156ea22 100644 --- a/Serializer.cpp +++ b/Serializer.cpp @@ -4,6 +4,17 @@ #include #include +int Serializer::add16(uint16 i) +{ + int ret=mData.size(); + for(int j=0; j>=8; + } + return ret; +} + int Serializer::add32(uint32 i) { int ret=mData.size(); @@ -47,8 +58,21 @@ int Serializer::addRaw(const std::vector &vector) return ret; } +bool Serializer::get16(uint16& o, int offset) const +{ + o=0; + if((offset+sizeof(o))>mData.size()) return false; + for(int i=0, o=0; imData.size()) return false; for(int i=0, o=0; imData.size()) return false; for(int i=0, o=0; i &data) : mData(data) { ; } // assemble functions + int add16(uint16); int add32(uint32); // ledger indexes, account sequence int add64(uint64); // timestamps, amounts int add160(const uint160&); // account names, hankos @@ -28,6 +29,7 @@ class Serializer int addRaw(const std::vector &vector); // disassemble functions + bool get16(uint16&, int offset) const; bool get32(uint32&, int offset) const; bool get64(uint64&, int offset) const; bool get160(uint160&, int offset) const;