ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2025-11-21 19:45:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Major optimization in the way we track SHAMaps.

Browse Source
This commit is contained in:
JoelKatz
2012-02-01 20:51:58 -08:00
parent 5888d19f2b
commit 08d6b55fed
6 changed files with 103 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
SHAMap.cpp
View File

@@ -67,7 +67,7 @@ SHAMapLeafNode::pointer SHAMap::checkCacheLeaf(const SHAMapNode& iNode)
SHAMapInnerNode::pointer SHAMap::checkCacheNode(const SHAMapNode& iNode)
{
assert(!iNode.isLeaf());
std::map<SHAMapNode, SHAMapInnerNode::pointer>::iterator it=mInnerNodeByID.find(iNode);
boost::unordered_map<SHAMapNode, SHAMapInnerNode::pointer>::iterator it=mInnerNodeByID.find(iNode);
if(it==mInnerNodeByID.end()) return SHAMapInnerNode::pointer();
return it->second;
}
@@ -89,10 +89,6 @@ SHAMapLeafNode::pointer SHAMap::walkToLeaf(const uint256& id, bool create, bool
throw SHAMapException(InvalidNode);
if(inNode->isEmptyBranch(branch))
{ // no nodes below this one
#ifdef DEBUG
std::cerr << "No nodes below level " << i << ", branch " << branch << std::endl;
std::cerr << " terminal node is " << inNode->getString() << std::endl;
#endif
if(!create) return SHAMapLeafNode::pointer();
return createLeaf(*inNode, id);
}
@@ -140,7 +136,7 @@ SHAMapInnerNode::pointer SHAMap::walkTo(const SHAMapNode& id)
SHAMapInnerNode::pointer next=getInner(inNode->getChildNodeID(branch), inNode->getChildHash(branch), false);
if(!next) // we don't have the next node
{
#ifdef DEBUG
#ifdef ST_DEBUG
std::cerr << "Unable to find node " << inNode->getChildNodeID(branch).getString() << std::endl;
#endif
return inNode;
@@ -166,7 +162,7 @@ SHAMapLeafNode::pointer SHAMap::getLeaf(const SHAMapNode& id, const uint256& has
if(leaf) return returnLeaf(leaf, modify);
std::vector<unsigned char> leafData;
if(!fetchNode(hash, leafData)) throw SHAMapException(MissingNode);
if(!fetchNode(hash, leafData)) return SHAMapLeafNode::pointer();
leaf=SHAMapLeafNode::pointer(new SHAMapLeafNode(id, leafData, mSeq));
if(leaf->getNodeHash()!=hash) throw SHAMapException(InvalidNode);
mLeafByID.insert(std::make_pair(id, leaf));
@@ -179,7 +175,7 @@ SHAMapInnerNode::pointer SHAMap::getInner(const SHAMapNode& id, const uint256& h
if(node) return returnNode(node, modify);
std::vector<unsigned char> rawNode;
if(!fetchNode(hash, rawNode)) throw SHAMapException(MissingNode);
if(!fetchNode(hash, rawNode)) return SHAMapInnerNode::pointer();
node=SHAMapInnerNode::pointer(new SHAMapInnerNode(id, rawNode, mSeq));
if(node->getNodeHash()!=hash) throw SHAMapException(InvalidNode);
@@ -421,7 +417,7 @@ SHAMapItem::pointer SHAMap::peekPrevItem(const uint256& id)
SHAMapLeafNode::pointer SHAMap::createLeaf(const SHAMapInnerNode& lowestParent, const uint256& id)
{ // caller must call dirtyUp if they populate the leaf
#ifdef DEBUG
#ifdef ST_DEBUG
std::cerr << "createLeaf(" << lowestParent.getString() << std::endl;
std::cerr << " for " << id.GetHex() << ")" << std::endl;
#endif
@@ -433,7 +429,7 @@ SHAMapLeafNode::pointer SHAMap::createLeaf(const SHAMapInnerNode& lowestParent,
}
SHAMapLeafNode::pointer newLeaf(new SHAMapLeafNode(SHAMapNode(SHAMapNode::leafDepth, id), mSeq));
mLeafByID[*newLeaf]=newLeaf;
#ifdef DEBUG
#ifdef ST_DEBUG
std::cerr << "made leaf " << newLeaf->getString() << std::endl;
#endif
return newLeaf;

24
SHAMap.h
View File

@@ -7,6 +7,7 @@
#include <boost/shared_ptr.hpp>
#include <boost/enable_shared_from_this.hpp>
#include <boost/unordered/unordered_map.hpp>
#include "types.h"
#include "uint256.h"
@@ -49,9 +50,9 @@ public:
bool isInner() const { return !isRoot() && !isLeaf(); }
virtual bool isPopulated() const { return false; }
SHAMapNode getParentNodeID() { return SHAMapNode(mDepth-1, mNodeID); }
SHAMapNode getChildNodeID(int m);
int selectBranch(const uint256& hash);
SHAMapNode getParentNodeID() const { return SHAMapNode(mDepth-1, mNodeID); }
SHAMapNode getChildNodeID(int m) const;
int selectBranch(const uint256& hash) const;
bool operator<(const SHAMapNode&) const;
bool operator>(const SHAMapNode&) const;
@@ -62,8 +63,10 @@ public:
bool operator<=(const SHAMapNode&) const;
bool operator>=(const SHAMapNode&) const;
std::size_t getHash() const;
virtual std::string getString() const;
void dump();
void dump() const;
static void ClassInit();
static uint256 getNodeID(int depth, const uint256& hash);
@@ -164,7 +167,6 @@ class SHAMapInnerNode : public SHAMapNode
public:
typedef boost::shared_ptr<SHAMapInnerNode> pointer;
private:
uint256 mHash;
uint256 mHashes[32];
@@ -208,6 +210,14 @@ enum SHAMapException
InvalidNode=2
};
class hash_SMN : std::unary_function<SHAMapNode, std::size_t>
{
public:
std::size_t operator() (const SHAMapNode& mn) const
{
return mn.getHash();
}
};
class SHAMap
{
@@ -219,7 +229,9 @@ private:
uint32 mSeq;
mutable boost::recursive_mutex mLock;
std::map<SHAMapNode, SHAMapLeafNode::pointer> mLeafByID;
std::map<SHAMapNode, SHAMapInnerNode::pointer> mInnerNodeByID;
boost::unordered_map<SHAMapNode, SHAMapInnerNode::pointer, hash_SMN> mInnerNodeByID;
boost::shared_ptr<std::map<SHAMapNode, SHAMapLeafNode::pointer> > mDirtyLeafNodes;
boost::shared_ptr<std::map<SHAMapNode, SHAMapInnerNode::pointer> > mDirtyInnerNodes;

19
SHAMapNodes.cpp
View File

@@ -81,13 +81,24 @@ uint256 SHAMapNode::getNodeID(int depth, const uint256& hash)
return hash & smMasks[depth];
}
std::size_t SHAMapNode::getHash() const
{
std::size_t ret=mDepth;
for(int i=0; i<5; i++)
{
ret*=2654435761U;
ret^=mNodeID.PeekAt(i);
}
return ret;
}
SHAMapNode::SHAMapNode(int depth, const uint256 &hash) : mDepth(depth)
{ // canonicalize the hash to a node ID for this depth
assert(depth>=0 && depth<=leafDepth);
mNodeID = getNodeID(depth, hash);
}
SHAMapNode SHAMapNode::getChildNodeID(int m)
SHAMapNode SHAMapNode::getChildNodeID(int m) const
{
assert(!isLeaf());
assert((m>=0) && (m<32));
@@ -99,7 +110,7 @@ SHAMapNode SHAMapNode::getChildNodeID(int m)
return ret;
}
int SHAMapNode::selectBranch(const uint256 &hash)
int SHAMapNode::selectBranch(const uint256 &hash) const
{
if(isLeaf()) // no nodes under this node
{
@@ -118,7 +129,7 @@ int SHAMapNode::selectBranch(const uint256 &hash)
return branch;
}
void SHAMapNode::dump()
void SHAMapNode::dump() const
{
std::cerr << getString() << std::endl;
}
@@ -172,7 +183,7 @@ bool SHAMapLeafNode::hasItem(const uint256& item) const
bool SHAMapLeafNode::addUpdateItem(SHAMapItem::pointer item, bool doHash)
{ // The node will almost never have more than one item in it
#ifdef DEBUG
#ifdef ST_DEBUG
std::cerr << "Leaf(" << getString() << ")" << std::endl;
std::cerr << " addi(" << item->getTag().GetHex() << std::endl;
#endif

77
SHAMapSync.cpp
View File

@@ -1,6 +1,8 @@
#include <stack>
#include <openssl/rand.h>
#include "SHAMap.h"
void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint256>& hashes, int max)
@@ -9,7 +11,7 @@ void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint2
if(root->isFullBelow())
{
#ifdef DEBUG
#ifdef GMN_DEBUG
std::cerr << "getMissingNodes: root is full below" << std::endl;
#endif
return;
@@ -23,7 +25,7 @@ void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint2
SHAMapInnerNode::pointer node=stack.top();
stack.pop();
#ifdef DEBUG
#ifdef GMN_DEBUG
std::cerr << "gMN: popped " << node->getString() << std::endl;
#endif
@@ -31,35 +33,29 @@ void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint2
{
if(!node->isEmptyBranch(i))
{
#ifdef DEBUG
#ifdef GNMN_DEBUG
std::cerr << "gMN: " << node->getString() << " has non-empty branch " << i << std::endl;
#endif
try
{
bool missing=false;
SHAMapNode childNID=node->getChildNodeID(i);
if(node->isChildLeaf())
{ // do we have this leaf node?
SHAMapLeafNode::pointer leaf=getLeaf(node->getChildNodeID(i), node->getChildHash(i), false);
assert(leaf);
if(!getLeaf(childNID, node->getChildHash(i), false)) missing=true;
}
else
{
SHAMapInnerNode::pointer desc=getInner(node->getChildNodeID(i), node->getChildHash(i), false);
assert(desc);
if(!desc->isFullBelow())
SHAMapInnerNode::pointer desc=getInner(childNID, node->getChildHash(i), false);
if(!desc)
missing=true;
else if(!desc->isFullBelow())
stack.push(desc);
}
}
catch(const SHAMapException &x)
if(missing && max-->0)
{
if(x!=SHAMapException(MissingNode)) throw(x);
if(max-->0)
{
#ifdef DEBUG
std::cerr << "gMN: need leaf " << node->getChildNodeID(i).getString() << std::endl;
#ifdef GMN_DEBUG
std::cerr << "gMN: need " << node->getChildNodeID(i).getString() << std::endl;
#endif
nodeIDs.push_back(node->getChildNodeID(i));
hashes.push_back(node->getChildHash(i));
}
nodeIDs.push_back(childNID);
}
}
}
@@ -253,7 +249,32 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vector<unsigned cha
mLeafByID[node]=leaf;
if(mDirtyLeafNodes) (*mDirtyLeafNodes)[node]=leaf;
// WRITEME: See if our parent(s) are full below
// FIXME: This should check all sources
SHAMapInnerNode::pointer pNode=checkCacheNode(node.getParentNodeID());
if(!pNode)
{
assert(false);
return false;
}
for(int i=0; i<32; i++)
if(!checkCacheLeaf(pNode->getChildNodeID(i)))
return true;
pNode->setFullBelow();
while(!pNode->isRoot())
{
pNode=checkCacheNode(pNode->getParentNodeID());
if(!pNode)
{
assert(false);
return false;
}
for(int i=0; i<32; i++)
if(!checkCacheNode(pNode->getChildNodeID(i)))
return true;
pNode->setFullBelow();
}
return true;
}
@@ -270,7 +291,7 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vector<unsigned cha
}
mInnerNodeByID[node]=newNode;
if(mDirtyInnerNodes) (*mDirtyInnerNodes)[node]=newNode;
#ifdef DEBUG
#ifdef ST_DEBUG
std::cerr << "Hooked: " << node.getString() << std::endl;
#endif
return true;
@@ -302,7 +323,7 @@ bool SHAMap::deepCompare(SHAMap& other)
return false;
}
#ifdef DEBUG
#ifdef DC_DEBUG
std::cerr << "Comparing inner nodes " << node->getString() << std::endl;
#endif
@@ -353,10 +374,14 @@ bool SHAMap::deepCompare(SHAMap& other)
bool SHAMap::syncTest()
{
unsigned int seed;
RAND_pseudo_bytes(reinterpret_cast<unsigned char *>(&seed), sizeof(seed));
srand(seed);
SHAMap source, destination;
// add random data to the source map
int items=3; // 10+rand()%400;
int items=10+rand()%4000;
for(int i=0; i<items; i++)
{
Serializer s;
@@ -365,7 +390,7 @@ bool SHAMap::syncTest()
s.add32(rand());
uint256 id=s.getSHA512Half();
source.addItem(SHAMapItem(id, s.peekData()));
#ifdef DEBUG
#ifdef ST_DEBUG
std::cerr << "Item: " << id.GetHex() << std::endl;
#endif
}
@@ -418,7 +443,7 @@ bool SHAMap::syncTest()
hashes.clear();
// get the list of nodes we know we need
destination.getMissingNodes(nodeIDs, hashes, 128);
destination.getMissingNodes(nodeIDs, hashes, 512);
if(!nodeIDs.size()) break;
#ifdef DEBUG

6
Serializer.cpp
View File

@@ -152,10 +152,10 @@ uint256 Serializer::getSHA512Half(int size) const
uint256 Serializer::getSHA512Half(const std::vector<unsigned char>& data, int size)
{
char buf[64];
uint256 j[2];
if((size<0)||(size>data.size())) size=data.size();
SHA512(&(data.front()), size, (unsigned char *) buf);
return * reinterpret_cast<uint256*>(&buf[0]);
SHA512(&(data.front()), size, (unsigned char *) j);
return j[0];
}
bool Serializer::checkSignature(int pubkeyOffset, int signatureOffset) const

5
uint256.h
View File

@@ -290,7 +290,10 @@ public:
return (!(a == b));
}
unsigned int PeekAt(int j) const
{
return pn[j];
}
std::string GetHex() const
{