ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2025-12-06 17:27:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Split SHAMapTreeNode into leaf and inner nodes.

Browse Source 
* This reduces the memory requirements of both leaf and inner nodes.
* The name SHAMapTreeNode is retained for leaf nodes so as to keep
  the public API of SHAMap stable.
This commit is contained in:
Howard Hinnant
2015-06-05 20:47:30 -04:00
committed by Nik Bougalis
parent e95ab65396
commit f875603525
11 changed files with 739 additions and 582 deletions
Download Patch File Download Diff File Expand all files Collapse all files

345
src/ripple/shamap/impl/SHAMapTreeNode.cpp
View File

@@ -31,55 +31,50 @@
namespace ripple {
std::mutex SHAMapTreeNode::childLock;
std::mutex SHAMapInnerNode::childLock;
SHAMapTreeNode::SHAMapTreeNode (std::uint32_t seq)
: mSeq (seq)
, mType (tnERROR)
, mIsBranch (0)
, mFullBelowGen (0)
SHAMapAbstractNode::~SHAMapAbstractNode() = default;
std::shared_ptr<SHAMapAbstractNode>
SHAMapInnerNode::clone(std::uint32_t seq) const
{
auto p = std::make_shared<SHAMapInnerNode>(seq);
p->mHash = mHash;
p->mIsBranch = mIsBranch;
p->mFullBelowGen = mFullBelowGen;
std::memcpy(p->mHashes, mHashes, sizeof(mHashes));
std::unique_lock <std::mutex> lock(childLock);
for (int i = 0; i < 16; ++i)
p->mChildren[i] = mChildren[i];
return std::move(p);
}
SHAMapTreeNode::SHAMapTreeNode (const SHAMapTreeNode& node, std::uint32_t seq)
: mHash (node.mHash)
, mSeq (seq)
, mType (node.mType)
, mIsBranch (node.mIsBranch)
, mFullBelowGen (0)
std::shared_ptr<SHAMapAbstractNode>
SHAMapTreeNode::clone(std::uint32_t seq) const
{
if (node.mItem)
mItem = node.mItem;
else
{
memcpy (mHashes, node.mHashes, sizeof (mHashes));
std::unique_lock <std::mutex> lock (childLock);
for (int i = 0; i < 16; ++i)
mChildren[i] = node.mChildren[i];
}
return std::make_shared<SHAMapTreeNode>(mItem, mType, seq, mHash);
}
SHAMapTreeNode::SHAMapTreeNode (std::shared_ptr<SHAMapItem> const& item,
TNType type, std::uint32_t seq)
: mItem (item)
, mSeq (seq)
, mType (type)
, mIsBranch (0)
, mFullBelowGen (0)
: SHAMapAbstractNode(type, seq)
, mItem (item)
{
assert (item->peekData ().size () >= 12);
updateHash ();
updateHash();
}
SHAMapTreeNode::SHAMapTreeNode (Blob const& rawNode,
std::uint32_t seq, SHANodeFormat format,
uint256 const& hash, bool hashValid)
: mSeq (seq)
, mType (tnERROR)
, mIsBranch (0)
, mFullBelowGen (0)
SHAMapTreeNode::SHAMapTreeNode (std::shared_ptr<SHAMapItem> const& item,
TNType type, std::uint32_t seq, uint256 const& hash)
: SHAMapAbstractNode(type, seq, hash)
, mItem (item)
{
assert (item->peekData ().size () >= 12);
}
std::shared_ptr<SHAMapAbstractNode>
SHAMapAbstractNode::make(Blob const& rawNode, std::uint32_t seq, SHANodeFormat format,
uint256 const& hash, bool hashValid)
{
if (format == snfWIRE)
{
@@ -110,11 +105,13 @@ SHAMapTreeNode::SHAMapTreeNode (Blob const& rawNode,
if (type == 0)
{
// transaction
mItem = std::make_shared<SHAMapItem>(
auto item = std::make_shared<SHAMapItem>(
sha512Half(HashPrefix::transactionID,
Slice(s.data(), s.size())),
s.peekData());
mType = tnTRANSACTION_NM;
if (hashValid)
return std::make_shared<SHAMapTreeNode>(item, tnTRANSACTION_NM, seq, hash);
return std::make_shared<SHAMapTreeNode>(item, tnTRANSACTION_NM, seq);
}
else if (type == 1)
{
@@ -128,8 +125,10 @@ SHAMapTreeNode::SHAMapTreeNode (Blob const& rawNode,
if (u.isZero ()) throw std::runtime_error ("invalid AS node");
mItem = std::make_shared<SHAMapItem> (u, s.peekData ());
mType = tnACCOUNT_STATE;
auto item = std::make_shared<SHAMapItem> (u, s.peekData ());
if (hashValid)
return std::make_shared<SHAMapTreeNode>(item, tnACCOUNT_STATE, seq, hash);
return std::make_shared<SHAMapTreeNode>(item, tnACCOUNT_STATE, seq);
}
else if (type == 2)
{
@@ -137,18 +136,23 @@ SHAMapTreeNode::SHAMapTreeNode (Blob const& rawNode,
if (len != 512)
throw std::runtime_error ("invalid FI node");
auto ret = std::make_shared<SHAMapInnerNode>(seq);
for (int i = 0; i < 16; ++i)
{
s.get256 (mHashes[i], i * 32);
s.get256 (ret->mHashes[i], i * 32);
if (mHashes[i].isNonZero ())
mIsBranch |= (1 << i);
if (ret->mHashes[i].isNonZero ())
ret->mIsBranch |= (1 << i);
}
mType = tnINNER;
if (hashValid)
ret->mHash = hash;
else
ret->updateHash();
return ret;
}
else if (type == 3)
{
auto ret = std::make_shared<SHAMapInnerNode>(seq);
// compressed inner
for (int i = 0; i < (len / 33); ++i)
{
@@ -157,13 +161,16 @@ SHAMapTreeNode::SHAMapTreeNode (Blob const& rawNode,
if ((pos < 0) || (pos >= 16)) throw std::runtime_error ("invalid CI node");
s.get256 (mHashes[pos], i * 33);
s.get256 (ret->mHashes[pos], i * 33);
if (mHashes[pos].isNonZero ())
mIsBranch |= (1 << pos);
if (ret->mHashes[pos].isNonZero ())
ret->mIsBranch |= (1 << pos);
}
mType = tnINNER;
if (hashValid)
ret->mHash = hash;
else
ret->updateHash();
return ret;
}
else if (type == 4)
{
@@ -178,8 +185,10 @@ SHAMapTreeNode::SHAMapTreeNode (Blob const& rawNode,
if (u.isZero ())
throw std::runtime_error ("invalid TM node");
mItem = std::make_shared<SHAMapItem> (u, s.peekData ());
mType = tnTRANSACTION_MD;
auto item = std::make_shared<SHAMapItem> (u, s.peekData ());
if (hashValid)
return std::make_shared<SHAMapTreeNode>(item, tnTRANSACTION_MD, seq, hash);
return std::make_shared<SHAMapTreeNode>(item, tnTRANSACTION_MD, seq);
}
}
@@ -202,10 +211,12 @@ SHAMapTreeNode::SHAMapTreeNode (Blob const& rawNode,
if (prefix == HashPrefix::transactionID)
{
mItem = std::make_shared<SHAMapItem>(
auto item = std::make_shared<SHAMapItem>(
sha512Half(make_Slice(rawNode)),
s.peekData ());
mType = tnTRANSACTION_NM;
if (hashValid)
return std::make_shared<SHAMapTreeNode>(item, tnTRANSACTION_NM, seq, hash);
return std::make_shared<SHAMapTreeNode>(item, tnTRANSACTION_NM, seq);
}
else if (prefix == HashPrefix::leafNode)
{
@@ -222,23 +233,28 @@ SHAMapTreeNode::SHAMapTreeNode (Blob const& rawNode,
throw std::runtime_error ("invalid PLN node");
}
mItem = std::make_shared<SHAMapItem> (u, s.peekData ());
mType = tnACCOUNT_STATE;
auto item = std::make_shared<SHAMapItem> (u, s.peekData ());
if (hashValid)
return std::make_shared<SHAMapTreeNode>(item, tnACCOUNT_STATE, seq, hash);
return std::make_shared<SHAMapTreeNode>(item, tnACCOUNT_STATE, seq);
}
else if (prefix == HashPrefix::innerNode)
{
if (s.getLength () != 512)
throw std::runtime_error ("invalid PIN node");
auto ret = std::make_shared<SHAMapInnerNode>(seq);
for (int i = 0; i < 16; ++i)
{
s.get256 (mHashes[i], i * 32);
s.get256 (ret->mHashes[i], i * 32);
if (mHashes[i].isNonZero ())
mIsBranch |= (1 << i);
if (ret->mHashes[i].isNonZero ())
ret->mIsBranch |= (1 << i);
}
mType = tnINNER;
if (hashValid)
ret->mHash = hash;
else
ret->updateHash();
return ret;
}
else if (prefix == HashPrefix::txNode)
{
@@ -249,8 +265,10 @@ SHAMapTreeNode::SHAMapTreeNode (Blob const& rawNode,
uint256 txID;
s.get256 (txID, s.getLength () - 32);
s.chop (32);
mItem = std::make_shared<SHAMapItem> (txID, s.peekData ());
mType = tnTRANSACTION_MD;
auto item = std::make_shared<SHAMapItem> (txID, s.peekData ());
if (hashValid)
return std::make_shared<SHAMapTreeNode>(item, tnTRANSACTION_MD, seq, hash);
return std::make_shared<SHAMapTreeNode>(item, tnTRANSACTION_MD, seq);
}
else
{
@@ -258,49 +276,50 @@ SHAMapTreeNode::SHAMapTreeNode (Blob const& rawNode,
throw std::runtime_error ("invalid node prefix");
}
}
else
{
assert (false);
throw std::runtime_error ("Unknown format");
}
if (hashValid)
{
mHash = hash;
#if RIPPLE_VERIFY_NODEOBJECT_KEYS
updateHash ();
assert (mHash == hash);
#endif
}
else
updateHash ();
assert (false);
throw std::runtime_error ("Unknown format");
}
bool SHAMapTreeNode::updateHash ()
bool
SHAMapInnerNode::updateHash()
{
uint256 nh;
if (mType == tnINNER)
if (mIsBranch != 0)
{
if (mIsBranch != 0)
{
// VFALCO This code assumes the layout of a base_uint
nh = sha512Half(HashPrefix::innerNode,
Slice(reinterpret_cast<unsigned char const*>(mHashes),
sizeof (mHashes)));
// VFALCO This code assumes the layout of a base_uint
nh = sha512Half(HashPrefix::innerNode,
Slice(reinterpret_cast<unsigned char const*>(mHashes),
sizeof (mHashes)));
#if RIPPLE_VERIFY_NODEOBJECT_KEYS
SHA512HalfHasher h;
using beast::hash_append;
hash_append(h, HashPrefix::innerNode, mHashes);
assert (nh == sha512Half(
static_cast<uint256>(h)));
SHA512HalfHasher h;
using beast::hash_append;
hash_append(h, HashPrefix::innerNode, mHashes);
assert (nh == sha512Half(
static_cast<uint256>(h)));
#endif
}
else
nh.zero ();
}
else if (mType == tnTRANSACTION_NM)
if (nh == mHash)
return false;
mHash = nh;
return true;
}
void
SHAMapInnerNode::updateHashDeep()
{
for (auto pos = 0; pos < 16; ++pos)
{
if (mChildren[pos] != nullptr)
mHashes[pos] = mChildren[pos]->getNodeHash();
}
updateHash();
}
bool
SHAMapTreeNode::updateHash()
{
uint256 nh;
if (mType == tnTRANSACTION_NM)
{
nh = sha512Half(HashPrefix::transactionID,
make_Slice(mItem->peekData()));
@@ -328,17 +347,7 @@ bool SHAMapTreeNode::updateHash ()
}
void
SHAMapTreeNode::updateHashDeep()
{
for (auto pos = 0; pos < 16; ++pos)
{
if (mChildren[pos] != nullptr)
mHashes[pos] = mChildren[pos]->mHash;
}
updateHash();
}
void SHAMapTreeNode::addRaw (Serializer& s, SHANodeFormat format)
SHAMapInnerNode::addRaw(Serializer& s, SHANodeFormat format)
{
assert ((format == snfPREFIX) || (format == snfWIRE) || (format == snfHASH));
@@ -383,6 +392,22 @@ void SHAMapTreeNode::addRaw (Serializer& s, SHANodeFormat format)
}
}
}
else
assert (false);
}
void
SHAMapTreeNode::addRaw(Serializer& s, SHANodeFormat format)
{
assert ((format == snfPREFIX) || (format == snfWIRE) || (format == snfHASH));
if (mType == tnERROR)
throw std::runtime_error ("invalid I node type");
if (format == snfHASH)
{
s.add256 (getNodeHash ());
}
else if (mType == tnACCOUNT_STATE)
{
if (format == snfPREFIX)
@@ -439,12 +464,12 @@ bool SHAMapTreeNode::setItem (std::shared_ptr<SHAMapItem> const& i, TNType type)
return updateHash ();
}
bool SHAMapTreeNode::isEmpty () const
bool SHAMapInnerNode::isEmpty () const
{
return mIsBranch == 0;
}
int SHAMapTreeNode::getBranchCount () const
int SHAMapInnerNode::getBranchCount () const
{
assert (isInner ());
int count = 0;
@@ -456,18 +481,10 @@ int SHAMapTreeNode::getBranchCount () const
return count;
}
void SHAMapTreeNode::makeInner ()
{
mItem.reset ();
mIsBranch = 0;
memset (mHashes, 0, sizeof (mHashes));
mType = tnINNER;
mHash.zero ();
}
#ifdef BEAST_DEBUG
void SHAMapTreeNode::dump (const SHAMapNodeID & id, beast::Journal journal)
void
SHAMapAbstractNode::dump(const SHAMapNodeID & id, beast::Journal journal)
{
if (journal.debug) journal.debug <<
"SHAMapTreeNode(" << id.getNodeID () << ")";
@@ -475,51 +492,59 @@ void SHAMapTreeNode::dump (const SHAMapNodeID & id, beast::Journal journal)
#endif // BEAST_DEBUG
std::string SHAMapTreeNode::getString (const SHAMapNodeID & id) const
std::string
SHAMapAbstractNode::getString(const SHAMapNodeID & id) const
{
std::string ret = "NodeID(";
ret += beast::lexicalCastThrow <std::string> (id.getDepth ());
ret += ",";
ret += to_string (id.getNodeID ());
ret += ")";
return ret;
}
if (isInner ())
std::string
SHAMapInnerNode::getString(const SHAMapNodeID & id) const
{
std::string ret = SHAMapAbstractNode::getString(id);
for (int i = 0; i < 16; ++i)
{
for (int i = 0; i < 16; ++i)
if (!isEmptyBranch (i))
{
ret += "\nb";
ret += beast::lexicalCastThrow <std::string> (i);
ret += " = ";
ret += to_string (mHashes[i]);
}
if (!isEmptyBranch (i))
{
ret += "\nb";
ret += beast::lexicalCastThrow <std::string> (i);
ret += " = ";
ret += to_string (mHashes[i]);
}
}
return ret;
}
if (isLeaf ())
{
if (mType == tnTRANSACTION_NM)
ret += ",txn\n";
else if (mType == tnTRANSACTION_MD)
ret += ",txn+md\n";
else if (mType == tnACCOUNT_STATE)
ret += ",as\n";
else
ret += ",leaf\n";
ret += " Tag=";
ret += to_string (peekItem()->getTag ());
ret += "\n Hash=";
ret += to_string (mHash);
ret += "/";
ret += beast::lexicalCast <std::string> (mItem->size());
}
std::string
SHAMapTreeNode::getString(const SHAMapNodeID & id) const
{
std::string ret = SHAMapAbstractNode::getString(id);
if (mType == tnTRANSACTION_NM)
ret += ",txn\n";
else if (mType == tnTRANSACTION_MD)
ret += ",txn+md\n";
else if (mType == tnACCOUNT_STATE)
ret += ",as\n";
else
ret += ",leaf\n";
ret += " Tag=";
ret += to_string (peekItem()->getTag ());
ret += "\n Hash=";
ret += to_string (mHash);
ret += "/";
ret += beast::lexicalCast <std::string> (mItem->size());
return ret;
}
// We are modifying an inner node
void
SHAMapTreeNode::setChild (int m, std::shared_ptr<SHAMapTreeNode> const& child)
SHAMapInnerNode::setChild(int m, std::shared_ptr<SHAMapAbstractNode> const& child)
{
assert ((m >= 0) && (m < 16));
assert (mType == tnINNER);
@@ -535,7 +560,7 @@ SHAMapTreeNode::setChild (int m, std::shared_ptr<SHAMapTreeNode> const& child)
}
// finished modifying, now make shareable
void SHAMapTreeNode::shareChild (int m, std::shared_ptr<SHAMapTreeNode> const& child)
void SHAMapInnerNode::shareChild (int m, std::shared_ptr<SHAMapAbstractNode> const& child)
{
assert ((m >= 0) && (m < 16));
assert (mType == tnINNER);
@@ -546,28 +571,31 @@ void SHAMapTreeNode::shareChild (int m, std::shared_ptr<SHAMapTreeNode> const& c
mChildren[m] = child;
}
SHAMapTreeNode* SHAMapTreeNode::getChildPointer (int branch)
SHAMapAbstractNode*
SHAMapInnerNode::getChildPointer (int branch)
{
assert (branch >= 0 && branch < 16);
assert (isInnerNode ());
assert (isInner());
std::unique_lock <std::mutex> lock (childLock);
return mChildren[branch].get ();
}
std::shared_ptr<SHAMapTreeNode> SHAMapTreeNode::getChild (int branch)
std::shared_ptr<SHAMapAbstractNode>
SHAMapInnerNode::getChild (int branch)
{
assert (branch >= 0 && branch < 16);
assert (isInnerNode ());
assert (isInner());
std::unique_lock <std::mutex> lock (childLock);
return mChildren[branch];
}
void SHAMapTreeNode::canonicalizeChild (int branch, std::shared_ptr<SHAMapTreeNode>& node)
std::shared_ptr<SHAMapAbstractNode>
SHAMapInnerNode::canonicalizeChild(int branch, std::shared_ptr<SHAMapAbstractNode> node)
{
assert (branch >= 0 && branch < 16);
assert (isInnerNode ());
assert (isInner());
assert (node);
assert (node->getNodeHash() == mHashes[branch]);
@@ -582,6 +610,7 @@ void SHAMapTreeNode::canonicalizeChild (int branch, std::shared_ptr<SHAMapTreeNo
// Hook this node up
mChildren[branch] = node;
}
return node;
}