ARCHIVE/xahaud
3
0
Fork 0
mirror of https://github.com/Xahau/xahaud.git synced 2025-12-06 17:27:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Store InnerNode children in sparse arrays:

Browse Source 
A large percentage of inner nodes only store a small number of children. Memory
can be saved by storing the inner node's children in sparse arrays. Measurements
show that on average a typical SHAMap's inner nodes can be stored using only 25%
of the original space.
This commit is contained in:
seelabs
2020-11-12 14:46:16 -05:00
committed by Nik Bougalis
parent 5c8e072b7f
commit cb0572d66e
5 changed files with 1146 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

281
src/ripple/shamap/impl/SHAMapInnerNode.cpp
View File

@@ -28,12 +28,12 @@
#include <ripple/protocol/HashPrefix.h>
#include <ripple/protocol/digest.h>
#include <ripple/shamap/SHAMapTreeNode.h>
#include <ripple/shamap/impl/TaggedPointer.ipp>
#include <openssl/sha.h>
#include <algorithm>
#include <array>
#include <atomic>
#include <iterator>
#include <mutex>
#include <utility>
@@ -42,17 +42,85 @@ namespace ripple {
std::mutex SHAMapInnerNode::childLock;
SHAMapInnerNode::SHAMapInnerNode(
std::uint32_t cowid,
std::uint8_t numAllocatedChildren)
: SHAMapTreeNode(cowid), hashesAndChildren_(numAllocatedChildren)
{
}
template <class F>
void
SHAMapInnerNode::iterChildren(F&& f) const
{
hashesAndChildren_.iterChildren(isBranch_, std::forward<F>(f));
}
template <class F>
void
SHAMapInnerNode::iterNonEmptyChildIndexes(F&& f) const
{
hashesAndChildren_.iterNonEmptyChildIndexes(isBranch_, std::forward<F>(f));
}
void
SHAMapInnerNode::resizeChildArrays(std::uint8_t toAllocate)
{
hashesAndChildren_ =
TaggedPointer(std::move(hashesAndChildren_), isBranch_, toAllocate);
}
std::optional<int>
SHAMapInnerNode::getChildIndex(int i) const
{
return hashesAndChildren_.getChildIndex(isBranch_, i);
}
std::shared_ptr<SHAMapTreeNode>
SHAMapInnerNode::clone(std::uint32_t cowid) const
{
auto p = std::make_shared<SHAMapInnerNode>(cowid);
auto const branchCount = getBranchCount();
auto const thisIsSparse = !hashesAndChildren_.isDense();
auto p = std::make_shared<SHAMapInnerNode>(cowid, branchCount);
p->hash_ = hash_;
p->mIsBranch = mIsBranch;
p->mFullBelowGen = mFullBelowGen;
p->mHashes = mHashes;
p->isBranch_ = isBranch_;
p->fullBelowGen_ = fullBelowGen_;
SHAMapHash *cloneHashes, *thisHashes;
std::shared_ptr<SHAMapTreeNode>*cloneChildren, *thisChildren;
// structured bindings can't be captured in c++ 17; use tie instead
std::tie(std::ignore, cloneHashes, cloneChildren) =
p->hashesAndChildren_.getHashesAndChildren();
std::tie(std::ignore, thisHashes, thisChildren) =
hashesAndChildren_.getHashesAndChildren();
if (thisIsSparse)
{
int cloneChildIndex = 0;
iterNonEmptyChildIndexes([&](auto branchNum, auto indexNum) {
cloneHashes[cloneChildIndex++] = thisHashes[indexNum];
});
}
else
{
iterNonEmptyChildIndexes([&](auto branchNum, auto indexNum) {
cloneHashes[branchNum] = thisHashes[indexNum];
});
}
std::lock_guard lock(childLock);
for (int i = 0; i < 16; ++i)
p->mChildren[i] = mChildren[i];
if (thisIsSparse)
{
int cloneChildIndex = 0;
iterNonEmptyChildIndexes([&](auto branchNum, auto indexNum) {
cloneChildren[cloneChildIndex++] = thisChildren[indexNum];
});
}
else
{
iterNonEmptyChildIndexes([&](auto branchNum, auto indexNum) {
cloneChildren[branchNum] = thisChildren[indexNum];
});
}
return p;
}
@@ -65,18 +133,21 @@ SHAMapInnerNode::makeFullInner(
if (data.size() != 512)
Throw<std::runtime_error>("Invalid FI node");
auto ret = std::make_shared<SHAMapInnerNode>(0);
auto ret = std::make_shared<SHAMapInnerNode>(0, branchFactor);
Serializer s(data.data(), data.size());
for (int i = 0; i < 16; ++i)
auto retHashes = ret->hashesAndChildren_.getHashes();
for (int i = 0; i < branchFactor; ++i)
{
s.getBitString(ret->mHashes[i].as_uint256(), i * 32);
s.getBitString(retHashes[i].as_uint256(), i * 32);
if (ret->mHashes[i].isNonZero())
ret->mIsBranch |= (1 << i);
if (retHashes[i].isNonZero())
ret->isBranch_ |= (1 << i);
}
ret->resizeChildArrays(ret->getBranchCount());
if (hashValid)
ret->hash_ = hash;
else
@@ -91,8 +162,9 @@ SHAMapInnerNode::makeCompressedInner(Slice data)
int len = s.getLength();
auto ret = std::make_shared<SHAMapInnerNode>(0);
auto ret = std::make_shared<SHAMapInnerNode>(0, branchFactor);
auto retHashes = ret->hashesAndChildren_.getHashes();
for (int i = 0; i < (len / 33); ++i)
{
int pos;
@@ -100,15 +172,17 @@ SHAMapInnerNode::makeCompressedInner(Slice data)
if (!s.get8(pos, 32 + (i * 33)))
Throw<std::runtime_error>("short CI node");
if ((pos < 0) || (pos >= 16))
if ((pos < 0) || (pos >= branchFactor))
Throw<std::runtime_error>("invalid CI node");
s.getBitString(ret->mHashes[pos].as_uint256(), i * 33);
s.getBitString(retHashes[pos].as_uint256(), i * 33);
if (ret->mHashes[pos].isNonZero())
ret->mIsBranch |= (1 << pos);
if (retHashes[pos].isNonZero())
ret->isBranch_ |= (1 << pos);
}
ret->resizeChildArrays(ret->getBranchCount());
ret->updateHash();
return ret;
@@ -118,13 +192,12 @@ void
SHAMapInnerNode::updateHash()
{
uint256 nh;
if (mIsBranch != 0)
if (isBranch_ != 0)
{
sha512_half_hasher h;
using beast::hash_append;
hash_append(h, HashPrefix::innerNode);
for (auto const& hh : mHashes)
hash_append(h, hh);
iterChildren([&](SHAMapHash const& hh) { hash_append(h, hh); });
nh = static_cast<typename sha512_half_hasher::result_type>(h);
}
hash_ = SHAMapHash{nh};
@@ -133,11 +206,15 @@ SHAMapInnerNode::updateHash()
void
SHAMapInnerNode::updateHashDeep()
{
for (auto pos = 0; pos < 16; ++pos)
{
if (mChildren[pos] != nullptr)
mHashes[pos] = mChildren[pos]->getHash();
}
SHAMapHash* hashes;
std::shared_ptr<SHAMapTreeNode>* children;
// structured bindings can't be captured in c++ 17; use tie instead
std::tie(std::ignore, hashes, children) =
hashesAndChildren_.getHashesAndChildren();
iterNonEmptyChildIndexes([&](auto branchNum, auto indexNum) {
if (children[indexNum] != nullptr)
hashes[indexNum] = children[indexNum]->getHash();
});
updateHash();
}
@@ -150,22 +227,17 @@ SHAMapInnerNode::serializeForWire(Serializer& s) const
if (getBranchCount() < 12)
{
// compressed node
for (int i = 0; i < mHashes.size(); ++i)
{
if (!isEmptyBranch(i))
{
s.addBitString(mHashes[i].as_uint256());
s.add8(i);
}
}
auto hashes = hashesAndChildren_.getHashes();
iterNonEmptyChildIndexes([&](auto branchNum, auto indexNum) {
s.addBitString(hashes[indexNum].as_uint256());
s.add8(branchNum);
});
s.add8(wireTypeCompressedInner);
}
else
{
for (auto const& hh : mHashes)
s.addBitString(hh.as_uint256());
iterChildren(
[&](SHAMapHash const& hh) { s.addBitString(hh.as_uint256()); });
s.add8(wireTypeInner);
}
}
@@ -176,42 +248,33 @@ SHAMapInnerNode::serializeWithPrefix(Serializer& s) const
assert(!isEmpty());
s.add32(HashPrefix::innerNode);
for (auto const& hh : mHashes)
s.addBitString(hh.as_uint256());
iterChildren(
[&](SHAMapHash const& hh) { s.addBitString(hh.as_uint256()); });
}
bool
SHAMapInnerNode::isEmpty() const
{
return mIsBranch == 0;
return isBranch_ == 0;
}
int
SHAMapInnerNode::getBranchCount() const
{
int count = 0;
for (int i = 0; i < 16; ++i)
if (!isEmptyBranch(i))
++count;
return count;
return popcnt16(isBranch_);
}
std::string
SHAMapInnerNode::getString(const SHAMapNodeID& id) const
{
std::string ret = SHAMapTreeNode::getString(id);
for (int i = 0; i < mHashes.size(); ++i)
{
if (!isEmptyBranch(i))
{
ret += "\n";
ret += std::to_string(i);
ret += " = ";
ret += to_string(mHashes[i]);
}
}
auto hashes = hashesAndChildren_.getHashes();
iterNonEmptyChildIndexes([&](auto branchNum, auto indexNum) {
ret += "\nb";
ret += std::to_string(branchNum);
ret += " = ";
ret += to_string(hashes[indexNum]);
});
return ret;
}
@@ -219,46 +282,79 @@ SHAMapInnerNode::getString(const SHAMapNodeID& id) const
void
SHAMapInnerNode::setChild(int m, std::shared_ptr<SHAMapTreeNode> const& child)
{
assert((m >= 0) && (m < 16));
assert((m >= 0) && (m < branchFactor));
assert(cowid_ != 0);
assert(child.get() != this);
mHashes[m].zero();
hash_.zero();
auto const dstIsBranch = [&] {
if (child)
return isBranch_ | (1 << m);
else
return isBranch_ & ~(1 << m);
}();
auto const dstToAllocate = popcnt16(dstIsBranch);
// change hashesAndChildren to remove the element, or make room for the
// added element, if necessary
hashesAndChildren_ = TaggedPointer(
std::move(hashesAndChildren_), isBranch_, dstIsBranch, dstToAllocate);
isBranch_ = dstIsBranch;
if (child)
mIsBranch |= (1 << m);
else
mIsBranch &= ~(1 << m);
mChildren[m] = child;
{
auto const childIndex = *getChildIndex(m);
auto [_, hashes, children] = hashesAndChildren_.getHashesAndChildren();
hashes[childIndex].zero();
children[childIndex] = child;
}
hash_.zero();
assert(getBranchCount() <= hashesAndChildren_.capacity());
}
// finished modifying, now make shareable
void
SHAMapInnerNode::shareChild(int m, std::shared_ptr<SHAMapTreeNode> const& child)
{
assert((m >= 0) && (m < 16));
assert((m >= 0) && (m < branchFactor));
assert(cowid_ != 0);
assert(child);
assert(child.get() != this);
mChildren[m] = child;
assert(!isEmptyBranch(m));
hashesAndChildren_.getChildren()[*getChildIndex(m)] = child;
}
SHAMapTreeNode*
SHAMapInnerNode::getChildPointer(int branch)
{
assert(branch >= 0 && branch < 16);
assert(branch >= 0 && branch < branchFactor);
assert(!isEmptyBranch(branch));
std::lock_guard lock(childLock);
return mChildren[branch].get();
return hashesAndChildren_.getChildren()[*getChildIndex(branch)].get();
}
std::shared_ptr<SHAMapTreeNode>
SHAMapInnerNode::getChild(int branch)
{
assert(branch >= 0 && branch < 16);
assert(branch >= 0 && branch < branchFactor);
assert(!isEmptyBranch(branch));
std::lock_guard lock(childLock);
return mChildren[branch];
return hashesAndChildren_.getChildren()[*getChildIndex(branch)];
}
SHAMapHash const&
SHAMapInnerNode::getChildHash(int m) const
{
assert((m >= 0) && (m < branchFactor));
if (auto const i = getChildIndex(m))
return hashesAndChildren_.getHashes()[*i];
return zeroSHAMapHash;
}
std::shared_ptr<SHAMapTreeNode>
@@ -266,20 +362,23 @@ SHAMapInnerNode::canonicalizeChild(
int branch,
std::shared_ptr<SHAMapTreeNode> node)
{
assert(branch >= 0 && branch < 16);
assert(branch >= 0 && branch < branchFactor);
assert(node);
assert(node->getHash() == mHashes[branch]);
assert(!isEmptyBranch(branch));
auto const childIndex = *getChildIndex(branch);
auto [_, hashes, children] = hashesAndChildren_.getHashesAndChildren();
assert(node->getHash() == hashes[childIndex]);
std::lock_guard lock(childLock);
if (mChildren[branch])
if (children[childIndex])
{
// There is already a node hooked up, return it
node = mChildren[branch];
node = children[childIndex];
}
else
{
// Hook this node up
mChildren[branch] = node;
children[childIndex] = node;
}
return node;
}
@@ -288,20 +387,38 @@ void
SHAMapInnerNode::invariants(bool is_root) const
{
unsigned count = 0;
for (int i = 0; i < 16; ++i)
auto [numAllocated, hashes, children] =
hashesAndChildren_.getHashesAndChildren();
if (numAllocated != branchFactor)
{
if (mHashes[i].isNonZero())
auto const branchCount = getBranchCount();
for (int i = 0; i < branchCount; ++i)
{
assert((mIsBranch & (1 << i)) != 0);
if (mChildren[i] != nullptr)
mChildren[i]->invariants();
assert(hashes[i].isNonZero());
if (children[i] != nullptr)
children[i]->invariants();
++count;
}
else
}
else
{
for (int i = 0; i < branchFactor; ++i)
{
assert((mIsBranch & (1 << i)) == 0);
if (hashes[i].isNonZero())
{
assert((isBranch_ & (1 << i)) != 0);
if (children[i] != nullptr)
children[i]->invariants();
++count;
}
else
{
assert((isBranch_ & (1 << i)) == 0);
}
}
}
if (!is_root)
{
assert(hash_.isNonZero());