ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2025-12-06 17:27:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Install SHAMapInnerNodeV2

Browse Source 
* Inner node optimization.
This commit is contained in:
Howard Hinnant
2016-01-27 18:45:29 -05:00
parent e499e908d2
commit 1c3ee48146
19 changed files with 1286 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
src/ripple/app/ledger/Ledger.cpp
View File

@@ -172,9 +172,9 @@ public:
Ledger::Ledger (create_genesis_t, Config const& config, Family& family)
: mImmutable (false)
, txMap_ (std::make_shared <SHAMap> (SHAMapType::TRANSACTION,
family))
family, SHAMap::version{1}))
, stateMap_ (std::make_shared <SHAMap> (SHAMapType::STATE,
family))
family, SHAMap::version{1}))
{
info_.seq = 1;
info_.drops = SYSTEM_CURRENCY_START;
@@ -209,9 +209,9 @@ Ledger::Ledger (uint256 const& parentHash,
beast::Journal j)
: mImmutable (true)
, txMap_ (std::make_shared <SHAMap> (
SHAMapType::TRANSACTION, transHash, family))
, stateMap_ (std::make_shared <SHAMap> (
SHAMapType::STATE, accountHash, family))
SHAMapType::TRANSACTION, transHash, family, SHAMap::version{1}))
, stateMap_ (std::make_shared <SHAMap> (SHAMapType::STATE, accountHash,
family, SHAMap::version{1}))
{
info_.seq = ledgerSeq;
info_.parentCloseTime = parentCloseTime;
@@ -257,7 +257,7 @@ Ledger::Ledger (Ledger const& prevLedger,
NetClock::time_point closeTime)
: mImmutable (false)
, txMap_ (std::make_shared <SHAMap> (SHAMapType::TRANSACTION,
prevLedger.stateMap_->family()))
prevLedger.stateMap_->family(), prevLedger.stateMap_->get_version()))
, stateMap_ (prevLedger.stateMap_->snapShot (true))
, fees_(prevLedger.fees_)
, rules_(prevLedger.rules_)
@@ -288,9 +288,9 @@ Ledger::Ledger (void const* data,
Config const& config, Family& family)
: mImmutable (true)
, txMap_ (std::make_shared <SHAMap> (
SHAMapType::TRANSACTION, family))
SHAMapType::TRANSACTION, family, SHAMap::version{1}))
, stateMap_ (std::make_shared <SHAMap> (
SHAMapType::STATE, family))
SHAMapType::STATE, family, SHAMap::version{1}))
{
SerialIter sit (data, size);
setRaw (sit, hasPrefix, family);
@@ -302,9 +302,9 @@ Ledger::Ledger (std::uint32_t ledgerSeq,
Family& family)
: mImmutable (false)
, txMap_ (std::make_shared <SHAMap> (
SHAMapType::TRANSACTION, family))
SHAMapType::TRANSACTION, family, SHAMap::version{1}))
, stateMap_ (std::make_shared <SHAMap> (
SHAMapType::STATE, family))
SHAMapType::STATE, family, SHAMap::version{1}))
{
info_.seq = ledgerSeq;
info_.closeTime = closeTime;
@@ -862,9 +862,9 @@ void Ledger::setRaw (SerialIter& sit, bool hasPrefix, Family& family)
info_.closeFlags = sit.get8 ();
updateHash ();
txMap_ = std::make_shared<SHAMap> (SHAMapType::TRANSACTION, info_.txHash,
family);
family, SHAMap::version{1});
stateMap_ = std::make_shared<SHAMap> (SHAMapType::STATE, info_.accountHash,
family);
family, SHAMap::version{1});
}
static bool saveValidatedLedger (
@@ -1200,6 +1200,30 @@ Ledger::getNeededAccountStateHashes (
return ret;
}
void
Ledger::make_v2()
{
assert (! mImmutable);
stateMap_ = stateMap_->make_v2();
txMap_ = txMap_->make_v2();
info_.validated = false;
updateHash();
}
void
Ledger::unshare() const
{
stateMap_->unshare();
txMap_->unshare();
}
void
Ledger::invariants() const
{
stateMap_->invariants();
txMap_->invariants();
}
//------------------------------------------------------------------------------
/*

3
src/ripple/app/ledger/Ledger.h
View File

@@ -312,6 +312,9 @@ public:
bool assertSane (beast::Journal ledgerJ);
void make_v2();
void invariants() const;
void unshare() const;
private:
class sles_iter_impl;
class txs_iter_impl;

4
src/ripple/app/ledger/impl/InboundLedgers.cpp
View File

@@ -259,9 +259,11 @@ public:
if (!node.has_nodeid () || !node.has_nodedata ())
return;
auto id_string = node.nodeid();
auto newNode = SHAMapAbstractNode::make(
Blob (node.nodedata().begin(), node.nodedata().end()),
0, snfWIRE, SHAMapHash{uZero}, false, app_.journal ("SHAMapNodeID"));
0, snfWIRE, SHAMapHash{uZero}, false, app_.journal ("SHAMapNodeID"),
SHAMapNodeID(id_string.data(), id_string.size()));
if (!newNode)
return;

2
src/ripple/app/ledger/impl/InboundTransactions.cpp
View File

@@ -81,7 +81,7 @@ public:
{
m_zeroSet.mSet = std::make_shared<SHAMap> (
SHAMapType::TRANSACTION, uint256(),
app_.family());
app_.family(), SHAMap::version{1});
m_zeroSet.mSet->setUnbacked();
}

16
src/ripple/app/ledger/impl/LedgerConsensusImp.cpp
View File

@@ -981,6 +981,12 @@ void LedgerConsensusImp::accept (std::shared_ptr<SHAMap> set)
auto buildLCL = std::make_shared<Ledger>(
*mPreviousLedger,
app_.timeKeeper().closeTime());
auto constexpr v2_ledger_seq_switch = 40'000'000;
if (buildLCL->info().seq > v2_ledger_seq_switch &&
!buildLCL->stateMap().is_v2())
{
buildLCL->make_v2();
}
// Set up to write SHAMap changes to our database,
// perform updates, extract changes
@@ -1016,6 +1022,14 @@ void LedgerConsensusImp::accept (std::shared_ptr<SHAMap> set)
buildLCL->updateSkipList ();
// unshare in case a nodestore load changed the
// version back, otherwise the map is inconsistent
if (buildLCL->info().seq > v2_ledger_seq_switch &&
!buildLCL->stateMap().is_v2())
{
buildLCL->unshare();
}
{
int asf = buildLCL->stateMap().flushDirty (
hotACCOUNT_NODE, buildLCL->info().seq);
@@ -1407,7 +1421,7 @@ void LedgerConsensusImp::takeInitialPosition (
std::shared_ptr<ReadView const> const& initialLedger)
{
std::shared_ptr<SHAMap> initialSet = std::make_shared <SHAMap> (
SHAMapType::TRANSACTION, app_.family());
SHAMapType::TRANSACTION, app_.family(), SHAMap::version{1});
initialSet->setUnbacked ();
// Build SHAMap containing all transactions in our open ledger

2
src/ripple/app/ledger/impl/TransactionAcquire.cpp
View File

@@ -48,7 +48,7 @@ TransactionAcquire::TransactionAcquire (Application& app, uint256 const& hash, c
, j_(app.journal("TransactionAcquire"))
{
mMap = std::make_shared<SHAMap> (SHAMapType::TRANSACTION, hash,
app_.family());
app_.family(), SHAMap::version{1});
mMap->setUnbacked ();
}

165
src/ripple/nodestore/impl/codec.h
View File

@@ -175,7 +175,7 @@ nodeobject_decompress (void const* in,
p, in_size, bf);
break;
}
case 2: // inner node
case 2: // compressed v1 inner node
{
auto const hs =
field<std::uint16_t>::size; // Mask
@@ -218,7 +218,7 @@ nodeobject_decompress (void const* in,
"nodeobject codec: long inner node");
break;
}
case 3: // full inner node
case 3: // full v1 inner node
{
if (in_size != 16 * 32) // hashes
Throw<codec_error> (
@@ -235,6 +235,80 @@ nodeobject_decompress (void const* in,
write(os, is(512), 512);
break;
}
case 5: // compressed v2 inner node
{
auto const hs =
field<std::uint16_t>::size; // Mask size
if (in_size < hs + 65)
Throw<codec_error> (
"nodeobject codec: short inner node");
istream is(p, in_size);
std::uint16_t mask;
read<std::uint16_t>(is, mask); // Mask
in_size -= hs;
std::uint8_t depth;
read<std::uint8_t>(is, depth);
in_size -= 1;
result.second = 525 + 1 + (depth+1)/2;
void* const out = bf(result.second);
result.first = out;
ostream os(out, result.second);
write<std::uint32_t>(os, 0);
write<std::uint32_t>(os, 0);
write<std::uint8_t> (os, hotUNKNOWN);
write<std::uint32_t>(os, HashPrefix::innerNodeV2);
if (mask == 0)
Throw<codec_error> (
"nodeobject codec: empty inner node");
std::uint16_t bit = 0x8000;
for (int i = 16; i--; bit >>= 1)
{
if (mask & bit)
{
if (in_size < 32)
Throw<codec_error> (
"nodeobject codec: short inner node");
std::memcpy(os.data(32), is(32), 32);
in_size -= 32;
}
else
{
std::memset(os.data(32), 0, 32);
}
}
write<std::uint8_t>(os, depth);
if (in_size < (depth+1)/2)
Throw<codec_error> (
"nodeobject codec: short inner node");
std::memcpy(os.data((depth+1)/2), is((depth+1)/2), (depth+1)/2);
in_size -= (depth+1)/2;
if (in_size > 0)
Throw<codec_error> (
"nodeobject codec: long inner node");
break;
}
case 6: // full v2 inner node
{
istream is(p, in_size);
std::uint8_t depth;
read<std::uint8_t>(is, depth);
in_size -= 1;
result.second = 525 + 1 + (depth+1)/2;
if (in_size != 16 * 32 + (depth+1)/2) // hashes and common
Throw<codec_error> (
"nodeobject codec: short full inner node");
void* const out = bf(result.second);
result.first = out;
ostream os(out, result.second);
write<std::uint32_t>(os, 0);
write<std::uint32_t>(os, 0);
write<std::uint8_t> (os, hotUNKNOWN);
write<std::uint32_t>(os, HashPrefix::innerNodeV2);
write(os, is(512), 512);
write<std::uint8_t>(os, depth);
write(os, is((depth+1)/2), (depth+1)/2);
break;
}
default:
Throw<codec_error> (
"nodeobject codec: bad type=" +
@@ -266,7 +340,7 @@ nodeobject_compress (void const* in,
using namespace beast::nudb::detail;
std::size_t type = 1;
// Check for inner node
// Check for inner node v1
if (in_size == 525)
{
istream is(in, in_size);
@@ -300,7 +374,7 @@ nodeobject_compress (void const* in,
std::size_t> result;
if (n < 16)
{
// 2 = inner node compressed
// 2 = v1 inner node compressed
auto const type = 2U;
auto const vs = size_varint(type);
result.second =
@@ -316,7 +390,7 @@ nodeobject_compress (void const* in,
write(os, vh.data(), n * 32);
return result;
}
// 3 = full inner node
// 3 = full v1 inner node
auto const type = 3U;
auto const vs = size_varint(type);
result.second =
@@ -332,6 +406,87 @@ nodeobject_compress (void const* in,
}
}
// Check for inner node v2
if (526 <= in_size && in_size <= 556)
{
istream is(in, in_size);
std::uint32_t index;
std::uint32_t unused;
std::uint8_t kind;
std::uint32_t prefix;
read<std::uint32_t>(is, index);
read<std::uint32_t>(is, unused);
read<std::uint8_t> (is, kind);
read<std::uint32_t>(is, prefix);
if (prefix == HashPrefix::innerNodeV2)
{
std::size_t n = 0;
std::uint16_t mask = 0;
std::array<
std::uint8_t, 512> vh;
for (unsigned bit = 0x8000;
bit; bit >>= 1)
{
void const* const h = is(32);
if (std::memcmp(
h, zero32(), 32) == 0)
continue;
std::memcpy(
vh.data() + 32 * n, h, 32);
mask |= bit;
++n;
}
std::uint8_t depth;
read<std::uint8_t>(is, depth);
std::array<std::uint8_t, 32> common{};
for (unsigned d = 0; d < (depth+1)/2; ++d)
read<std::uint8_t>(is, common[d]);
std::pair<void const*,
std::size_t> result;
if (n < 16)
{
// 5 = v2 inner node compressed
auto const type = 5U;
auto const vs = size_varint(type);
result.second =
vs +
field<std::uint16_t>::size + // mask
n * 32 + // hashes
1 + // depth
(depth+1)/2; // common prefix
std::uint8_t* out = reinterpret_cast<
std::uint8_t*>(bf(result.second));
result.first = out;
ostream os(out, result.second);
write<varint>(os, type);
write<std::uint16_t>(os, mask);
write<std::uint8_t>(os, depth);
write(os, vh.data(), n * 32);
for (unsigned d = 0; d < (depth+1)/2; ++d)
write<std::uint8_t>(os, common[d]);
return result;
}
// 6 = full v2 inner node
auto const type = 6U;
auto const vs = size_varint(type);
result.second =
vs +
n * 32 + // hashes
1 + // depth
(depth+1)/2; // common prefix
std::uint8_t* out = reinterpret_cast<
std::uint8_t*>(bf(result.second));
result.first = out;
ostream os(out, result.second);
write<varint>(os, type);
write<std::uint8_t>(os, depth);
write(os, vh.data(), n * 32);
for (unsigned d = 0; d < (depth+1)/2; ++d)
write<std::uint8_t>(os, common[d]);
return result;
}
}
std::array<std::uint8_t, varint_traits<
std::size_t>::max> vi;
auto const vn = write_varint(

5
src/ripple/protocol/HashPrefix.h
View File

@@ -76,9 +76,12 @@ public:
/** account state */
static HashPrefix const leafNode;
/** inner node in tree */
/** inner node in V1 tree */
static HashPrefix const innerNode;
/** inner node in V2 tree */
static HashPrefix const innerNodeV2;
/** ledger master data for signing */
static HashPrefix const ledgerMaster;

1
src/ripple/protocol/impl/HashPrefix.cpp
View File

@@ -29,6 +29,7 @@ HashPrefix const HashPrefix::transactionID ('T', 'X', 'N');
HashPrefix const HashPrefix::txNode ('S', 'N', 'D');
HashPrefix const HashPrefix::leafNode ('M', 'L', 'N');
HashPrefix const HashPrefix::innerNode ('M', 'I', 'N');
HashPrefix const HashPrefix::innerNodeV2 ('I', 'N', 'R');
HashPrefix const HashPrefix::ledgerMaster ('L', 'W', 'R');
HashPrefix const HashPrefix::txSign ('S', 'T', 'X');
HashPrefix const HashPrefix::txMultiSign ('S', 'M', 'T');

48
src/ripple/shamap/SHAMap.h
View File

@@ -89,6 +89,18 @@ private:
bool backed_ = true; // Map is backed by the database
public:
class version
{
int v_;
public:
explicit version(int v) : v_(v) {}
friend bool operator==(version const& x, version const& y)
{return x.v_ == y.v_;}
friend bool operator!=(version const& x, version const& y)
{return !(x == y);}
};
using DeltaItem = std::pair<std::shared_ptr<SHAMapItem const>,
std::shared_ptr<SHAMapItem const>>;
using Delta = std::map<uint256, DeltaItem>;
@@ -101,13 +113,14 @@ public:
SHAMap (
SHAMapType t,
Family& f,
std::uint32_t seq = 1
version v
);
SHAMap (
SHAMapType t,
uint256 const& hash,
Family& f);
Family& f,
version v);
Family&
family()
@@ -209,8 +222,14 @@ public:
std::function<void (SHAMapHash const&, const Blob&)>) const;
void setUnbacked ();
bool is_v2() const;
version get_version() const;
std::shared_ptr<SHAMap> make_v1() const;
std::shared_ptr<SHAMap> make_v2() const;
int unshare ();
void dump (bool withHashes = false) const;
void invariants() const;
private:
using SharedPtrNodeStack =
@@ -219,7 +238,6 @@ private:
std::shared_ptr<SHAMapItem const> const&>;
void visitDifferences(SHAMap const* have, std::function<bool(SHAMapAbstractNode&)>) const;
int unshare ();
// tree node cache operations
std::shared_ptr<SHAMapAbstractNode> getCache (SHAMapHash const& hash) const;
@@ -263,7 +281,7 @@ private:
std::shared_ptr<SHAMapAbstractNode> node) const;
SHAMapTreeNode* firstBelow (std::shared_ptr<SHAMapAbstractNode>,
SharedPtrNodeStack& stack) const;
SharedPtrNodeStack& stack, int branch = 0) const;
// Simple descent
// Get a child of the specified node
@@ -291,8 +309,8 @@ private:
bool hasInnerNode (SHAMapNodeID const& nodeID, SHAMapHash const& hash) const;
bool hasLeafNode (uint256 const& tag, SHAMapHash const& hash) const;
SHAMapItem const* peekFirstItem(SharedPtrNodeStack& stack) const;
SHAMapItem const* peekNextItem(uint256 const& id, SharedPtrNodeStack& stack) const;
SHAMapTreeNode const* peekFirstItem(SharedPtrNodeStack& stack) const;
SHAMapTreeNode const* peekNextItem(uint256 const& id, SharedPtrNodeStack& stack) const;
bool walkBranch (SHAMapAbstractNode* node,
std::shared_ptr<SHAMapItem const> const& otherMapItem,
bool isFirstMap, Delta & differences, int & maxCount) const;
@@ -349,6 +367,13 @@ SHAMap::setUnbacked ()
backed_ = false;
}
inline
bool
SHAMap::is_v2() const
{
return std::dynamic_pointer_cast<SHAMapInnerNodeV2>(root_) != nullptr;
}
//------------------------------------------------------------------------------
class SHAMap::const_iterator
@@ -386,8 +411,11 @@ private:
inline
SHAMap::const_iterator::const_iterator(SHAMap const* map)
: map_(map)
, item_(map_->peekFirstItem(stack_))
, item_(nullptr)
{
auto temp = map_->peekFirstItem(stack_);
if (temp)
item_ = temp->peekItem().get();
}
inline
@@ -424,7 +452,11 @@ inline
SHAMap::const_iterator&
SHAMap::const_iterator::operator++()
{
item_ = map_->peekNextItem(item_->key(), stack_);
auto temp = map_->peekNextItem(item_->key(), stack_);
if (temp)
item_ = temp->peekItem().get();
else
item_ = nullptr;
return *this;
}

4
src/ripple/shamap/SHAMapNodeID.h
View File

@@ -38,6 +38,7 @@ private:
public:
SHAMapNodeID ();
SHAMapNodeID (int depth, uint256 const& hash);
SHAMapNodeID (void const* ptr, int len);
bool isValid () const;
@@ -64,10 +65,9 @@ public:
SHAMapNodeID getChildNodeID (int m) const;
int selectBranch (uint256 const& hash) const;
int getDepth () const;
bool has_common_prefix(SHAMapNodeID const& other) const;
private:
SHAMapNodeID (int depth, uint256 const& hash);
static uint256 const& Masks (int depth);
friend std::ostream& operator<< (std::ostream& out, SHAMapNodeID const& node);

78
src/ripple/shamap/SHAMapTreeNode.h
View File

@@ -128,10 +128,13 @@ public:
virtual void addRaw (Serializer&, SHANodeFormat format) const = 0;
virtual std::string getString (SHAMapNodeID const&) const;
virtual std::shared_ptr<SHAMapAbstractNode> clone(std::uint32_t seq) const = 0;
virtual uint256 const& key() const = 0;
virtual void invariants(bool is_v2, bool is_root = false) const = 0;
static std::shared_ptr<SHAMapAbstractNode>
make(Blob const& rawNode, std::uint32_t seq, SHANodeFormat format,
SHAMapHash const& hash, bool hashValid, beast::Journal j);
SHAMapHash const& hash, bool hashValid, beast::Journal j,
SHAMapNodeID const& id = SHAMapNodeID{});
// debugging
#ifdef BEAST_DEBUG
@@ -139,6 +142,8 @@ public:
#endif
};
class SHAMapInnerNodeV2;
class SHAMapInnerNode
: public SHAMapAbstractNode
{
@@ -149,7 +154,7 @@ class SHAMapInnerNode
static std::mutex childLock;
public:
SHAMapInnerNode(std::uint32_t seq = 0);
SHAMapInnerNode(std::uint32_t seq);
std::shared_ptr<SHAMapAbstractNode> clone(std::uint32_t seq) const override;
bool isEmpty () const;
@@ -172,11 +177,47 @@ public:
void updateHashDeep();
void addRaw (Serializer&, SHANodeFormat format) const override;
std::string getString (SHAMapNodeID const&) const override;
uint256 const& key() const override;
void invariants(bool is_v2, bool is_root = false) const override;
friend std::shared_ptr<SHAMapAbstractNode>
SHAMapAbstractNode::make(Blob const& rawNode, std::uint32_t seq,
SHANodeFormat format, SHAMapHash const& hash, bool hashValid,
beast::Journal j);
beast::Journal j, SHAMapNodeID const& id);
friend class SHAMapInnerNodeV2;
};
class SHAMapTreeNode;
// SHAMapInnerNodeV2 is a "version 2" inner node. It always has at least two children
// unless it is the root node and there is only one leaf in the tree, in which case
// that leaf is a direct child of the root.
class SHAMapInnerNodeV2
: public SHAMapInnerNode
{
uint256 common_ = {};
int depth_ = 64;
public:
explicit SHAMapInnerNodeV2(std::uint32_t seq);
SHAMapInnerNodeV2(std::uint32_t seq, int depth);
std::shared_ptr<SHAMapAbstractNode> clone(std::uint32_t seq) const override;
uint256 const& common() const;
int depth() const;
bool has_common_prefix(uint256 const& key) const;
int get_common_prefix(uint256 const& key) const;
void set_common(int depth, uint256 const& key);
void addRaw(Serializer& s, SHANodeFormat format) const override;
uint256 const& key() const override;
void setChildren(std::shared_ptr<SHAMapTreeNode> const& child1,
std::shared_ptr<SHAMapTreeNode> const& child2);
void invariants(bool is_v2, bool is_root = false) const override;
friend std::shared_ptr<SHAMapAbstractNode>
SHAMapAbstractNode::make(Blob const& rawNode, std::uint32_t seq,
SHANodeFormat format, SHAMapHash const& hash, bool hashValid,
beast::Journal j, SHAMapNodeID const& id);
};
// SHAMapTreeNode represents a leaf, and may eventually be renamed to reflect that.
@@ -197,6 +238,8 @@ public:
std::shared_ptr<SHAMapAbstractNode> clone(std::uint32_t seq) const override;
void addRaw (Serializer&, SHANodeFormat format) const override;
uint256 const& key() const override;
void invariants(bool is_v2, bool is_root = false) const override;
public: // public only to SHAMap
@@ -325,6 +368,35 @@ SHAMapInnerNode::setFullBelowGen (std::uint32_t gen)
mFullBelowGen = gen;
}
// SHAMapInnerNodeV2
inline
SHAMapInnerNodeV2::SHAMapInnerNodeV2(std::uint32_t seq)
: SHAMapInnerNode(seq)
{
}
inline
SHAMapInnerNodeV2::SHAMapInnerNodeV2(std::uint32_t seq, int depth)
: SHAMapInnerNode(seq)
, depth_(depth)
{
}
inline
uint256 const&
SHAMapInnerNodeV2::common() const
{
return common_;
}
inline
int
SHAMapInnerNodeV2::depth() const
{
return depth_;
}
// SHAMapTreeNode
inline

487
src/ripple/shamap/impl/SHAMap.cpp
View File

@@ -27,29 +27,34 @@ namespace ripple {
SHAMap::SHAMap (
SHAMapType t,
Family& f,
std::uint32_t seq)
version v)
: f_ (f)
, journal_(f.journal())
, seq_ (seq)
, seq_ (1)
, state_ (SHAMapState::Modifying)
, type_ (t)
{
assert (seq_ != 0);
root_ = std::make_shared<SHAMapInnerNode> (seq_);
if (v == version{2})
root_ = std::make_shared<SHAMapInnerNodeV2>(seq_, 0);
else
root_ = std::make_shared<SHAMapInnerNode>(seq_);
}
SHAMap::SHAMap (
SHAMapType t,
uint256 const& hash,
Family& f)
Family& f,
version v)
: f_ (f)
, journal_(f.journal())
, seq_ (1)
, state_ (SHAMapState::Synching)
, type_ (t)
{
root_ = std::make_shared<SHAMapInnerNode> (seq_);
if (v == version{2})
root_ = std::make_shared<SHAMapInnerNodeV2>(seq_, 0);
else
root_ = std::make_shared<SHAMapInnerNode>(seq_);
}
SHAMap::~SHAMap ()
@@ -60,7 +65,7 @@ SHAMap::~SHAMap ()
std::shared_ptr<SHAMap>
SHAMap::snapShot (bool isMutable) const
{
auto ret = std::make_shared<SHAMap> (type_, f_);
auto ret = std::make_shared<SHAMap> (type_, f_, get_version());
SHAMap& newMap = *ret;
if (!isMutable)
@@ -78,6 +83,72 @@ SHAMap::snapShot (bool isMutable) const
return ret;
}
std::shared_ptr<SHAMap>
SHAMap::make_v2() const
{
assert(!is_v2());
auto ret = std::make_shared<SHAMap>(type_, f_, version{2});
ret->seq_ = seq_ + 1;
SharedPtrNodeStack stack;
for (auto leaf = peekFirstItem(stack); leaf != nullptr;
leaf = peekNextItem(leaf->peekItem()->key(), stack))
{
auto node_type = leaf->getType();
ret->addGiveItem(leaf->peekItem(),
node_type != SHAMapTreeNode::tnACCOUNT_STATE,
node_type == SHAMapTreeNode::tnTRANSACTION_MD);
}
NodeObjectType t;
switch (type_)
{
case SHAMapType::TRANSACTION:
t = hotTRANSACTION_NODE;
break;
case SHAMapType::STATE:
t = hotACCOUNT_NODE;
break;
default:
t = hotUNKNOWN;
break;
}
ret->unshare();
ret->flushDirty(t, ret->seq_);
return ret;
}
std::shared_ptr<SHAMap>
SHAMap::make_v1() const
{
assert(is_v2());
auto ret = std::make_shared<SHAMap>(type_, f_, version{1});
ret->seq_ = seq_ + 1;
SharedPtrNodeStack stack;
for (auto leaf = peekFirstItem(stack); leaf != nullptr;
leaf = peekNextItem(leaf->peekItem()->key(), stack))
{
auto node_type = leaf->getType();
ret->addGiveItem(leaf->peekItem(),
node_type != SHAMapTreeNode::tnACCOUNT_STATE,
node_type == SHAMapTreeNode::tnTRANSACTION_MD);
}
NodeObjectType t;
switch (type_)
{
case SHAMapType::TRANSACTION:
t = hotTRANSACTION_NODE;
break;
case SHAMapType::STATE:
t = hotACCOUNT_NODE;
break;
default:
t = hotUNKNOWN;
break;
}
ret->unshare();
ret->flushDirty(t, ret->seq_);
return ret;
}
void
SHAMap::dirtyUp (SharedPtrNodeStack& stack,
uint256 const& target, std::shared_ptr<SHAMapAbstractNode> child)
@@ -103,10 +174,6 @@ SHAMap::dirtyUp (SharedPtrNodeStack& stack,
node = unshareNode(std::move(node), nodeID);
node->setChild (branch, child);
#ifdef ST_DEBUG
JLOG(journal_.trace()) <<
"dirtyUp sets branch " << branch << " to " << prevHash;
#endif
child = std::move (node);
}
}
@@ -119,16 +186,39 @@ SHAMap::walkTowardsKey(uint256 const& id, SharedPtrNodeStack* stack) const
SHAMapNodeID nodeID;
if (stack != nullptr)
stack->push({inNode, nodeID});
auto const isv2 = is_v2();
while (inNode->isInner())
{
auto const branch = nodeID.selectBranch (id);
auto const inner = std::static_pointer_cast<SHAMapInnerNode>(std::move(inNode));
if (isv2)
{
auto n = std::static_pointer_cast<SHAMapInnerNodeV2>(inNode);
if (!n->has_common_prefix(id))
return nullptr;
}
auto const branch = nodeID.selectBranch (id);
if (inner->isEmptyBranch (branch))
return nullptr;
inNode = descendThrow (inner, branch);
nodeID = nodeID.getChildNodeID (branch);
if (isv2)
{
if (inNode->isInner())
{
auto n = std::dynamic_pointer_cast<SHAMapInnerNodeV2>(inNode);
assert(n);
nodeID = SHAMapNodeID{n->depth(), n->common()};
}
else
{
nodeID = SHAMapNodeID{64, inNode->key()};
}
}
else
{
nodeID = nodeID.getChildNodeID (branch);
}
if (stack != nullptr)
stack->push({inNode, nodeID});
}
@@ -158,6 +248,26 @@ SHAMap::fetchNodeFromDB (SHAMapHash const& hash) const
{
node = SHAMapAbstractNode::make(obj->getData(),
0, snfPREFIX, hash, true, f_.journal());
if (node && node->isInner())
{
bool isv2 = std::dynamic_pointer_cast<SHAMapInnerNodeV2>(node) != nullptr;
if (isv2 != is_v2())
{
auto root = std::dynamic_pointer_cast<SHAMapInnerNode>(root_);
assert(root);
assert(root->isEmpty());
if (isv2)
{
auto temp = make_v2();
swap(temp->root_, const_cast<std::shared_ptr<SHAMapAbstractNode>&>(root_));
}
else
{
auto temp = make_v1();
swap(temp->root_, const_cast<std::shared_ptr<SHAMapAbstractNode>&>(root_));
}
}
}
if (node)
canonicalize (hash, node);
}
@@ -189,6 +299,26 @@ SHAMap::checkFilter(SHAMapHash const& hash,
{
node = SHAMapAbstractNode::make(
nodeData, 0, snfPREFIX, hash, true, f_.journal ());
if (node && node->isInner())
{
bool isv2 = std::dynamic_pointer_cast<SHAMapInnerNodeV2>(node) != nullptr;
if (isv2 != is_v2())
{
auto root = std::dynamic_pointer_cast<SHAMapInnerNode>(root_);
assert(root);
assert(root->isEmpty());
if (isv2)
{
auto temp = make_v2();
swap(temp->root_, const_cast<std::shared_ptr<SHAMapAbstractNode>&>(root_));
}
else
{
auto temp = make_v1();
swap(temp->root_, const_cast<std::shared_ptr<SHAMapAbstractNode>&>(root_));
}
}
}
if (node)
{
filter->gotNode (true, hash,
@@ -316,7 +446,6 @@ SHAMap::descend (SHAMapInnerNode * parent, SHAMapNodeID const& parentID,
assert ((branch >= 0) && (branch < 16));
assert (!parent->isEmptyBranch (branch));
SHAMapNodeID childID = parentID.getChildNodeID (branch);
SHAMapAbstractNode* child = parent->getChildPointer (branch);
auto const& childHash = parent->getChildHash (branch);
@@ -331,7 +460,17 @@ SHAMap::descend (SHAMapInnerNode * parent, SHAMapNodeID const& parentID,
}
}
return std::make_pair (child, childID);
if (child && is_v2())
{
if (child->isInner())
{
auto n = static_cast<SHAMapInnerNodeV2*>(child);
return std::make_pair(child, SHAMapNodeID{n->depth(), n->key()});
}
return std::make_pair(child, SHAMapNodeID{64, child->key()});
}
return std::make_pair (child, parentID.getChildNodeID (branch));
}
SHAMapAbstractNode*
@@ -363,9 +502,17 @@ SHAMap::descendAsync (SHAMapInnerNode* parent, int branch,
if (!obj)
return nullptr;
ptr = SHAMapAbstractNode::make(
obj->getData(), 0, snfPREFIX, hash, true, f_.journal ());
ptr = SHAMapAbstractNode::make(obj->getData(), 0, snfPREFIX,
hash, true, f_.journal());
if (ptr && ptr->isInner())
{
bool isv2 = std::dynamic_pointer_cast<SHAMapInnerNodeV2>(ptr) != nullptr;
if (isv2 != is_v2())
{
assert(false);
}
}
if (ptr && backed_)
canonicalize (hash, ptr);
}
@@ -398,22 +545,52 @@ SHAMap::unshareNode (std::shared_ptr<Node> node, SHAMapNodeID const& nodeID)
SHAMapTreeNode*
SHAMap::firstBelow(std::shared_ptr<SHAMapAbstractNode> node,
SharedPtrNodeStack& stack) const
SharedPtrNodeStack& stack, int branch) const
{
// Return the first item at or below this node
if (node->isLeaf())
return static_cast<SHAMapTreeNode*>(node.get());
{
auto n = std::static_pointer_cast<SHAMapTreeNode>(node);
stack.push({node, {64, n->peekItem()->key()}});
return n.get();
}
auto inner = std::static_pointer_cast<SHAMapInnerNode>(node);
if (stack.empty())
stack.push({inner, SHAMapNodeID{}});
else
{
if (is_v2())
{
auto inner2 = std::static_pointer_cast<SHAMapInnerNodeV2>(inner);
stack.push({inner2, {inner2->depth(), inner2->common()}});
}
else
{
stack.push({inner, stack.top().second.getChildNodeID(branch)});
}
}
for (int i = 0; i < 16;)
{
if (!inner->isEmptyBranch(i))
{
node = descendThrow(inner, i);
assert(!stack.empty());
stack.push({node, stack.top().second.getChildNodeID(i)});
if (node->isLeaf())
return static_cast<SHAMapTreeNode*>(node.get());
{
auto n = std::static_pointer_cast<SHAMapTreeNode>(node);
stack.push({n, {64, n->peekItem()->key()}});
return n.get();
}
inner = std::static_pointer_cast<SHAMapInnerNode>(node);
if (is_v2())
{
auto inner2 = std::static_pointer_cast<SHAMapInnerNodeV2>(inner);
stack.push({inner2, {inner2->depth(), inner2->common()}});
}
else
{
stack.push({inner, stack.top().second.getChildNodeID(branch)});
}
i = 0; // scan all 16 branches of this new node
}
else
@@ -464,11 +641,10 @@ SHAMap::onlyBelow (SHAMapAbstractNode* node) const
static std::shared_ptr<
SHAMapItem const> const nullConstSHAMapItem;
SHAMapItem const*
SHAMapTreeNode const*
SHAMap::peekFirstItem(SharedPtrNodeStack& stack) const
{
assert(stack.empty());
stack.push({root_, SHAMapNodeID{}});
SHAMapTreeNode* node = firstBelow(root_, stack);
if (!node)
{
@@ -476,10 +652,10 @@ SHAMap::peekFirstItem(SharedPtrNodeStack& stack) const
stack.pop();
return nullptr;
}
return node->peekItem().get();
return node;
}
SHAMapItem const*
SHAMapTreeNode const*
SHAMap::peekNextItem(uint256 const& id, SharedPtrNodeStack& stack) const
{
assert(!stack.empty());
@@ -496,13 +672,11 @@ SHAMap::peekNextItem(uint256 const& id, SharedPtrNodeStack& stack) const
if (!inner->isEmptyBranch(i))
{
node = descendThrow(inner, i);
nodeID = nodeID.getChildNodeID(i);
stack.push({node, nodeID});
auto leaf = firstBelow(node, stack);
auto leaf = firstBelow(node, stack, i);
if (!leaf)
Throw<SHAMapMissingNode> (type_, id);
assert(leaf->isLeaf());
return leaf->peekItem().get();
return leaf;
}
}
stack.pop();
@@ -555,6 +729,7 @@ SHAMap::upper_bound(uint256 const& id) const
walkTowardsKey(id, &stack);
std::shared_ptr<SHAMapAbstractNode> node;
SHAMapNodeID nodeID;
auto const isv2 = is_v2();
while (!stack.empty())
{
std::tie(node, nodeID) = stack.top();
@@ -567,14 +742,27 @@ SHAMap::upper_bound(uint256 const& id) const
else
{
auto inner = std::static_pointer_cast<SHAMapInnerNode>(node);
for (auto branch = nodeID.selectBranch(id) + 1; branch < 16; ++branch)
int branch;
if (isv2)
{
auto n = std::static_pointer_cast<SHAMapInnerNodeV2>(inner);
if (n->has_common_prefix(id))
branch = nodeID.selectBranch(id) + 1;
else if (id < n->common())
branch = 0;
else
branch = 16;
}
else
{
branch = nodeID.selectBranch(id) + 1;
}
for (; branch < 16; ++branch)
{
if (!inner->isEmptyBranch(branch))
{
node = descendThrow(inner, branch);
nodeID = nodeID.getChildNodeID(branch);
stack.push({node, nodeID});
auto leaf = firstBelow(node, stack);
auto leaf = firstBelow(node, stack, branch);
if (!leaf)
Throw<SHAMapMissingNode> (type_, id);
return const_iterator(this, leaf->peekItem().get(),
@@ -615,61 +803,76 @@ bool SHAMap::delItem (uint256 const& id)
// What gets attached to the end of the chain
// (For now, nothing, since we deleted the leaf)
SHAMapHash prevHash;
std::shared_ptr<SHAMapAbstractNode> prevNode;
while (!stack.empty ())
{
auto node = std::dynamic_pointer_cast<SHAMapInnerNode>(stack.top ().first);
SHAMapNodeID nodeID = stack.top ().second;
stack.pop ();
assert (node);
auto node = std::static_pointer_cast<SHAMapInnerNode>(stack.top().first);
SHAMapNodeID nodeID = stack.top().second;
stack.pop();
node = unshareNode(std::move(node), nodeID);
node->setChild (nodeID.selectBranch (id), prevNode);
node->setChild(nodeID.selectBranch(id), prevNode);
if (!nodeID.isRoot ())
{
// we may have made this a node with 1 or 0 children
// And, if so, we need to remove this branch
int bc = node->getBranchCount ();
if (bc == 0)
int bc = node->getBranchCount();
if (is_v2())
{
// no children below this branch
prevHash.zero();
prevNode.reset ();
}
else if (bc == 1)
{
// If there's only one item, pull up on the thread
std::shared_ptr<SHAMapItem const> item = onlyBelow (node.get ());
if (item)
assert(bc != 0);
if (bc == 1)
{
for (int i = 0; i < 16; ++i)
{
if (!node->isEmptyBranch (i))
{
node->setChild (i, nullptr);
prevNode = descendThrow(node, i);
break;
}
}
prevNode = std::make_shared<SHAMapTreeNode>(item, type, node->getSeq());
prevHash = prevNode->getNodeHash();
}
else
else // bc >= 2
{
prevHash = node->getNodeHash ();
prevNode = std::move (node);
// This node is now the end of the branch
prevNode = std::move(node);
}
}
else
{
// This node is now the end of the branch
prevHash = node->getNodeHash ();
prevNode = std::move (node);
if (bc == 0)
{
// no children below this branch
prevNode.reset ();
}
else if (bc == 1)
{
// If there's only one item, pull up on the thread
auto item = onlyBelow (node.get ());
if (item)
{
for (int i = 0; i < 16; ++i)
{
if (!node->isEmptyBranch (i))
{
node->setChild (i, nullptr);
break;
}
}
prevNode = std::make_shared<SHAMapTreeNode>(item, type, node->getSeq());
}
else
{
prevNode = std::move (node);
}
}
else
{
// This node is now the end of the branch
prevNode = std::move (node);
}
}
}
}
@@ -677,6 +880,20 @@ bool SHAMap::delItem (uint256 const& id)
return true;
}
static
uint256
prefix(unsigned depth, uint256 const& key)
{
uint256 r{};
auto x = r.begin();
auto y = key.begin();
for (auto i = 0; i < depth/2; ++i, ++x, ++y)
*x = *y;
if (depth & 1)
*x = *y & 0xF0;
return r;
}
bool
SHAMap::addGiveItem (std::shared_ptr<SHAMapItem const> const& item,
bool isTransaction, bool hasMeta)
@@ -705,48 +922,97 @@ SHAMap::addGiveItem (std::shared_ptr<SHAMapItem const> const& item,
return false;
}
node = unshareNode(std::move(node), nodeID);
if (node->isInner ())
if (is_v2())
{
// easy case, we end on an inner node
auto inner = std::static_pointer_cast<SHAMapInnerNode>(node);
int branch = nodeID.selectBranch (tag);
assert (inner->isEmptyBranch (branch));
auto newNode = std::make_shared<SHAMapTreeNode> (item, type, seq_);
inner->setChild (branch, newNode);
}
else
{
// this is a leaf node that has to be made an inner node holding two items
auto leaf = std::static_pointer_cast<SHAMapTreeNode>(node);
std::shared_ptr<SHAMapItem const> otherItem = leaf->peekItem ();
assert (otherItem && (tag != otherItem->key()));
node = std::make_shared<SHAMapInnerNode>(node->getSeq());
int b1, b2;
while ((b1 = nodeID.selectBranch (tag)) ==
(b2 = nodeID.selectBranch (otherItem->key())))
if (node->isInner())
{
stack.push ({node, nodeID});
// we need a new inner node, since both go on same branch at this level
nodeID = nodeID.getChildNodeID (b1);
node = std::make_shared<SHAMapInnerNode> (seq_);
auto inner = std::static_pointer_cast<SHAMapInnerNodeV2>(node);
if (inner->has_common_prefix(tag))
{
int branch = nodeID.selectBranch(tag);
assert(inner->isEmptyBranch(branch));
auto newNode = std::make_shared<SHAMapTreeNode>(item, type, seq_);
inner->setChild(branch, newNode);
}
else
{
assert(!stack.empty());
auto parent = unshareNode(
std::static_pointer_cast<SHAMapInnerNodeV2>(stack.top().first),
stack.top().second);
stack.top().first = parent;
auto parent_depth = parent->depth();
auto depth = inner->get_common_prefix(tag);
auto new_inner = std::make_shared<SHAMapInnerNodeV2>(seq_);
auto nodeID = SHAMapNodeID{depth, prefix(depth, inner->common())};
new_inner->setChild(nodeID.selectBranch(inner->common()), inner);
nodeID = SHAMapNodeID{depth, prefix(depth, tag)};
new_inner->setChild(nodeID.selectBranch(tag),
std::make_shared<SHAMapTreeNode>(item, type, seq_));
new_inner->set_common(depth, prefix(depth, tag));
nodeID = SHAMapNodeID{parent_depth, prefix(parent_depth, tag)};
parent->setChild(nodeID.selectBranch(tag), new_inner);
node = new_inner;
}
}
else
{
auto leaf = std::static_pointer_cast<SHAMapTreeNode>(node);
auto inner = std::make_shared<SHAMapInnerNodeV2>(seq_);
inner->setChildren(leaf, std::make_shared<SHAMapTreeNode>(item, type, seq_));
assert(!stack.empty());
auto parent = unshareNode(
std::static_pointer_cast<SHAMapInnerNodeV2>(stack.top().first),
stack.top().second);
stack.top().first = parent;
node = inner;
}
}
else // !is_v2()
{
if (node->isInner ())
{
// easy case, we end on an inner node
auto inner = std::static_pointer_cast<SHAMapInnerNode>(node);
int branch = nodeID.selectBranch (tag);
assert (inner->isEmptyBranch (branch));
auto newNode = std::make_shared<SHAMapTreeNode> (item, type, seq_);
inner->setChild (branch, newNode);
}
else
{
// this is a leaf node that has to be made an inner node holding two items
auto leaf = std::static_pointer_cast<SHAMapTreeNode>(node);
std::shared_ptr<SHAMapItem const> otherItem = leaf->peekItem ();
assert (otherItem && (tag != otherItem->key()));
// we can add the two leaf nodes here
assert (node->isInner ());
node = std::make_shared<SHAMapInnerNode>(node->getSeq());
std::shared_ptr<SHAMapTreeNode> newNode =
std::make_shared<SHAMapTreeNode> (item, type, seq_);
assert (newNode->isValid () && newNode->isLeaf ());
auto inner = std::static_pointer_cast<SHAMapInnerNode>(node);
inner->setChild (b1, newNode);
int b1, b2;
newNode = std::make_shared<SHAMapTreeNode> (otherItem, type, seq_);
assert (newNode->isValid () && newNode->isLeaf ());
inner->setChild (b2, newNode);
while ((b1 = nodeID.selectBranch (tag)) ==
(b2 = nodeID.selectBranch (otherItem->key())))
{
stack.push ({node, nodeID});
// we need a new inner node, since both go on same branch at this level
nodeID = nodeID.getChildNodeID (b1);
node = std::make_shared<SHAMapInnerNode> (seq_);
}
// we can add the two leaf nodes here
assert (node->isInner ());
std::shared_ptr<SHAMapTreeNode> newNode =
std::make_shared<SHAMapTreeNode> (item, type, seq_);
assert (newNode->isValid () && newNode->isLeaf ());
auto inner = std::static_pointer_cast<SHAMapInnerNode>(node);
inner->setChild (b1, newNode);
newNode = std::make_shared<SHAMapTreeNode> (otherItem, type, seq_);
assert (newNode->isValid () && newNode->isLeaf ());
inner->setChild (b2, newNode);
}
}
dirtyUp (stack, tag, node);
@@ -931,7 +1197,10 @@ SHAMap::walkSubTree (bool doWrite, NodeObjectType t, std::uint32_t seq)
if (node->isEmpty ())
{ // replace empty root with a new empty root
root_ = std::make_shared <SHAMapInnerNode> (0);
if (is_v2())
root_ = std::make_shared<SHAMapInnerNodeV2>(0, 0);
else
root_ = std::make_shared<SHAMapInnerNode>(0);
return 1;
}
@@ -1089,4 +1358,22 @@ SHAMap::canonicalize(SHAMapHash const& hash, std::shared_ptr<SHAMapAbstractNode>
f_.treecache().canonicalize (hash.as_uint256(), node);
}
SHAMap::version
SHAMap::get_version() const
{
if (is_v2())
return version{2};
return version{1};
}
void
SHAMap::invariants() const
{
(void)getHash(); // update node hashes
auto node = root_.get();
assert(node != nullptr);
assert(!node->isLeaf());
node->invariants(is_v2(), true);
}
} // ripple

21
src/ripple/shamap/impl/SHAMapNodeID.cpp
View File

@@ -102,7 +102,7 @@ std::string SHAMapNodeID::getRawString () const
SHAMapNodeID SHAMapNodeID::getChildNodeID (int m) const
{
assert ((m >= 0) && (m < 16));
assert (mDepth <= 64);
assert (mDepth < 64);
uint256 child (mNodeID);
child.begin ()[mDepth / 2] |= (mDepth & 1) ? m : (m << 4);
@@ -143,6 +143,25 @@ int SHAMapNodeID::selectBranch (uint256 const& hash) const
return branch;
}
bool
SHAMapNodeID::has_common_prefix(SHAMapNodeID const& other) const
{
assert(mDepth <= other.mDepth);
auto x = mNodeID.begin();
auto y = other.mNodeID.begin();
for (unsigned i = 0; i < mDepth/2; ++i, ++x, ++y)
{
if (*x != *y)
return false;
}
if (mDepth & 1)
{
auto i = mDepth/2;
return (*(mNodeID.begin() + i) & 0xF0) == (*(other.mNodeID.begin() + i) & 0xF0);
}
return true;
}
void SHAMapNodeID::dump (beast::Journal journal) const
{
JLOG(journal.debug()) <<

75
src/ripple/shamap/impl/SHAMapSync.cpp
View File

@@ -25,27 +25,17 @@
namespace ripple {
// VFALCO TODO tidy up this global
static const uint256 uZero;
static bool visitLeavesHelper (
std::function <void (std::shared_ptr<SHAMapItem const> const&)> const& function,
SHAMapAbstractNode& node)
{
// Adapt visitNodes to visitLeaves
if (!node.isInner ())
function (static_cast<SHAMapTreeNode&>(node).peekItem ());
return false;
}
void
SHAMap::visitLeaves(
std::function<void(std::shared_ptr<SHAMapItem const> const& item)> const& leafFunction) const
{
visitNodes (std::bind (visitLeavesHelper,
std::cref (leafFunction), std::placeholders::_1));
visitNodes(
[&leafFunction](SHAMapAbstractNode& node)
{
if (!node.isInner())
leafFunction(static_cast<SHAMapTreeNode&>(node).peekItem());
return false;
});
}
void SHAMap::visitNodes(std::function<bool (SHAMapAbstractNode&)> const& function) const
@@ -216,7 +206,10 @@ SHAMap::getMissingNodes(std::size_t max, SHAMapSyncFilter* filter)
// Switch to processing the child node
node = static_cast<SHAMapInnerNode*>(d);
nodeID = childID;
if (auto v2Node = dynamic_cast<SHAMapInnerNodeV2*>(node))
nodeID = SHAMapNodeID{v2Node->depth(), v2Node->key()};
else
nodeID = childID;
firstChild = rand_int(255);
currentChild = 0;
fullBelow = true;
@@ -340,13 +333,20 @@ bool SHAMap::getNodeFat (SHAMapNodeID wanted,
return false;
node = descendThrow(inner, branch);
nodeID = nodeID.getChildNodeID (branch);
if (auto v2Node = dynamic_cast<SHAMapInnerNodeV2*>(node))
nodeID = SHAMapNodeID{v2Node->depth(), v2Node->key()};
else
nodeID = nodeID.getChildNodeID (branch);
}
if (!node || (nodeID != wanted))
if (node == nullptr ||
(dynamic_cast<SHAMapInnerNodeV2*>(node) != nullptr &&
!wanted.has_common_prefix(nodeID)) ||
(dynamic_cast<SHAMapInnerNodeV2*>(node) == nullptr && wanted != nodeID))
{
JLOG(journal_.warn()) <<
"peer requested node that is not in the map: " << wanted;
JLOG(journal_.warn())
<< "peer requested node that is not in the map:\n"
<< wanted << " but found\n" << nodeID;
return false;
}
@@ -383,8 +383,12 @@ bool SHAMap::getNodeFat (SHAMapNodeID wanted,
{
if (! inner->isEmptyBranch (i))
{
auto childID = nodeID.getChildNodeID (i);
auto childNode = descendThrow (inner, i);
SHAMapNodeID childID;
if (auto v2Node = dynamic_cast<SHAMapInnerNodeV2*>(childNode))
childID = SHAMapNodeID{v2Node->depth(), v2Node->key()};
else
childID = nodeID.getChildNodeID (i);
if (childNode->isInner () &&
((depth > 1) || (bc == 1)))
@@ -430,7 +434,7 @@ SHAMapAddNode SHAMap::addRootNode (SHAMapHash const& hash, Blob const& rootNode,
assert (seq_ >= 1);
auto node = SHAMapAbstractNode::make(
rootNode, 0, format, SHAMapHash{uZero}, false, f_.journal ());
rootNode, 0, format, SHAMapHash{}, false, f_.journal ());
if (!node || !node->isValid() || node->getNodeHash () != hash)
return SHAMapAddNode::invalid ();
@@ -467,6 +471,8 @@ SHAMap::addKnownNode (const SHAMapNodeID& node, Blob const& rawNode,
}
std::uint32_t generation = f_.fullbelow().getGeneration();
auto newNode = SHAMapAbstractNode::make(rawNode, 0, snfWIRE,
SHAMapHash{}, false, f_.journal(), node);
SHAMapNodeID iNodeID;
auto iNode = root_.get();
@@ -490,9 +496,10 @@ SHAMap::addKnownNode (const SHAMapNodeID& node, Blob const& rawNode,
auto prevNode = inner;
std::tie(iNode, iNodeID) = descend(inner, iNodeID, branch, filter);
if (!iNode)
if (iNode == nullptr)
{
if (iNodeID != node)
if ((std::dynamic_pointer_cast<SHAMapInnerNodeV2>(newNode) && !iNodeID.has_common_prefix(node)) ||
(!std::dynamic_pointer_cast<SHAMapInnerNodeV2>(newNode) && iNodeID != node))
{
// Either this node is broken or we didn't request it (yet)
JLOG(journal_.warn()) << "unable to hook node " << node;
@@ -503,9 +510,6 @@ SHAMap::addKnownNode (const SHAMapNodeID& node, Blob const& rawNode,
return SHAMapAddNode::invalid ();
}
auto newNode = SHAMapAbstractNode::make(
rawNode, 0, snfWIRE, SHAMapHash{uZero}, false, f_.journal ());
if (!newNode || !newNode->isValid() || childHash != newNode->getNodeHash ())
{
JLOG(journal_.warn()) << "Corrupt node received";
@@ -519,9 +523,15 @@ SHAMap::addKnownNode (const SHAMapNodeID& node, Blob const& rawNode,
return SHAMapAddNode::useful ();
}
#ifndef NDEBUG
if (newNode && newNode->isInner())
{
bool isv2 = std::dynamic_pointer_cast<SHAMapInnerNodeV2>(newNode) != nullptr;
assert(isv2 == is_v2());
}
#endif
if (backed_)
canonicalize (childHash, newNode);
newNode = prevNode->canonicalizeChild (branch, std::move(newNode));
if (filter)
@@ -674,6 +684,11 @@ There's no point in including the leaves of transaction trees.
void SHAMap::getFetchPack (SHAMap const* have, bool includeLeaves, int max,
std::function<void (SHAMapHash const&, const Blob&)> func) const
{
if (have != nullptr && have->is_v2() != is_v2())
{
JLOG(journal_.info()) << "Can not get fetch pack when versions are different.";
return;
}
visitDifferences (have,
[includeLeaves, &max, &func] (SHAMapAbstractNode& smn) -> bool
{

315
src/ripple/shamap/impl/SHAMapTreeNode.cpp
View File

@@ -46,7 +46,31 @@ SHAMapInnerNode::clone(std::uint32_t seq) const
std::memcpy(p->mHashes, mHashes, sizeof(mHashes));
std::unique_lock <std::mutex> lock(childLock);
for (int i = 0; i < 16; ++i)
{
p->mChildren[i] = mChildren[i];
assert(std::dynamic_pointer_cast<SHAMapInnerNodeV2>(p->mChildren[i]) == nullptr);
}
return std::move(p);
}
std::shared_ptr<SHAMapAbstractNode>
SHAMapInnerNodeV2::clone(std::uint32_t seq) const
{
auto p = std::make_shared<SHAMapInnerNodeV2>(seq);
p->mHash = mHash;
p->mIsBranch = mIsBranch;
p->mFullBelowGen = mFullBelowGen;
std::memcpy(p->mHashes, mHashes, sizeof(mHashes));
p->common_ = common_;
p->depth_ = depth_;
std::unique_lock <std::mutex> lock(childLock);
for (int i = 0; i < 16; ++i)
{
p->mChildren[i] = mChildren[i];
if (p->mChildren[i] != nullptr)
assert(std::dynamic_pointer_cast<SHAMapInnerNodeV2>(p->mChildren[i]) != nullptr ||
std::dynamic_pointer_cast<SHAMapTreeNode>(p->mChildren[i]) != nullptr);
}
return std::move(p);
}
@@ -75,7 +99,8 @@ SHAMapTreeNode::SHAMapTreeNode (std::shared_ptr<SHAMapItem const> const& item,
std::shared_ptr<SHAMapAbstractNode>
SHAMapAbstractNode::make(Blob const& rawNode, std::uint32_t seq, SHANodeFormat format,
SHAMapHash const& hash, bool hashValid, beast::Journal j)
SHAMapHash const& hash, bool hashValid, beast::Journal j,
SHAMapNodeID const& id)
{
if (format == snfWIRE)
{
@@ -86,9 +111,8 @@ SHAMapAbstractNode::make(Blob const& rawNode, std::uint32_t seq, SHANodeFormat f
int type = rawNode.back ();
int len = s.getLength ();
if ((type < 0) || (type > 4))
if ((type < 0) || (type > 6))
return {};
if (type == 0)
{
// transaction
@@ -176,6 +200,49 @@ SHAMapAbstractNode::make(Blob const& rawNode, std::uint32_t seq, SHANodeFormat f
return std::make_shared<SHAMapTreeNode>(item, tnTRANSACTION_MD, seq, hash);
return std::make_shared<SHAMapTreeNode>(item, tnTRANSACTION_MD, seq);
}
else if (type == 5)
{
// full inner
if (len != 512)
Throw<std::runtime_error> ("invalid FI node");
auto ret = std::make_shared<SHAMapInnerNodeV2>(seq);
for (int i = 0; i < 16; ++i)
{
s.get256 (ret->mHashes[i].as_uint256(), i * 32);
if (ret->mHashes[i].isNonZero ())
ret->mIsBranch |= (1 << i);
}
ret->set_common(id.getDepth(), id.getNodeID());
if (hashValid)
ret->mHash = hash;
else
ret->updateHash();
return ret;
}
else if (type == 6)
{
auto ret = std::make_shared<SHAMapInnerNodeV2>(seq);
// compressed inner
for (int i = 0; i < (len / 33); ++i)
{
int pos;
if (! s.get8 (pos, 32 + (i * 33)))
Throw<std::runtime_error> ("short CI node");
if ((pos < 0) || (pos >= 16))
Throw<std::runtime_error> ("invalid CI node");
s.get256 (ret->mHashes[pos].as_uint256(), i * 33);
if (ret->mHashes[pos].isNonZero ())
ret->mIsBranch |= (1 << pos);
}
ret->set_common(id.getDepth(), id.getNodeID());
if (hashValid)
ret->mHash = hash;
else
ret->updateHash();
return ret;
}
}
else if (format == snfPREFIX)
@@ -224,11 +291,20 @@ SHAMapAbstractNode::make(Blob const& rawNode, std::uint32_t seq, SHANodeFormat f
return std::make_shared<SHAMapTreeNode>(item, tnACCOUNT_STATE, seq, hash);
return std::make_shared<SHAMapTreeNode>(item, tnACCOUNT_STATE, seq);
}
else if (prefix == HashPrefix::innerNode)
else if ((prefix == HashPrefix::innerNode) || (prefix == HashPrefix::innerNodeV2))
{
if (s.getLength () != 512)
auto len = s.getLength();
bool isV2 = (prefix == HashPrefix::innerNodeV2);
if ((len < 512) || (!isV2 && (len != 512)))
Throw<std::runtime_error> ("invalid PIN node");
auto ret = std::make_shared<SHAMapInnerNode>(seq);
std::shared_ptr<SHAMapInnerNode> ret;
if (isV2)
ret = std::make_shared<SHAMapInnerNodeV2>(seq);
else
ret = std::make_shared<SHAMapInnerNode>(seq);
for (int i = 0; i < 16; ++i)
{
s.get256 (ret->mHashes[i].as_uint256(), i * 32);
@@ -236,6 +312,22 @@ SHAMapAbstractNode::make(Blob const& rawNode, std::uint32_t seq, SHANodeFormat f
if (ret->mHashes[i].isNonZero ())
ret->mIsBranch |= (1 << i);
}
if (isV2)
{
auto temp = std::static_pointer_cast<SHAMapInnerNodeV2>(ret);
s.get8(temp->depth_, 512);
auto n = (temp->depth_ + 1) / 2;
if (len != 512 + 1 + n)
Throw<std::runtime_error> ("invalid PIN node");
auto x = temp->common_.begin();
for (auto i = 0; i < n; ++i, ++x)
{
int byte;
s.get8(byte, 512+1+i);
*x = byte;
}
}
if (hashValid)
ret->mHash = hash;
else
@@ -349,7 +441,7 @@ SHAMapInnerNode::addRaw(Serializer& s, SHANodeFormat format) const
for (int i = 0; i < 16; ++i)
s.add256 (mHashes[i].as_uint256());
}
else
else // format == snfWIRE
{
if (getBranchCount () < 12)
{
@@ -376,6 +468,33 @@ SHAMapInnerNode::addRaw(Serializer& s, SHANodeFormat format) const
assert (false);
}
void
SHAMapInnerNodeV2::addRaw(Serializer& s, SHANodeFormat format) const
{
if (format == snfPREFIX)
{
s.add32 (HashPrefix::innerNodeV2);
for (int i = 0 ; i < 16; ++i)
s.add256 (mHashes[i].as_uint256());
s.add8(depth_);
auto x = common_.begin();
for (auto i = 0; i < (depth_+1)/2; ++i, ++x)
s.add8(*x);
}
else
{
SHAMapInnerNode::addRaw(s, format);
if (format == snfWIRE)
{
auto& data = s.modData();
data.back() += 3;
}
}
}
void
SHAMapTreeNode::addRaw(Serializer& s, SHANodeFormat format) const
{
@@ -593,5 +712,187 @@ SHAMapInnerNode::canonicalizeChild(int branch, std::shared_ptr<SHAMapAbstractNod
return node;
}
bool
SHAMapInnerNodeV2::has_common_prefix(uint256 const& key) const
{
auto x = common_.begin();
auto y = key.begin();
for (unsigned i = 0; i < depth_/2; ++i, ++x, ++y)
{
if (*x != *y)
return false;
}
if (depth_ & 1)
{
auto i = depth_/2;
return (*(common_.begin() + i) & 0xF0) == (*(key.begin() + i) & 0xF0);
}
return true;
}
int
SHAMapInnerNodeV2::get_common_prefix(uint256 const& key) const
{
auto x = common_.begin();
auto y = key.begin();
auto r = 0;
for (unsigned i = 0; i < depth_/2; ++i, ++x, ++y, r += 2)
{
if (*x != *y)
{
if ((*x & 0xF0) == (*y & 0xF0))
++r;
return r;
}
}
if (depth_ & 1)
{
auto i = depth_/2;
if ((*(common_.begin() + i) & 0xF0) == (*(key.begin() + i) & 0xF0))
++r;
}
return r;
}
void
SHAMapInnerNodeV2::setChildren(std::shared_ptr<SHAMapTreeNode> const& child1,
std::shared_ptr<SHAMapTreeNode> const& child2)
{
assert(child1->peekItem()->key() != child2->peekItem()->key());
auto k1 = child1->peekItem()->key().begin();
auto k2 = child2->peekItem()->key().begin();
auto k = common_.begin();
for (depth_ = 0; *k1 == *k2; ++depth_, ++k1, ++k2, ++k)
*k = *k1;
unsigned b1;
unsigned b2;
if ((*k1 & 0xF0) == (*k2 & 0xF0))
{
*k = *k1 & 0xF0;
b1 = *k1 & 0x0F;
b2 = *k2 & 0x0F;
depth_ = 2*depth_ + 1;
}
else
{
b1 = *k1 >> 4;
b2 = *k2 >> 4;
depth_ = 2*depth_;
}
mChildren[b1] = child1;
mIsBranch |= 1 << b1;
mChildren[b2] = child2;
mIsBranch |= 1 << b2;
}
void
SHAMapInnerNodeV2::set_common(int depth, uint256 const& common)
{
depth_ = depth;
common_ = common;
}
uint256 const&
SHAMapInnerNode::key() const
{
Throw<std::logic_error>("SHAMapInnerNode::key() should never be called");
static uint256 x;
return x;
}
uint256 const&
SHAMapInnerNodeV2::key() const
{
return common_;
}
uint256 const&
SHAMapTreeNode::key() const
{
return mItem->key();
}
void
SHAMapInnerNode::invariants(bool is_v2, bool is_root) const
{
assert(!is_v2);
assert(mType == tnINNER);
unsigned count = 0;
for (int i = 0; i < 16; ++i)
{
if (mHashes[i].isNonZero())
{
assert((mIsBranch & (1 << i)) != 0);
if (mChildren[i] != nullptr)
mChildren[i]->invariants(is_v2);
++count;
}
else
{
assert((mIsBranch & (1 << i)) == 0);
}
}
if (!is_root)
{
assert(mHash.isNonZero());
assert(count >= 1);
}
assert((count == 0) ? mHash.isZero() : mHash.isNonZero());
}
void
SHAMapInnerNodeV2::invariants(bool is_v2, bool is_root) const
{
assert(is_v2);
assert(mType == tnINNER);
unsigned count = 0;
for (int i = 0; i < 16; ++i)
{
if (mHashes[i].isNonZero())
{
assert((mIsBranch & (1 << i)) != 0);
if (mChildren[i] != nullptr)
{
assert(mHashes[i] == mChildren[i]->getNodeHash());
#ifndef NDEBUG
auto const& childID = mChildren[i]->key();
// Make sure this child it attached to the correct branch
SHAMapNodeID nodeID {depth(), common()};
assert (i == nodeID.selectBranch(childID));
#endif
assert(has_common_prefix(childID));
mChildren[i]->invariants(is_v2);
}
++count;
}
else
{
assert((mIsBranch & (1 << i)) == 0);
}
}
if (!is_root)
{
assert(mHash.isNonZero());
assert(count >= 2);
assert(depth_ > 0);
}
else
{
assert(depth_ == 0);
}
if (count == 0)
assert(mHash.isZero());
else
assert(mHash.isNonZero());
}
void
SHAMapTreeNode::invariants(bool, bool) const
{
assert(mType >= tnTRANSACTION_NM);
assert(mHash.isNonZero());
assert(mItem != nullptr);
}
} // ripple

2
src/ripple/shamap/tests/FetchPack.test.cpp
View File

@@ -121,7 +121,7 @@ public:
beast::Journal const j; // debug journal
TestFamily f(j);
std::shared_ptr <Table> t1 (std::make_shared <Table> (
SHAMapType::FREE, f));
SHAMapType::FREE, f, SHAMap::version{2}));
pass ();

159
src/ripple/shamap/tests/SHAMap.test.cpp
View File

@@ -28,6 +28,76 @@
namespace ripple {
namespace tests {
static_assert( std::is_nothrow_destructible <SHAMap>{}, "");
static_assert(!std::is_default_constructible<SHAMap>{}, "");
static_assert(!std::is_copy_constructible <SHAMap>{}, "");
static_assert(!std::is_copy_assignable <SHAMap>{}, "");
static_assert(!std::is_move_constructible <SHAMap>{}, "");
static_assert(!std::is_move_assignable <SHAMap>{}, "");
static_assert( std::is_nothrow_destructible <SHAMap::version>{}, "");
static_assert(!std::is_default_constructible <SHAMap::version>{}, "");
static_assert( std::is_trivially_copy_constructible<SHAMap::version>{}, "");
static_assert( std::is_trivially_copy_assignable <SHAMap::version>{}, "");
static_assert( std::is_trivially_move_constructible<SHAMap::version>{}, "");
static_assert( std::is_trivially_move_assignable <SHAMap::version>{}, "");
static_assert( std::is_nothrow_destructible <SHAMap::const_iterator>{}, "");
static_assert( std::is_default_constructible<SHAMap::const_iterator>{}, "");
static_assert( std::is_copy_constructible <SHAMap::const_iterator>{}, "");
static_assert( std::is_copy_assignable <SHAMap::const_iterator>{}, "");
static_assert( std::is_move_constructible <SHAMap::const_iterator>{}, "");
static_assert( std::is_move_assignable <SHAMap::const_iterator>{}, "");
static_assert( std::is_nothrow_destructible <SHAMapItem>{}, "");
static_assert(!std::is_default_constructible<SHAMapItem>{}, "");
static_assert( std::is_copy_constructible <SHAMapItem>{}, "");
static_assert( std::is_copy_assignable <SHAMapItem>{}, "");
static_assert( std::is_move_constructible <SHAMapItem>{}, "");
static_assert( std::is_move_assignable <SHAMapItem>{}, "");
static_assert( std::is_nothrow_destructible <SHAMapNodeID>{}, "");
static_assert( std::is_default_constructible<SHAMapNodeID>{}, "");
static_assert( std::is_copy_constructible <SHAMapNodeID>{}, "");
static_assert( std::is_copy_assignable <SHAMapNodeID>{}, "");
static_assert( std::is_move_constructible <SHAMapNodeID>{}, "");
static_assert( std::is_move_assignable <SHAMapNodeID>{}, "");
static_assert( std::is_nothrow_destructible <SHAMapHash>{}, "");
static_assert( std::is_default_constructible<SHAMapHash>{}, "");
static_assert( std::is_copy_constructible <SHAMapHash>{}, "");
static_assert( std::is_copy_assignable <SHAMapHash>{}, "");
static_assert( std::is_move_constructible <SHAMapHash>{}, "");
static_assert( std::is_move_assignable <SHAMapHash>{}, "");
static_assert(!std::is_nothrow_destructible <SHAMapAbstractNode>{}, "");
static_assert(!std::is_default_constructible<SHAMapAbstractNode>{}, "");
static_assert(!std::is_copy_constructible <SHAMapAbstractNode>{}, "");
static_assert(!std::is_copy_assignable <SHAMapAbstractNode>{}, "");
static_assert(!std::is_move_constructible <SHAMapAbstractNode>{}, "");
static_assert(!std::is_move_assignable <SHAMapAbstractNode>{}, "");
static_assert( std::is_nothrow_destructible <SHAMapInnerNode>{}, "");
static_assert(!std::is_default_constructible<SHAMapInnerNode>{}, "");
static_assert(!std::is_copy_constructible <SHAMapInnerNode>{}, "");
static_assert(!std::is_copy_assignable <SHAMapInnerNode>{}, "");
static_assert(!std::is_move_constructible <SHAMapInnerNode>{}, "");
static_assert(!std::is_move_assignable <SHAMapInnerNode>{}, "");
static_assert( std::is_nothrow_destructible <SHAMapInnerNodeV2>{}, "");
static_assert(!std::is_default_constructible<SHAMapInnerNodeV2>{}, "");
static_assert(!std::is_copy_constructible <SHAMapInnerNodeV2>{}, "");
static_assert(!std::is_copy_assignable <SHAMapInnerNodeV2>{}, "");
static_assert(!std::is_move_constructible <SHAMapInnerNodeV2>{}, "");
static_assert(!std::is_move_assignable <SHAMapInnerNodeV2>{}, "");
static_assert( std::is_nothrow_destructible <SHAMapTreeNode>{}, "");
static_assert(!std::is_default_constructible<SHAMapTreeNode>{}, "");
static_assert(!std::is_copy_constructible <SHAMapTreeNode>{}, "");
static_assert(!std::is_copy_assignable <SHAMapTreeNode>{}, "");
static_assert(!std::is_move_constructible <SHAMapTreeNode>{}, "");
static_assert(!std::is_move_assignable <SHAMapTreeNode>{}, "");
inline bool operator== (SHAMapItem const& a, SHAMapItem const& b) { return a.key() == b.key(); }
inline bool operator!= (SHAMapItem const& a, SHAMapItem const& b) { return a.key() != b.key(); }
inline bool operator== (SHAMapItem const& a, uint256 const& b) { return a.key() == b; }
@@ -48,11 +118,13 @@ public:
void run ()
{
run (true);
run (false);
run (true, SHAMap::version{1});
run (false, SHAMap::version{1});
run (true, SHAMap::version{2});
run (false, SHAMap::version{2});
}
void run (bool backed)
void run (bool backed, SHAMap::version v)
{
if (backed)
testcase ("add/traverse backed");
@@ -70,13 +142,16 @@ public:
h4.SetHex ("b92891fe4ef6cee585fdc6fda2e09eb4d386363158ec3321b8123e5a772c6ca8");
h5.SetHex ("a92891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7");
SHAMap sMap (SHAMapType::FREE, f);
SHAMap sMap (SHAMapType::FREE, f, v);
sMap.invariants();
if (! backed)
sMap.setUnbacked ();
SHAMapItem i1 (h1, IntToVUC (1)), i2 (h2, IntToVUC (2)), i3 (h3, IntToVUC (3)), i4 (h4, IntToVUC (4)), i5 (h5, IntToVUC (5));
unexpected (!sMap.addItem (SHAMapItem{i2}, true, false), "no add");
sMap.invariants();
unexpected (!sMap.addItem (SHAMapItem{i1}, true, false), "no add");
sMap.invariants();
auto i = sMap.begin();
auto e = sMap.end();
@@ -86,8 +161,11 @@ public:
++i;
unexpected (i != e, "bad traverse");
sMap.addItem (SHAMapItem{i4}, true, false);
sMap.invariants();
sMap.delItem (i2.key());
sMap.invariants();
sMap.addItem (SHAMapItem{i3}, true, false);
sMap.invariants();
i = sMap.begin();
e = sMap.end();
unexpected (i == e || (*i != i1), "bad traverse");
@@ -105,13 +183,29 @@ public:
SHAMapHash mapHash = sMap.getHash ();
std::shared_ptr<SHAMap> map2 = sMap.snapShot (false);
map2->invariants();
unexpected (sMap.getHash () != mapHash, "bad snapshot");
unexpected (map2->getHash () != mapHash, "bad snapshot");
SHAMap::Delta delta;
expect(sMap.compare(*map2, delta, 100), "There should be no differences");
expect(delta.empty(), "The delta should be empty");
unexpected (!sMap.delItem (sMap.begin()->key()), "bad mod");
sMap.invariants();
unexpected (sMap.getHash () == mapHash, "bad snapshot");
unexpected (map2->getHash () != mapHash, "bad snapshot");
expect(sMap.compare(*map2, delta, 100), "There should be 1 difference");
expect(delta.size() == 1, "The delta should be size 1");
expect(delta.begin()->first == h1, "Should be the first key");
expect(delta.begin()->second.first == nullptr, "Must be null");
expect(delta.begin()->second.second->key() == h1, "The difference is the first key");
sMap.dump();
auto const is_v2 = sMap.is_v2();
if (backed)
testcase ("build/tear backed");
else
@@ -128,16 +222,30 @@ public:
keys[7].SetHex ("292891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
std::vector<uint256> hashes(8);
hashes[0].SetHex ("B7387CFEA0465759ADC718E8C42B52D2309D179B326E239EB5075C64B6281F7F");
hashes[1].SetHex ("FBC195A9592A54AB44010274163CB6BA95F497EC5BA0A8831845467FB2ECE266");
hashes[2].SetHex ("4E7D2684B65DFD48937FFB775E20175C43AF0C94066F7D5679F51AE756795B75");
hashes[3].SetHex ("7A2F312EB203695FFD164E038E281839EEF06A1B99BFC263F3CECC6C74F93E07");
hashes[4].SetHex ("395A6691A372387A703FB0F2C6D2C405DAF307D0817F8F0E207596462B0E3A3E");
hashes[5].SetHex ("D044C0A696DE3169CC70AE216A1564D69DE96582865796142CE7D98A84D9DDE4");
hashes[6].SetHex ("76DCC77C4027309B5A91AD164083264D70B77B5E43E08AEDA5EBF94361143615");
hashes[7].SetHex ("DF4220E93ADC6F5569063A01B4DC79F8DB9553B6A3222ADE23DEA02BBE7230E5");
if (is_v2)
{
hashes[0].SetHex ("B7387CFEA0465759ADC718E8C42B52D2309D179B326E239EB5075C64B6281F7F");
hashes[1].SetHex ("6A70885D21024F9F4F50E688B365D0E017266F53AE3E77B52AAEF84E167FE942");
hashes[2].SetHex ("BA635322BDF510CCFC0578C194C1F87DA2D97C1A55A469F6E7A463BE963663D7");
hashes[3].SetHex ("5F751361AC7A161DED4D6EAAC4B587C7C01E50E1F9EC3DD61207BD6B196E7DB1");
hashes[4].SetHex ("FC1C57DD4BF15E37961E0B3064C43E60A9DC26EA332D7A6178FE2284901DB49F");
hashes[5].SetHex ("4FCDFE944E8E19E35FF60E7BDA7DB21B1CD99670BCF158FEF8F0F8B49BF5C9AD");
hashes[6].SetHex ("31F6DE8152FBE77EAD59805631FCDDB71F1BC6A5E9BD8FA3948D82D1CE1F93D6");
hashes[7].SetHex ("00895AD3B161D483C4EF7B5469B0D305685222B5102C2C3F614FD84AD50C4B14");
}
else
{
hashes[0].SetHex ("B7387CFEA0465759ADC718E8C42B52D2309D179B326E239EB5075C64B6281F7F");
hashes[1].SetHex ("FBC195A9592A54AB44010274163CB6BA95F497EC5BA0A8831845467FB2ECE266");
hashes[2].SetHex ("4E7D2684B65DFD48937FFB775E20175C43AF0C94066F7D5679F51AE756795B75");
hashes[3].SetHex ("7A2F312EB203695FFD164E038E281839EEF06A1B99BFC263F3CECC6C74F93E07");
hashes[4].SetHex ("395A6691A372387A703FB0F2C6D2C405DAF307D0817F8F0E207596462B0E3A3E");
hashes[5].SetHex ("D044C0A696DE3169CC70AE216A1564D69DE96582865796142CE7D98A84D9DDE4");
hashes[6].SetHex ("76DCC77C4027309B5A91AD164083264D70B77B5E43E08AEDA5EBF94361143615");
hashes[7].SetHex ("DF4220E93ADC6F5569063A01B4DC79F8DB9553B6A3222ADE23DEA02BBE7230E5");
}
SHAMap map (SHAMapType::FREE, f);
SHAMap map (SHAMapType::FREE, f, v);
if (! backed)
map.setUnbacked ();
@@ -147,11 +255,31 @@ public:
SHAMapItem item (keys[i], IntToVUC (i));
expect (map.addItem (std::move(item), true, false), "unable to add item");
expect (map.getHash().as_uint256() == hashes[i], "bad buildup map hash");
map.invariants();
}
if (v == SHAMap::version{1})
{
expect(!map.is_v2(), "map should be version 1");
auto map_v2 = map.make_v2();
expect(map_v2 != nullptr, "make_v2 should never return nullptr");
expect(map_v2->is_v2(), "map should be version 2");
map_v2->invariants();
auto i1 = map.begin();
auto e1 = map.end();
auto i2 = map_v2->begin();
auto e2 = map_v2->end();
for (; i1 != e1; ++i1, ++i2)
{
expect(i2 != e2, "make_v2 size mismatch");
expect(*i1 == *i2, "make_v2, item mismatch");
}
expect(i2 == e2, "make_v2 size mismatch");
}
for (int i = keys.size() - 1; i >= 0; --i)
{
expect (map.getHash().as_uint256() == hashes[i], "bad teardown hash");
expect (map.delItem (keys[i]), "unable to remove item");
map.invariants();
}
expect (map.getHash() == zero, "bad final empty map hash");
}
@@ -173,11 +301,14 @@ public:
keys[7].SetHex ("292891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
tests::TestFamily f{beast::Journal{}};
SHAMap map{SHAMapType::FREE, f};
SHAMap map{SHAMapType::FREE, f, v};
if (! backed)
map.setUnbacked ();
for (auto const& k : keys)
{
map.addItem(SHAMapItem{k, IntToVUC(0)}, true, false);
map.invariants();
}
int i = 7;
for (auto const& k : map)

42
src/ripple/shamap/tests/SHAMapSync.test.cpp
View File

@@ -80,21 +80,54 @@ public:
return true;
}
void run ()
void run()
{
std::cerr << "Run, version 1" << std::endl;
run(SHAMap::version{1});
std::cerr << "Run, version 2" << std::endl;
run(SHAMap::version{2});
}
void run(SHAMap::version v)
{
beast::Journal const j; // debug journal
TestFamily f(j);
SHAMap source (SHAMapType::FREE, f);
SHAMap destination (SHAMapType::FREE, f);
SHAMap source (SHAMapType::FREE, f, v);
SHAMap destination (SHAMapType::FREE, f, v);
int items = 10000;
for (int i = 0; i < items; ++i)
{
source.addItem (std::move(*makeRandomAS ()), false, false);
if (i % 100 == 0)
source.invariants();
}
source.invariants();
expect (confuseMap (source, 500), "ConfuseMap");
source.invariants();
source.setImmutable ();
int count = 0;
source.visitLeaves([&count](auto const& item)
{
++count;
});
expect(count == items, "These must be equal");
std::vector<SHAMapMissingNode> missingNodes;
source.walkMap(missingNodes, 2048);
expect(missingNodes.empty(), "should be empty");
std::vector<SHAMapNodeID> nodeIDs, gotNodeIDs;
std::vector< Blob > gotNodes;
std::vector<uint256> hashes;
std::vector<SHAMapNodeID>::iterator nodeIDIterator;
std::vector< Blob >::iterator rawNodeIterator;
destination.setSynching ();
{
@@ -158,6 +191,9 @@ public:
destination.clearSynching ();
expect (source.deepCompare (destination), "Deep Compare");
std::cerr << "Checking destination invariants" << std::endl;
destination.invariants();
}
};