ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2025-11-30 16:05:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor and improve the SHAMap code:

Browse Source 
This commit combines a number of cleanups, targeting both the
code structure and the code logic. Large changes include:

 - Using more strongly-typed classes for SHAMap nodes, instead of relying
   on runtime-time detection of class types. This change saves 16 bytes
   of memory per node.
 - Improving the interface of SHAMap::addGiveItem and SHAMap::addItem to
   avoid the need for passing two bool arguments.
 - Documenting the "copy-on-write" semantics that SHAMap uses to
   efficiently track changes in individual nodes.
 - Removing unused code and simplifying several APIs.
 - Improving function naming.
This commit is contained in:
Nik Bougalis
2020-11-13 23:30:43 -08:00
parent 5def79e93c
commit 1bb294afbc
49 changed files with 1641 additions and 1378 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Builds/CMake/RippledCore.cmake
View File

@@ -635,7 +635,9 @@ target_sources (rippled PRIVATE
src/ripple/shamap/impl/NodeFamily.cpp
src/ripple/shamap/impl/SHAMap.cpp
src/ripple/shamap/impl/SHAMapDelta.cpp
src/ripple/shamap/impl/SHAMapInnerNode.cpp
src/ripple/shamap/impl/SHAMapItem.cpp
src/ripple/shamap/impl/SHAMapLeafNode.cpp
src/ripple/shamap/impl/SHAMapNodeID.cpp
src/ripple/shamap/impl/SHAMapSync.cpp
src/ripple/shamap/impl/SHAMapTreeNode.cpp

5
src/ripple/app/consensus/RCLConsensus.cpp
View File

@@ -315,9 +315,8 @@ RCLConsensus::Adaptor::onClose(
Serializer s(2048);
tx.first->add(s);
initialSet->addItem(
SHAMapItem(tx.first->getTransactionID(), std::move(s)),
true,
false);
SHAMapNodeType::tnTRANSACTION_NM,
SHAMapItem(tx.first->getTransactionID(), std::move(s)));
}
// Add pseudo-transactions to the set

3
src/ripple/app/consensus/RCLCxTx.h
View File

@@ -91,7 +91,8 @@ public:
insert(Tx const& t)
{
return map_->addItem(
SHAMapItem{t.id(), t.tx_.peekData()}, true, false);
SHAMapNodeType::tnTRANSACTION_NM,
SHAMapItem{t.id(), t.tx_.peekData()});
}
/** Remove a transaction from the set.

2
src/ripple/app/ledger/AccountStateSF.cpp
View File

@@ -27,7 +27,7 @@ AccountStateSF::gotNode(
SHAMapHash const& nodeHash,
std::uint32_t ledgerSeq,
Blob&& nodeData,
SHAMapTreeNode::TNType) const
SHAMapNodeType) const
{
db_.store(
hotACCOUNT_NODE, std::move(nodeData), nodeHash.as_uint256(), ledgerSeq);

2
src/ripple/app/ledger/AccountStateSF.h
View File

@@ -42,7 +42,7 @@ public:
SHAMapHash const& nodeHash,
std::uint32_t ledgerSeq,
Blob&& nodeData,
SHAMapTreeNode::TNType type) const override;
SHAMapNodeType type) const override;
boost::optional<Blob>
getNode(SHAMapHash const& nodeHash) const override;

4
src/ripple/app/ledger/ConsensusTransSetSF.cpp
View File

@@ -41,14 +41,14 @@ ConsensusTransSetSF::gotNode(
SHAMapHash const& nodeHash,
std::uint32_t,
Blob&& nodeData,
SHAMapTreeNode::TNType type) const
SHAMapNodeType type) const
{
if (fromFilter)
return;
m_nodeCache.insert(nodeHash, nodeData);
if ((type == SHAMapTreeNode::tnTRANSACTION_NM) && (nodeData.size() > 16))
if ((type == SHAMapNodeType::tnTRANSACTION_NM) && (nodeData.size() > 16))
{
// this is a transaction, and we didn't have it
JLOG(j_.debug())

2
src/ripple/app/ledger/ConsensusTransSetSF.h
View File

@@ -45,7 +45,7 @@ public:
SHAMapHash const& nodeHash,
std::uint32_t ledgerSeq,
Blob&& nodeData,
SHAMapTreeNode::TNType type) const override;
SHAMapNodeType type) const override;
boost::optional<Blob>
getNode(SHAMapHash const& nodeHash) const override;

20
src/ripple/app/ledger/Ledger.cpp
View File

@@ -202,7 +202,7 @@ Ledger::Ledger(
rawInsert(sle);
}
stateMap_->flushDirty(hotACCOUNT_NODE, info_.seq);
stateMap_->flushDirty(hotACCOUNT_NODE);
setImmutable(config);
}
@@ -354,7 +354,7 @@ bool
Ledger::addSLE(SLE const& sle)
{
SHAMapItem item(sle.key(), sle.getSerializer());
return stateMap_->addItem(std::move(item), false, false);
return stateMap_->addItem(SHAMapNodeType::tnACCOUNT_STATE, std::move(item));
}
//------------------------------------------------------------------------------
@@ -499,8 +499,9 @@ Ledger::rawInsert(std::shared_ptr<SLE> const& sle)
{
Serializer ss;
sle->add(ss);
auto item = std::make_shared<SHAMapItem const>(sle->key(), std::move(ss));
if (!stateMap_->addGiveItem(std::move(item), false, false))
if (!stateMap_->addGiveItem(
SHAMapNodeType::tnACCOUNT_STATE,
std::make_shared<SHAMapItem const>(sle->key(), std::move(ss))))
LogicError("Ledger::rawInsert: key already exists");
}
@@ -509,9 +510,9 @@ Ledger::rawReplace(std::shared_ptr<SLE> const& sle)
{
Serializer ss;
sle->add(ss);
auto item = std::make_shared<SHAMapItem const>(sle->key(), std::move(ss));
if (!stateMap_->updateGiveItem(std::move(item), false, false))
if (!stateMap_->updateGiveItem(
SHAMapNodeType::tnACCOUNT_STATE,
std::make_shared<SHAMapItem const>(sle->key(), std::move(ss))))
LogicError("Ledger::rawReplace: key not found");
}
@@ -527,8 +528,9 @@ Ledger::rawTxInsert(
Serializer s(txn->getDataLength() + metaData->getDataLength() + 16);
s.addVL(txn->peekData());
s.addVL(metaData->peekData());
auto item = std::make_shared<SHAMapItem const>(key, std::move(s));
if (!txMap().addGiveItem(std::move(item), true, true))
if (!txMap().addGiveItem(
SHAMapNodeType::tnTRANSACTION_MD,
std::make_shared<SHAMapItem const>(key, std::move(s))))
LogicError("duplicate_tx: " + to_string(key));
}

2
src/ripple/app/ledger/TransactionMaster.h
View File

@@ -69,7 +69,7 @@ public:
std::shared_ptr<STTx const>
fetch(
std::shared_ptr<SHAMapItem> const& item,
SHAMapTreeNode::TNType type,
SHAMapNodeType type,
std::uint32_t uCommitLedger);
// return value: true = we had the transaction already

4
src/ripple/app/ledger/TransactionStateSF.cpp
View File

@@ -27,10 +27,10 @@ TransactionStateSF::gotNode(
SHAMapHash const& nodeHash,
std::uint32_t ledgerSeq,
Blob&& nodeData,
SHAMapTreeNode::TNType type) const
SHAMapNodeType type) const
{
assert(type != SHAMapTreeNode::tnTRANSACTION_NM);
assert(type != SHAMapNodeType::tnTRANSACTION_NM);
db_.store(
hotTRANSACTION_NODE,
std::move(nodeData),

2
src/ripple/app/ledger/TransactionStateSF.h
View File

@@ -42,7 +42,7 @@ public:
SHAMapHash const& nodeHash,
std::uint32_t ledgerSeq,
Blob&& nodeData,
SHAMapTreeNode::TNType type) const override;
SHAMapNodeType type) const override;
boost::optional<Blob>
getNode(SHAMapHash const& nodeHash) const override;

6
src/ripple/app/ledger/impl/BuildLedger.cpp
View File

@@ -67,10 +67,8 @@ buildLedgerImpl(
// Write the final version of all modified SHAMap
// nodes to the node store to preserve the new LCL
int const asf =
built->stateMap().flushDirty(hotACCOUNT_NODE, built->info().seq);
int const tmf =
built->txMap().flushDirty(hotTRANSACTION_NODE, built->info().seq);
int const asf = built->stateMap().flushDirty(hotACCOUNT_NODE);
int const tmf = built->txMap().flushDirty(hotTRANSACTION_NODE);
JLOG(j.debug()) << "Flushed " << asf << " accounts and " << tmf
<< " transaction nodes";
}

40
src/ripple/app/ledger/impl/InboundLedger.cpp
View File

@@ -235,36 +235,42 @@ InboundLedger::~InboundLedger()
}
}
std::vector<uint256>
InboundLedger::neededTxHashes(int max, SHAMapSyncFilter* filter) const
static std::vector<uint256>
neededHashes(
uint256 const& root,
SHAMap& map,
int max,
SHAMapSyncFilter* filter)
{
std::vector<uint256> ret;
if (mLedger->info().txHash.isNonZero())
if (!root.isZero())
{
if (mLedger->txMap().getHash().isZero())
ret.push_back(mLedger->info().txHash);
if (map.getHash().isZero())
ret.push_back(root);
else
ret = mLedger->txMap().getNeededHashes(max, filter);
{
auto mn = map.getMissingNodes(max, filter);
ret.reserve(mn.size());
for (auto const& n : mn)
ret.push_back(n.second);
}
}
return ret;
}
std::vector<uint256>
InboundLedger::neededTxHashes(int max, SHAMapSyncFilter* filter) const
{
return neededHashes(mLedger->info().txHash, mLedger->txMap(), max, filter);
}
std::vector<uint256>
InboundLedger::neededStateHashes(int max, SHAMapSyncFilter* filter) const
{
std::vector<uint256> ret;
if (mLedger->info().accountHash.isNonZero())
{
if (mLedger->stateMap().getHash().isZero())
ret.push_back(mLedger->info().accountHash);
else
ret = mLedger->stateMap().getNeededHashes(max, filter);
}
return ret;
return neededHashes(
mLedger->info().accountHash, mLedger->stateMap(), max, filter);
}
LedgerInfo

6
src/ripple/app/ledger/impl/InboundLedgers.cpp
View File

@@ -247,8 +247,8 @@ public:
if (!node.has_nodeid() || !node.has_nodedata())
return;
auto newNode = SHAMapAbstractNode::makeFromWire(
makeSlice(node.nodedata()));
auto newNode =
SHAMapTreeNode::makeFromWire(makeSlice(node.nodedata()));
if (!newNode)
return;
@@ -257,7 +257,7 @@ public:
newNode->serializeWithPrefix(s);
app_.getLedgerMaster().addFetchPack(
newNode->getNodeHash().as_uint256(),
newNode->getHash().as_uint256(),
std::make_shared<Blob>(s.begin(), s.end()));
}
}

156
src/ripple/app/ledger/impl/LedgerMaster.cpp
View File

@@ -2033,6 +2033,64 @@ LedgerMaster::gotFetchPack(bool progress, std::uint32_t seq)
}
}
/** Populate a fetch pack with data from the map the recipient wants.
A recipient may or may not have the map that they are asking for. If
they do, we can optimize the transfer by not including parts of the
map that they are already have.
@param have The map that the recipient already has (if any).
@param cnt The maximum number of nodes to return.
@param into The protocol object into which we add information.
@param seq The sequence number of the ledger the map is a part of.
@param withLeaves True if leaf nodes should be included.
@note: The withLeaves parameter is configurable even though the
code, so far, only ever sets the parameter to true.
The rationale is that for transaction trees, it may make
sense to not include the leaves if the fetch pack is being
constructed for someone attempting to get a recent ledger
for which they already have the transactions.
However, for historical ledgers, which is the only use we
have for fetch packs right now, it makes sense to include
the transactions because the caller is unlikely to have
them.
*/
static void
populateFetchPack(
SHAMap const& want,
SHAMap const* have,
std::uint32_t cnt,
protocol::TMGetObjectByHash* into,
std::uint32_t seq,
bool withLeaves = true)
{
assert(cnt != 0);
Serializer s(1024);
want.visitDifferences(
have,
[&s, withLeaves, &cnt, into, seq](SHAMapTreeNode const& n) -> bool {
if (!withLeaves && n.isLeaf())
return true;
s.erase();
n.serializeWithPrefix(s);
auto const& hash = n.getHash().as_uint256();
protocol::TMIndexedObject* obj = into->add_objects();
obj->set_ledgerseq(seq);
obj->set_hash(hash.data(), hash.size());
obj->set_data(s.getDataPtr(), s.getLength());
return --cnt != 0;
});
}
void
LedgerMaster::makeFetchPack(
std::weak_ptr<Peer> const& wPeer,
@@ -2058,55 +2116,46 @@ LedgerMaster::makeFetchPack(
if (!peer)
return;
auto haveLedger = getLedgerByHash(haveLedgerHash);
auto have = getLedgerByHash(haveLedgerHash);
if (!haveLedger)
if (!have)
{
JLOG(m_journal.info())
<< "Peer requests fetch pack for ledger we don't have: "
<< haveLedger;
<< "Peer requests fetch pack for ledger we don't have: " << have;
peer->charge(Resource::feeRequestNoReply);
return;
}
if (haveLedger->open())
if (have->open())
{
JLOG(m_journal.warn())
<< "Peer requests fetch pack from open ledger: " << haveLedger;
<< "Peer requests fetch pack from open ledger: " << have;
peer->charge(Resource::feeInvalidRequest);
return;
}
if (haveLedger->info().seq < getEarliestFetch())
if (have->info().seq < getEarliestFetch())
{
JLOG(m_journal.debug()) << "Peer requests fetch pack that is too early";
peer->charge(Resource::feeInvalidRequest);
return;
}
auto wantLedger = getLedgerByHash(haveLedger->info().parentHash);
auto want = getLedgerByHash(have->info().parentHash);
if (!wantLedger)
if (!want)
{
JLOG(m_journal.info())
<< "Peer requests fetch pack for ledger whose predecessor we "
<< "don't have: " << haveLedger;
<< "don't have: " << have;
peer->charge(Resource::feeRequestNoReply);
return;
}
auto fpAppender = [](protocol::TMGetObjectByHash* reply,
std::uint32_t ledgerSeq,
SHAMapHash const& hash,
const Blob& blob) {
protocol::TMIndexedObject& newObj = *(reply->add_objects());
newObj.set_ledgerseq(ledgerSeq);
newObj.set_hash(hash.as_uint256().begin(), 256 / 8);
newObj.set_data(&blob[0], blob.size());
};
try
{
Serializer hdr(128);
protocol::TMGetObjectByHash reply;
reply.set_query(false);
@@ -2121,56 +2170,49 @@ LedgerMaster::makeFetchPack(
// 2. Add the nodes for the AccountStateMap of that ledger.
// 3. If there are transactions, add the nodes for the
// transactions of the ledger.
// 4. If the FetchPack now contains greater than or equal to
// 256 entries then stop.
// 4. If the FetchPack now contains at least 512 entries then stop.
// 5. If not very much time has elapsed, then loop back and repeat
// the same process adding the previous ledger to the FetchPack.
do
{
std::uint32_t lSeq = wantLedger->info().seq;
std::uint32_t lSeq = want->info().seq;
protocol::TMIndexedObject& newObj = *reply.add_objects();
newObj.set_hash(wantLedger->info().hash.data(), 256 / 8);
Serializer s(256);
s.add32(HashPrefix::ledgerMaster);
addRaw(wantLedger->info(), s);
newObj.set_data(s.getDataPtr(), s.getLength());
newObj.set_ledgerseq(lSeq);
{
// Serialize the ledger header:
hdr.erase();
wantLedger->stateMap().getFetchPack(
&haveLedger->stateMap(),
true,
16384,
std::bind(
fpAppender,
&reply,
lSeq,
std::placeholders::_1,
std::placeholders::_2));
hdr.add32(HashPrefix::ledgerMaster);
addRaw(want->info(), hdr);
if (wantLedger->info().txHash.isNonZero())
wantLedger->txMap().getFetchPack(
nullptr,
true,
512,
std::bind(
fpAppender,
&reply,
lSeq,
std::placeholders::_1,
std::placeholders::_2));
// Add the data
protocol::TMIndexedObject* obj = reply.add_objects();
obj->set_hash(
want->info().hash.data(), want->info().hash.size());
obj->set_data(hdr.getDataPtr(), hdr.getLength());
obj->set_ledgerseq(lSeq);
}
populateFetchPack(
want->stateMap(), &have->stateMap(), 16384, &reply, lSeq);
// We use nullptr here because transaction maps are per ledger
// and so the requestor is unlikely to already have it.
if (want->info().txHash.isNonZero())
populateFetchPack(want->txMap(), nullptr, 512, &reply, lSeq);
if (reply.objects().size() >= 512)
break;
// move may save a ref/unref
haveLedger = std::move(wantLedger);
wantLedger = getLedgerByHash(haveLedger->info().parentHash);
} while (wantLedger && UptimeClock::now() <= uptime + 1s);
have = std::move(want);
want = getLedgerByHash(have->info().parentHash);
} while (want && UptimeClock::now() <= uptime + 1s);
auto msg = std::make_shared<Message>(reply, protocol::mtGET_OBJECTS);
JLOG(m_journal.info())
<< "Built fetch pack with " << reply.objects().size() << " nodes";
auto msg = std::make_shared<Message>(reply, protocol::mtGET_OBJECTS);
<< "Built fetch pack with " << reply.objects().size() << " nodes ("
<< msg->getBufferSize() << " bytes)";
peer->send(msg);
}
catch (std::exception const&)

6
src/ripple/app/ledger/impl/TransactionMaster.cpp
View File

@@ -108,7 +108,7 @@ TransactionMaster::fetch(
std::shared_ptr<STTx const>
TransactionMaster::fetch(
std::shared_ptr<SHAMapItem> const& item,
SHAMapTreeNode::TNType type,
SHAMapNodeType type,
std::uint32_t uCommitLedger)
{
std::shared_ptr<STTx const> txn;
@@ -116,12 +116,12 @@ TransactionMaster::fetch(
if (!iTx)
{
if (type == SHAMapTreeNode::tnTRANSACTION_NM)
if (type == SHAMapNodeType::tnTRANSACTION_NM)
{
SerialIter sit(item->slice());
txn = std::make_shared<STTx const>(std::ref(sit));
}
else if (type == SHAMapTreeNode::tnTRANSACTION_MD)
else if (type == SHAMapNodeType::tnTRANSACTION_MD)
{
auto blob = SerialIter{item->data(), item->size()}.getVL();
txn = std::make_shared<STTx const>(

3
src/ripple/app/main/Application.cpp
View File

@@ -1906,8 +1906,7 @@ ApplicationImp::loadLedgerFromFile(std::string const& name)
}
}
loadLedger->stateMap().flushDirty(
hotACCOUNT_NODE, loadLedger->info().seq);
loadLedger->stateMap().flushDirty(hotACCOUNT_NODE);
loadLedger->setAccepted(
closeTime, closeTimeResolution, !closeTimeEstimated, *config_);

5
src/ripple/app/misc/AmendmentTable.h
View File

@@ -155,10 +155,9 @@ public:
amendTx.add(s);
initialPosition->addGiveItem(
SHAMapNodeType::tnTRANSACTION_NM,
std::make_shared<SHAMapItem>(
amendTx.getTransactionID(), s.peekData()),
true,
false);
amendTx.getTransactionID(), s.peekData()));
}
}
};

6
src/ripple/app/misc/FeeVoteImpl.cpp
View File

@@ -244,9 +244,9 @@ FeeVoteImpl::doVoting(
Serializer s;
feeTx.add(s);
auto tItem = std::make_shared<SHAMapItem>(txID, s.peekData());
if (!initialPosition->addGiveItem(std::move(tItem), true, false))
if (!initialPosition->addGiveItem(
SHAMapNodeType::tnTRANSACTION_NM,
std::make_shared<SHAMapItem>(txID, s.peekData())))
{
JLOG(journal_.warn()) << "Ledger already had fee change";
}

3
src/ripple/app/misc/NegativeUNLVote.cpp
View File

@@ -119,7 +119,8 @@ NegativeUNLVote::addTx(
Serializer s;
negUnlTx.add(s);
if (!initialSet->addGiveItem(
std::make_shared<SHAMapItem>(txID, s.peekData()), true, false))
SHAMapNodeType::tnTRANSACTION_NM,
std::make_shared<SHAMapItem>(txID, s.peekData())))
{
JLOG(j_.warn()) << "N-UNL: ledger seq=" << seq
<< ", add ttUNL_MODIFY tx failed";

6
src/ripple/app/misc/SHAMapStoreImp.cpp
View File

@@ -299,12 +299,10 @@ SHAMapStoreImp::fdRequired() const
}
bool
SHAMapStoreImp::copyNode(
std::uint64_t& nodeCount,
SHAMapAbstractNode const& node)
SHAMapStoreImp::copyNode(std::uint64_t& nodeCount, SHAMapTreeNode const& node)
{
// Copy a single record from node to dbRotating_
dbRotating_->fetchNodeObject(node.getNodeHash().as_uint256());
dbRotating_->fetchNodeObject(node.getHash().as_uint256());
if (!(++nodeCount % checkHealthInterval_))
{
if (health())

2
src/ripple/app/misc/SHAMapStoreImp.h
View File

@@ -194,7 +194,7 @@ public:
private:
// callback for visitNodes
bool
copyNode(std::uint64_t& nodeCount, SHAMapAbstractNode const& node);
copyNode(std::uint64_t& nodeCount, SHAMapTreeNode const& node);
void
run();
void

4
src/ripple/nodestore/impl/Database.cpp
View File

@@ -244,11 +244,11 @@ Database::storeLedger(
}
bool error = false;
auto visit = [&](SHAMapAbstractNode& node) {
auto visit = [&](SHAMapTreeNode& node) {
if (!isStopping())
{
if (auto nodeObject = srcDB.fetchNodeObject(
node.getNodeHash().as_uint256(), srcLedger.info().seq))
node.getHash().as_uint256(), srcLedger.info().seq))
{
batch.emplace_back(std::move(nodeObject));
if (batch.size() < batchWritePreallocationSize || storeBatch())

8
src/ripple/nodestore/impl/Shard.cpp
View File

@@ -405,11 +405,11 @@ Shard::storeLedger(
}
bool error = false;
auto visit = [&](SHAMapAbstractNode& node) {
auto visit = [&](SHAMapTreeNode const& node) {
if (!stop_)
{
if (auto nodeObject = srcDB.fetchNodeObject(
node.getNodeHash().as_uint256(), srcLedger->info().seq))
node.getHash().as_uint256(), srcLedger->info().seq))
{
batch.emplace_back(std::move(nodeObject));
if (batch.size() < batchWritePreallocationSize || storeBatch())
@@ -1288,10 +1288,10 @@ Shard::verifyLedger(
return fail("Invalid ledger account hash");
bool error{false};
auto visit = [this, &error](SHAMapAbstractNode& node) {
auto visit = [this, &error](SHAMapTreeNode const& node) {
if (stop_)
return false;
if (!verifyFetch(node.getNodeHash().as_uint256()))
if (!verifyFetch(node.getHash().as_uint256()))
error = true;
return !error;
};

4
src/ripple/overlay/Message.h
View File

@@ -64,6 +64,10 @@ public:
int type,
boost::optional<PublicKey> const& validator = {});
/** Retrieve the size of the packed but uncompressed message data. */
std::size_t
getBufferSize();
/** Retrieve the packed message data. If compressed message is requested but
* the message is not compressible then the uncompressed buffer is returned.
* @param compressed Request compressed (Compress::On) or

6
src/ripple/overlay/impl/Message.cpp
View File

@@ -177,6 +177,12 @@ Message::setHeader(
}
}
std::size_t
Message::getBufferSize()
{
return buffer_.size();
}
std::vector<uint8_t> const&
Message::getBuffer(Compressed tryCompressed)
{

18
src/ripple/rpc/handlers/Tx.cpp
View File

@@ -49,24 +49,6 @@ isValidated(LedgerMaster& ledgerMaster, std::uint32_t seq, uint256 const& hash)
return ledgerMaster.getHashBySeq(seq) == hash;
}
bool
getMetaHex(Ledger const& ledger, uint256 const& transID, std::string& hex)
{
SHAMapTreeNode::TNType type;
auto const item = ledger.txMap().peekItem(transID, type);
if (!item)
return false;
if (type != SHAMapTreeNode::tnTRANSACTION_MD)
return false;
SerialIter it(item->slice());
it.getVL(); // skip transaction
hex = strHex(makeSlice(it.getVL()));
return true;
}
struct TxResult
{
Transaction::pointer txn;

92
src/ripple/shamap/README.md
View File

@@ -17,14 +17,14 @@ or account state can be used to navigate the trie.
A `SHAMap` is a trie with two node types:
1. SHAMapInnerNode
2. SHAMapTreeNode
2. SHAMapLeafNode
Both of these nodes directly inherit from SHAMapAbstractNode which holds data
Both of these nodes directly inherit from SHAMapTreeNode which holds data
common to both of the node types.
All non-leaf nodes have type SHAMapInnerNode.
All leaf nodes have type SHAMapTreeNode.
All leaf nodes have type SHAMapLeafNode.
The root node is always a SHAMapInnerNode.
@@ -193,7 +193,7 @@ continues, unless the path indicates a child that does not exist. And in this
case, `nullptr` is returned to indicate no leaf node along the given path
exists. Otherwise a leaf node is found and a (non-owning) pointer to it is
returned. At each step, if a stack is requested, a
`pair<shared_ptr<SHAMapAbstractNode>, SHAMapNodeID>` is pushed onto the stack.
`pair<shared_ptr<SHAMapTreeNode>, SHAMapNodeID>` is pushed onto the stack.
When a child node is found by `selectBranch`, the traversal to that node
consists of two steps:
@@ -220,11 +220,11 @@ is this case stands for 'No Throw'.
The `fetchNodeNT()` method goes through three phases:
1. By calling `getCache()` we attempt to locate the missing node in the
1. By calling `cacheLookup()` we attempt to locate the missing node in the
TreeNodeCache. The TreeNodeCache is a cache of immutable SHAMapTreeNodes
that are shared across all `SHAMap`s.
Any SHAMapTreeNode that is immutable has a sequence number of zero
Any SHAMapLeafNode that is immutable has a sequence number of zero
(sharable). When a mutable `SHAMap` is created then its SHAMapTreeNodes are
given non-zero sequence numbers (unsharable). But all nodes in the
TreeNodeCache are immutable, so if one is found here, its sequence number
@@ -250,17 +250,17 @@ the `SHAMap`, node `TreeNodeCache` or database, then we don't create duplicates
by favoring the copy already in the `TreeNodeCache`.
By using `canonicalize()` we manage a thread race condition where two different
threads might both recognize the lack of a SHAMapTreeNode at the same time
threads might both recognize the lack of a SHAMapLeafNode at the same time
(during a fetch). If they both attempt to insert the node into the `SHAMap`, then
`canonicalize` makes sure that the first node in wins and the slower thread
receives back a pointer to the node inserted by the faster thread. Recall
that these two `SHAMap`s will share the same `TreeNodeCache`.
## TreeNodeCache ##
## `TreeNodeCache` ##
The `TreeNodeCache` is a `std::unordered_map` keyed on the hash of the
`SHAMap` node. The stored type consists of `shared_ptr<SHAMapAbstractNode>`,
`weak_ptr<SHAMapAbstractNode>`, and a time point indicating the most recent
`SHAMap` node. The stored type consists of `shared_ptr<SHAMapTreeNode>`,
`weak_ptr<SHAMapTreeNode>`, and a time point indicating the most recent
access of this node in the cache. The time point is based on
`std::chrono::steady_clock`.
@@ -271,7 +271,7 @@ and logging, and a target age for the contained nodes. When the target age
for a node is exceeded, and there are no more references to the node, the
node is removed from the `TreeNodeCache`.
## FullBelowCache ##
## `FullBelowCache` ##
This cache remembers which trie keys have all of their children resident in a
`SHAMap`. This optimizes the process of acquiring a complete trie. This is used
@@ -284,38 +284,51 @@ nodes, and thus that subtree does not need to be walked. These nodes are stored
in the FullBelowCache. Subsequent walks check the FullBelowCache first when
encountering a node, and ignore that subtree if found.
## SHAMapAbstractNode ##
## `SHAMapTreeNode` ##
This is a base class for the two concrete node types. It holds the following
common data:
This is an abstract base class for the concrete node types. It holds the
following common data:
1. A node type, one of:
a. error
b. inner
c. transaction with no metadata
d. transaction with metadata
e. account state
2. A hash
3. A sequence number
1. A hash
2. An identifier used to perform copy-on-write operations
## SHAMapInnerNode ##
### `SHAMapInnerNode` ###
SHAMapInnerNode publicly inherits directly from SHAMapAbstractNode. It holds
`SHAMapInnerNode` publicly inherits directly from `SHAMapTreeNode`. It holds
the following data:
1. Up to 16 child nodes, each held with a shared_ptr.
2. A hash for each child.
3. A 16-bit bitset with a 1 bit set for each child that exists.
4. Flag to aid online delete and consistency with data on disk.
3. A bitset to indicate which of the 16 children exist.
4. An identifier used to determine whether the map below this node is
fully populated
## SHAMapTreeNode ##
### `SHAMapLeafNode` ###
SHAMapTreeNode publicly inherits directly from SHAMapAbstractNode. It holds the
`SHAMapLeafNode` is an abstract class which publicly inherits directly from
`SHAMapTreeNode`. It isIt holds the
following data:
1. A shared_ptr to a const SHAMapItem.
#### `SHAMapAccountStateLeafNode` ####
`SHAMapAccountStateLeafNode` is a class which publicly inherits directly from
`SHAMapLeafNode`. It is used to represent entries (i.e. account objects, escrow
objects, trust lines, etc.) in a state map.
#### `SHAMapTxLeafNode` ####
`SHAMapTxLeafNode` is a class which publicly inherits directly from
`SHAMapLeafNode`. It is used to represent transactions in a state map.
#### `SHAMapTxPlusMetaLeafNode` ####
`SHAMapTxPlusMetaLeafNode` is a class which publicly inherits directly from
`SHAMapLeafNode`. It is used to represent transactions along with metadata
associated with this transaction in a state map.
## SHAMapItem ##
This holds the following data:
@@ -323,27 +336,4 @@ This holds the following data:
1. uint256. The hash of the data.
2. vector<unsigned char>. The data (transactions, account info).
## SHAMap Improvements ##
Here's a simple one: the SHAMapTreeNode::mAccessSeq member is currently not used
and could be removed.
Here's a more important change. The trie structure is currently embedded in the
SHAMapTreeNodes themselves. It doesn't have to be that way, and that should be
fixed.
When we navigate the trie (say, like `SHAMap::walkTo()`) we currently ask each
node for information that we could determine locally. We know the depth because
we know how many nodes we have traversed. We know the ID that we need because
that's how we're steering. So we don't need to store the ID in the node. The
next refactor should remove all calls to `SHAMapTreeNode::GetID()`.
Then we can remove the NodeID member from SHAMapTreeNode.
Then we can change the `SHAMap::mTNBtID` member to be `mTNByHash`.
An additional possible refactor would be to have a base type, SHAMapTreeNode,
and derive from that InnerNode and LeafNode types. That would remove some
storage (the array of 16 hashes) from the LeafNodes. That refactor would also
have the effect of simplifying methods like `isLeaf()` and `hasItem()`.

132
src/ripple/shamap/SHAMap.h
View File

@@ -27,7 +27,9 @@
#include <ripple/shamap/Family.h>
#include <ripple/shamap/FullBelowCache.h>
#include <ripple/shamap/SHAMapAddNode.h>
#include <ripple/shamap/SHAMapInnerNode.h>
#include <ripple/shamap/SHAMapItem.h>
#include <ripple/shamap/SHAMapLeafNode.h>
#include <ripple/shamap/SHAMapMissingNode.h>
#include <ripple/shamap/SHAMapTreeNode.h>
#include <ripple/shamap/TreeNodeCache.h>
@@ -95,13 +97,18 @@ class SHAMap
private:
Family& f_;
beast::Journal journal_;
std::uint32_t seq_;
std::uint32_t ledgerSeq_ = 0; // sequence number of ledger this is part of
std::shared_ptr<SHAMapAbstractNode> root_;
/** ID to distinguish this map for all others we're sharing nodes with. */
std::uint32_t cowid_ = 1;
/** The sequence of the ledger that this map references, if any. */
std::uint32_t ledgerSeq_ = 0;
std::shared_ptr<SHAMapTreeNode> root_;
mutable SHAMapState state_;
SHAMapType type_;
bool backed_ = true; // Map is backed by the database
bool full_ = false; // Map is believed complete in database
SHAMapType const type_;
bool backed_ = true; // Map is backed by the database
mutable bool full_ = false; // Map is believed complete in database
public:
/** Each non-leaf node has 16 children (the 'radix tree' part of the map) */
@@ -115,7 +122,6 @@ public:
std::shared_ptr<SHAMapItem const>>;
using Delta = std::map<uint256, DeltaItem>;
~SHAMap();
SHAMap(SHAMap const&) = delete;
SHAMap&
operator=(SHAMap const&) = delete;
@@ -125,6 +131,8 @@ public:
SHAMap(SHAMapType t, uint256 const& hash, Family& f);
~SHAMap() = default;
Family const&
family() const
{
@@ -171,26 +179,26 @@ public:
fetchRoot(SHAMapHash const& hash, SHAMapSyncFilter* filter);
// normal hash access functions
/** Does the tree have an item with the given ID? */
bool
hasItem(uint256 const& id) const;
bool
delItem(uint256 const& id);
bool
addItem(SHAMapItem&& i, bool isTransaction, bool hasMeta);
addItem(SHAMapNodeType type, SHAMapItem&& i);
SHAMapHash
getHash() const;
// save a copy if you have a temporary anyway
bool
updateGiveItem(
std::shared_ptr<SHAMapItem const>,
bool isTransaction,
bool hasMeta);
updateGiveItem(SHAMapNodeType type, std::shared_ptr<SHAMapItem const>);
bool
addGiveItem(
std::shared_ptr<SHAMapItem const>,
bool isTransaction,
bool hasMeta);
addGiveItem(SHAMapNodeType type, std::shared_ptr<SHAMapItem const> item);
// Save a copy if you need to extend the life
// of the SHAMapItem beyond this SHAMap
@@ -198,8 +206,6 @@ public:
peekItem(uint256 const& id) const;
std::shared_ptr<SHAMapItem const> const&
peekItem(uint256 const& id, SHAMapHash& hash) const;
std::shared_ptr<SHAMapItem const> const&
peekItem(uint256 const& id, SHAMapTreeNode::TNType& type) const;
// traverse functions
const_iterator
@@ -211,7 +217,7 @@ public:
If function returns false, visitNodes exits.
*/
void
visitNodes(std::function<bool(SHAMapAbstractNode&)> const& function) const;
visitNodes(std::function<bool(SHAMapTreeNode&)> const& function) const;
/** Visit every node in this SHAMap that
is not present in the specified SHAMap
@@ -222,7 +228,7 @@ public:
void
visitDifferences(
SHAMap const* have,
std::function<bool(SHAMapAbstractNode&)>) const;
std::function<bool(SHAMapTreeNode const&)>) const;
/** Visit every leaf node in this SHAMap
@@ -250,9 +256,9 @@ public:
bool
getNodeFat(
SHAMapNodeID node,
SHAMapNodeID const& wanted,
std::vector<SHAMapNodeID>& nodeIDs,
std::vector<Blob>& rawNode,
std::vector<Blob>& rawNodes,
bool fatLeaves,
std::uint32_t depth) const;
@@ -260,8 +266,6 @@ public:
void
serializeRoot(Serializer& s) const;
std::vector<uint256>
getNeededHashes(int max, SHAMapSyncFilter* filter);
SHAMapAddNode
addRootNode(
SHAMapHash const& hash,
@@ -290,26 +294,21 @@ public:
bool
compare(SHAMap const& otherMap, Delta& differences, int maxCount) const;
/** Convert any modified nodes to shared. */
int
flushDirty(NodeObjectType t, std::uint32_t seq);
unshare();
/** Flush modified nodes to the nodestore and convert them to shared. */
int
flushDirty(NodeObjectType t);
void
walkMap(std::vector<SHAMapMissingNode>& missingNodes, int maxMissing) const;
bool
deepCompare(SHAMap& other) const; // Intended for debug/test only
using fetchPackEntry_t = std::pair<uint256, Blob>;
void
getFetchPack(
SHAMap const* have,
bool includeLeaves,
int max,
std::function<void(SHAMapHash const&, const Blob&)>) const;
void
setUnbacked();
int
unshare();
void
dump(bool withHashes = false) const;
@@ -317,29 +316,29 @@ public:
invariants() const;
private:
using SharedPtrNodeStack = std::stack<
std::pair<std::shared_ptr<SHAMapAbstractNode>, SHAMapNodeID>>;
using SharedPtrNodeStack =
std::stack<std::pair<std::shared_ptr<SHAMapTreeNode>, SHAMapNodeID>>;
using DeltaRef = std::pair<
std::shared_ptr<SHAMapItem const> const&,
std::shared_ptr<SHAMapItem const> const&>;
// tree node cache operations
std::shared_ptr<SHAMapAbstractNode>
getCache(SHAMapHash const& hash) const;
std::shared_ptr<SHAMapTreeNode>
cacheLookup(SHAMapHash const& hash) const;
void
canonicalize(SHAMapHash const& hash, std::shared_ptr<SHAMapAbstractNode>&)
canonicalize(SHAMapHash const& hash, std::shared_ptr<SHAMapTreeNode>&)
const;
// database operations
std::shared_ptr<SHAMapAbstractNode>
std::shared_ptr<SHAMapTreeNode>
fetchNodeFromDB(SHAMapHash const& hash) const;
std::shared_ptr<SHAMapAbstractNode>
std::shared_ptr<SHAMapTreeNode>
fetchNodeNT(SHAMapHash const& hash) const;
std::shared_ptr<SHAMapAbstractNode>
std::shared_ptr<SHAMapTreeNode>
fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter* filter) const;
std::shared_ptr<SHAMapAbstractNode>
std::shared_ptr<SHAMapTreeNode>
fetchNode(SHAMapHash const& hash) const;
std::shared_ptr<SHAMapAbstractNode>
std::shared_ptr<SHAMapTreeNode>
checkFilter(SHAMapHash const& hash, SHAMapSyncFilter* filter) const;
/** Update hashes up to the root */
@@ -347,16 +346,16 @@ private:
dirtyUp(
SharedPtrNodeStack& stack,
uint256 const& target,
std::shared_ptr<SHAMapAbstractNode> terminal);
std::shared_ptr<SHAMapTreeNode> terminal);
/** Walk towards the specified id, returning the node. Caller must check
if the return is nullptr, and if not, if the node->peekItem()->key() ==
id */
SHAMapTreeNode*
SHAMapLeafNode*
walkTowardsKey(uint256 const& id, SharedPtrNodeStack* stack = nullptr)
const;
/** Return nullptr if key not found */
SHAMapTreeNode*
SHAMapLeafNode*
findKey(uint256 const& id) const;
/** Unshare the node, allowing it to be modified */
@@ -370,38 +369,35 @@ private:
preFlushNode(std::shared_ptr<Node> node) const;
/** write and canonicalize modified node */
std::shared_ptr<SHAMapAbstractNode>
writeNode(
NodeObjectType t,
std::uint32_t seq,
std::shared_ptr<SHAMapAbstractNode> node) const;
std::shared_ptr<SHAMapTreeNode>
writeNode(NodeObjectType t, std::shared_ptr<SHAMapTreeNode> node) const;
SHAMapTreeNode*
SHAMapLeafNode*
firstBelow(
std::shared_ptr<SHAMapAbstractNode>,
std::shared_ptr<SHAMapTreeNode>,
SharedPtrNodeStack& stack,
int branch = 0) const;
// Simple descent
// Get a child of the specified node
SHAMapAbstractNode*
SHAMapTreeNode*
descend(SHAMapInnerNode*, int branch) const;
SHAMapAbstractNode*
SHAMapTreeNode*
descendThrow(SHAMapInnerNode*, int branch) const;
std::shared_ptr<SHAMapAbstractNode>
std::shared_ptr<SHAMapTreeNode>
descend(std::shared_ptr<SHAMapInnerNode> const&, int branch) const;
std::shared_ptr<SHAMapAbstractNode>
std::shared_ptr<SHAMapTreeNode>
descendThrow(std::shared_ptr<SHAMapInnerNode> const&, int branch) const;
// Descend with filter
SHAMapAbstractNode*
SHAMapTreeNode*
descendAsync(
SHAMapInnerNode* parent,
int branch,
SHAMapSyncFilter* filter,
bool& pending) const;
std::pair<SHAMapAbstractNode*, SHAMapNodeID>
std::pair<SHAMapTreeNode*, SHAMapNodeID>
descend(
SHAMapInnerNode* parent,
SHAMapNodeID const& parentID,
@@ -410,31 +406,31 @@ private:
// Non-storing
// Does not hook the returned node to its parent
std::shared_ptr<SHAMapAbstractNode>
std::shared_ptr<SHAMapTreeNode>
descendNoStore(std::shared_ptr<SHAMapInnerNode> const&, int branch) const;
/** If there is only one leaf below this node, get its contents */
std::shared_ptr<SHAMapItem const> const&
onlyBelow(SHAMapAbstractNode*) const;
onlyBelow(SHAMapTreeNode*) const;
bool
hasInnerNode(SHAMapNodeID const& nodeID, SHAMapHash const& hash) const;
bool
hasLeafNode(uint256 const& tag, SHAMapHash const& hash) const;
SHAMapTreeNode const*
SHAMapLeafNode const*
peekFirstItem(SharedPtrNodeStack& stack) const;
SHAMapTreeNode const*
SHAMapLeafNode const*
peekNextItem(uint256 const& id, SharedPtrNodeStack& stack) const;
bool
walkBranch(
SHAMapAbstractNode* node,
SHAMapTreeNode* node,
std::shared_ptr<SHAMapItem const> const& otherMapItem,
bool isFirstMap,
Delta& differences,
int& maxCount) const;
int
walkSubTree(bool doWrite, NodeObjectType t, std::uint32_t seq);
walkSubTree(bool doWrite, NodeObjectType t);
// Structure to track information about call to
// getMissingNodes while it's in progress

93
src/ripple/shamap/SHAMapAccountStateLeafNode.h Normal file
View File

@@ -0,0 +1,93 @@
//------------------------------------------------------------------------------
/*
This file is part of rippled: https://github.com/ripple/rippled
Copyright (c) 2012, 2013 Ripple Labs Inc.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#ifndef RIPPLE_SHAMAP_SHAMAPACCOUNTSTATELEAFNODE_H_INCLUDED
#define RIPPLE_SHAMAP_SHAMAPACCOUNTSTATELEAFNODE_H_INCLUDED
#include <ripple/basics/CountedObject.h>
#include <ripple/protocol/HashPrefix.h>
#include <ripple/protocol/digest.h>
#include <ripple/shamap/SHAMapItem.h>
#include <ripple/shamap/SHAMapLeafNode.h>
#include <ripple/shamap/SHAMapNodeID.h>
namespace ripple {
/** A leaf node for a state object. */
class SHAMapAccountStateLeafNode final
: public SHAMapLeafNode,
public CountedObject<SHAMapAccountStateLeafNode>
{
public:
SHAMapAccountStateLeafNode(
std::shared_ptr<SHAMapItem const> item,
std::uint32_t cowid)
: SHAMapLeafNode(std::move(item), cowid)
{
updateHash();
}
SHAMapAccountStateLeafNode(
std::shared_ptr<SHAMapItem const> item,
std::uint32_t cowid,
SHAMapHash const& hash)
: SHAMapLeafNode(std::move(item), cowid, hash)
{
}
std::shared_ptr<SHAMapTreeNode>
clone(std::uint32_t cowid) const final override
{
return std::make_shared<SHAMapAccountStateLeafNode>(
item_, cowid, hash_);
}
SHAMapNodeType
getType() const final override
{
return SHAMapNodeType::tnACCOUNT_STATE;
}
void
updateHash() final override
{
hash_ = SHAMapHash{sha512Half(
HashPrefix::leafNode, makeSlice(item_->peekData()), item_->key())};
}
void
serializeForWire(Serializer& s) const final override
{
s.addRaw(item_->peekData());
s.addBitString(item_->key());
s.add8(wireTypeAccountState);
}
void
serializeWithPrefix(Serializer& s) const final override
{
s.add32(HashPrefix::leafNode);
s.addRaw(item_->peekData());
s.addBitString(item_->key());
}
};
} // namespace ripple
#endif

155
src/ripple/shamap/SHAMapInnerNode.h Normal file
View File

@@ -0,0 +1,155 @@
//------------------------------------------------------------------------------
/*
This file is part of rippled: https://github.com/ripple/rippled
Copyright (c) 2012, 2013 Ripple Labs Inc.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#ifndef RIPPLE_SHAMAP_SHAMAPINNERNODE_H_INCLUDED
#define RIPPLE_SHAMAP_SHAMAPINNERNODE_H_INCLUDED
#include <ripple/basics/TaggedCache.h>
#include <ripple/beast/utility/Journal.h>
#include <ripple/shamap/SHAMapItem.h>
#include <ripple/shamap/SHAMapNodeID.h>
#include <ripple/shamap/SHAMapTreeNode.h>
#include <bitset>
#include <cstdint>
#include <memory>
#include <mutex>
#include <optional>
#include <string>
namespace ripple {
class SHAMapInnerNode final : public SHAMapTreeNode,
public CountedObject<SHAMapInnerNode>
{
std::array<SHAMapHash, 16> mHashes;
std::shared_ptr<SHAMapTreeNode> mChildren[16];
int mIsBranch = 0;
std::uint32_t mFullBelowGen = 0;
static std::mutex childLock;
public:
SHAMapInnerNode(std::uint32_t cowid);
std::shared_ptr<SHAMapTreeNode>
clone(std::uint32_t cowid) const override;
SHAMapNodeType
getType() const override
{
return SHAMapNodeType::tnINNER;
}
bool
isLeaf() const override
{
return false;
}
bool
isInner() const override
{
return true;
}
bool
isEmpty() const;
bool
isEmptyBranch(int m) const;
int
getBranchCount() const;
SHAMapHash const&
getChildHash(int m) const;
void
setChild(int m, std::shared_ptr<SHAMapTreeNode> const& child);
void
shareChild(int m, std::shared_ptr<SHAMapTreeNode> const& child);
SHAMapTreeNode*
getChildPointer(int branch);
std::shared_ptr<SHAMapTreeNode>
getChild(int branch);
virtual std::shared_ptr<SHAMapTreeNode>
canonicalizeChild(int branch, std::shared_ptr<SHAMapTreeNode> node);
// sync functions
bool
isFullBelow(std::uint32_t generation) const;
void
setFullBelowGen(std::uint32_t gen);
void
updateHash() override;
/** Recalculate the hash of all children and this node. */
void
updateHashDeep();
void
serializeForWire(Serializer&) const override;
void
serializeWithPrefix(Serializer&) const override;
std::string
getString(SHAMapNodeID const&) const override;
void
invariants(bool is_root = false) const override;
static std::shared_ptr<SHAMapTreeNode>
makeFullInner(Slice data, SHAMapHash const& hash, bool hashValid);
static std::shared_ptr<SHAMapTreeNode>
makeCompressedInner(Slice data);
};
inline SHAMapInnerNode::SHAMapInnerNode(std::uint32_t cowid)
: SHAMapTreeNode(cowid)
{
}
inline bool
SHAMapInnerNode::isEmptyBranch(int m) const
{
return (mIsBranch & (1 << m)) == 0;
}
inline SHAMapHash const&
SHAMapInnerNode::getChildHash(int m) const
{
assert(m >= 0 && m < 16);
return mHashes[m];
}
inline bool
SHAMapInnerNode::isFullBelow(std::uint32_t generation) const
{
return mFullBelowGen == generation;
}
inline void
SHAMapInnerNode::setFullBelowGen(std::uint32_t gen)
{
mFullBelowGen = gen;
}
} // namespace ripple
#endif

82
src/ripple/shamap/SHAMapLeafNode.h Normal file
View File

@@ -0,0 +1,82 @@
//------------------------------------------------------------------------------
/*
This file is part of rippled: https://github.com/ripple/rippled
Copyright (c) 2012, 2013 Ripple Labs Inc.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#ifndef RIPPLE_SHAMAP_SHAMAPLEAFNODE_H_INCLUDED
#define RIPPLE_SHAMAP_SHAMAPLEAFNODE_H_INCLUDED
#include <ripple/shamap/SHAMapItem.h>
#include <ripple/shamap/SHAMapNodeID.h>
#include <ripple/shamap/SHAMapTreeNode.h>
#include <cstdint>
#include <memory>
namespace ripple {
class SHAMapLeafNode : public SHAMapTreeNode
{
protected:
std::shared_ptr<SHAMapItem const> item_;
SHAMapLeafNode(std::shared_ptr<SHAMapItem const> item, std::uint32_t cowid);
SHAMapLeafNode(
std::shared_ptr<SHAMapItem const> item,
std::uint32_t cowid,
SHAMapHash const& hash);
public:
SHAMapLeafNode(const SHAMapLeafNode&) = delete;
SHAMapLeafNode&
operator=(const SHAMapLeafNode&) = delete;
bool
isLeaf() const final override
{
return true;
}
bool
isInner() const final override
{
return false;
}
void
invariants(bool is_root = false) const final override;
public:
std::shared_ptr<SHAMapItem const> const&
peekItem() const;
/** Set the item that this node points to and update the node's hash.
@param i the new item
@return false if the change was, effectively, a noop (that is, if the
hash was unchanged); true otherwise.
*/
bool
setItem(std::shared_ptr<SHAMapItem const> i);
std::string
getString(SHAMapNodeID const&) const final override;
};
} // namespace ripple
#endif

3
src/ripple/shamap/SHAMapNodeID.h
View File

@@ -41,7 +41,8 @@ public:
SHAMapNodeID(SHAMapNodeID const& other) = default;
SHAMapNodeID(unsigned int depth, uint256 const& hash);
SHAMapNodeID& operator=(SHAMapNodeID const& other) = default;
SHAMapNodeID&
operator=(SHAMapNodeID const& other) = default;
bool
isRoot() const

2
src/ripple/shamap/SHAMapSyncFilter.h
View File

@@ -43,7 +43,7 @@ public:
SHAMapHash const& nodeHash,
std::uint32_t ledgerSeq,
Blob&& nodeData,
SHAMapTreeNode::TNType type) const = 0;
SHAMapNodeType type) const = 0;
virtual boost::optional<Blob>
getNode(SHAMapHash const& nodeHash) const = 0;

403
src/ripple/shamap/SHAMapTreeNode.h
View File

@@ -33,8 +33,17 @@
namespace ripple {
// These are wire-protocol identifiers used during serialization to encode the
// type of a node. They should not be arbitrarily be changed.
static constexpr unsigned char const wireTypeTransaction = 0;
static constexpr unsigned char const wireTypeAccountState = 1;
static constexpr unsigned char const wireTypeInner = 2;
static constexpr unsigned char const wireTypeCompressedInner = 3;
static constexpr unsigned char const wireTypeTransactionWithMeta = 4;
// A SHAMapHash is the hash of a node in a SHAMap, and also the
// type of the hash of the entire SHAMap.
class SHAMapHash
{
uint256 hash_;
@@ -114,49 +123,120 @@ operator!=(SHAMapHash const& x, SHAMapHash const& y)
return !(x == y);
}
class SHAMapAbstractNode
enum class SHAMapNodeType {
tnINNER = 1,
tnTRANSACTION_NM = 2, // transaction, no metadata
tnTRANSACTION_MD = 3, // transaction, with metadata
tnACCOUNT_STATE = 4
};
class SHAMapTreeNode
{
public:
enum TNType {
tnINNER = 1,
tnTRANSACTION_NM = 2, // transaction, no metadata
tnTRANSACTION_MD = 3, // transaction, with metadata
tnACCOUNT_STATE = 4
};
protected:
SHAMapHash hash_;
/** Determines the owning SHAMap, if any. Used for copy-on-write semantics.
If this value is 0, the node is not dirty and does not need to be
flushed. It is eligible for sharing and may be included multiple
SHAMap instances.
*/
std::uint32_t cowid_;
protected:
TNType mType;
SHAMapHash mHash;
std::uint32_t mSeq;
SHAMapTreeNode(SHAMapTreeNode const&) = delete;
SHAMapTreeNode&
operator=(SHAMapTreeNode const&) = delete;
protected:
virtual ~SHAMapAbstractNode() = 0;
SHAMapAbstractNode(SHAMapAbstractNode const&) = delete;
SHAMapAbstractNode&
operator=(SHAMapAbstractNode const&) = delete;
/** Construct a node
SHAMapAbstractNode(TNType type, std::uint32_t seq);
SHAMapAbstractNode(TNType type, std::uint32_t seq, SHAMapHash const& hash);
@param cowid The identifier of a SHAMap. For more, see #cowid_
@param hash The hash associated with this node, if any.
*/
/** @{ */
explicit SHAMapTreeNode(std::uint32_t cowid) noexcept : cowid_(cowid)
{
}
explicit SHAMapTreeNode(
std::uint32_t cowid,
SHAMapHash const& hash) noexcept
: hash_(hash), cowid_(cowid)
{
}
/** @} */
public:
virtual ~SHAMapTreeNode() noexcept = default;
/** \defgroup SHAMap Copy-on-Write Support
By nature, a node may appear in multiple SHAMap instances. Rather than
actually duplicating these nodes, SHAMap opts to be memory efficient
and uses copy-on-write semantics for nodes.
Only nodes that are not modified and don't need to be flushed back can
be shared. Once a node needs to be changed, it must first be copied and
the copy must marked as not shareable.
Note that just because a node may not be *owned* by a given SHAMap
instance does not mean that the node is NOT a part of any SHAMap. It
only means that the node is not owned exclusively by any one SHAMap.
For more on copy-on-write, check out:
https://en.wikipedia.org/wiki/Copy-on-write
*/
/** @{ */
/** Returns the SHAMap that owns this node.
@return the ID of the SHAMap that owns this node, or 0 if the node
is not owned by any SHAMap and is a candidate for sharing.
*/
std::uint32_t
getSeq() const;
void
setSeq(std::uint32_t s);
SHAMapHash const&
getNodeHash() const;
TNType
getType() const;
bool
isLeaf() const;
bool
isInner() const;
bool
isInBounds(SHAMapNodeID const& id) const;
cowid() const
{
return cowid_;
}
virtual bool
/** If this node is shared with another map, mark it as no longer shared.
Only nodes that are not modified and do not need to be flushed back
should be marked as unshared.
*/
void
unshare()
{
cowid_ = 0;
}
/** Make a copy of this node, setting the owner. */
virtual std::shared_ptr<SHAMapTreeNode>
clone(std::uint32_t cowid) const = 0;
/** @} */
/** Recalculate the hash of this node. */
virtual void
updateHash() = 0;
/** Return the hash of this node. */
SHAMapHash const&
getHash() const
{
return hash_;
}
/** Determines the type of node. */
virtual SHAMapNodeType
getType() const = 0;
/** Determines if this is a leaf node. */
virtual bool
isLeaf() const = 0;
/** Determines if this is an inner node. */
virtual bool
isInner() const = 0;
/** Serialize the node in a format appropriate for sending over the wire */
virtual void
serializeForWire(Serializer&) const = 0;
@@ -167,268 +247,27 @@ public:
virtual std::string
getString(SHAMapNodeID const&) const;
virtual std::shared_ptr<SHAMapAbstractNode>
clone(std::uint32_t seq) const = 0;
virtual uint256 const&
key() const = 0;
virtual void
invariants(bool is_root = false) const = 0;
static std::shared_ptr<SHAMapAbstractNode>
static std::shared_ptr<SHAMapTreeNode>
makeFromPrefix(Slice rawNode, SHAMapHash const& hash);
static std::shared_ptr<SHAMapAbstractNode>
static std::shared_ptr<SHAMapTreeNode>
makeFromWire(Slice rawNode);
private:
static std::shared_ptr<SHAMapAbstractNode>
makeTransaction(
Slice data,
std::uint32_t seq,
SHAMapHash const& hash,
bool hashValid);
static std::shared_ptr<SHAMapTreeNode>
makeTransaction(Slice data, SHAMapHash const& hash, bool hashValid);
static std::shared_ptr<SHAMapAbstractNode>
makeAccountState(
Slice data,
std::uint32_t seq,
SHAMapHash const& hash,
bool hashValid);
static std::shared_ptr<SHAMapTreeNode>
makeAccountState(Slice data, SHAMapHash const& hash, bool hashValid);
static std::shared_ptr<SHAMapAbstractNode>
makeTransactionWithMeta(
Slice data,
std::uint32_t seq,
SHAMapHash const& hash,
bool hashValid);
static std::shared_ptr<SHAMapTreeNode>
makeTransactionWithMeta(Slice data, SHAMapHash const& hash, bool hashValid);
};
class SHAMapInnerNode : public SHAMapAbstractNode,
public CountedObject<SHAMapInnerNode>
{
std::array<SHAMapHash, 16> mHashes;
std::shared_ptr<SHAMapAbstractNode> mChildren[16];
int mIsBranch = 0;
std::uint32_t mFullBelowGen = 0;
static std::mutex childLock;
public:
SHAMapInnerNode(std::uint32_t seq);
std::shared_ptr<SHAMapAbstractNode>
clone(std::uint32_t seq) const override;
bool
isEmpty() const;
bool
isEmptyBranch(int m) const;
int
getBranchCount() const;
SHAMapHash const&
getChildHash(int m) const;
void
setChild(int m, std::shared_ptr<SHAMapAbstractNode> const& child);
void
shareChild(int m, std::shared_ptr<SHAMapAbstractNode> const& child);
SHAMapAbstractNode*
getChildPointer(int branch);
std::shared_ptr<SHAMapAbstractNode>
getChild(int branch);
virtual std::shared_ptr<SHAMapAbstractNode>
canonicalizeChild(int branch, std::shared_ptr<SHAMapAbstractNode> node);
// sync functions
bool
isFullBelow(std::uint32_t generation) const;
void
setFullBelowGen(std::uint32_t gen);
bool
updateHash() override;
void
updateHashDeep();
void
serializeForWire(Serializer&) const override;
void
serializeWithPrefix(Serializer&) const override;
std::string
getString(SHAMapNodeID const&) const override;
uint256 const&
key() const override;
void
invariants(bool is_root = false) const override;
static std::shared_ptr<SHAMapAbstractNode>
makeFullInner(
Slice data,
std::uint32_t seq,
SHAMapHash const& hash,
bool hashValid);
static std::shared_ptr<SHAMapAbstractNode>
makeCompressedInner(Slice data, std::uint32_t seq);
};
// SHAMapTreeNode represents a leaf, and may eventually be renamed to reflect
// that.
class SHAMapTreeNode : public SHAMapAbstractNode,
public CountedObject<SHAMapTreeNode>
{
private:
std::shared_ptr<SHAMapItem const> mItem;
public:
SHAMapTreeNode(const SHAMapTreeNode&) = delete;
SHAMapTreeNode&
operator=(const SHAMapTreeNode&) = delete;
SHAMapTreeNode(
std::shared_ptr<SHAMapItem const> item,
TNType type,
std::uint32_t seq);
SHAMapTreeNode(
std::shared_ptr<SHAMapItem const> item,
TNType type,
std::uint32_t seq,
SHAMapHash const& hash);
std::shared_ptr<SHAMapAbstractNode>
clone(std::uint32_t seq) const override;
void
serializeForWire(Serializer&) const override;
void
serializeWithPrefix(Serializer&) const override;
uint256 const&
key() const override;
void
invariants(bool is_root = false) const override;
public: // public only to SHAMap
// item node function
bool
hasItem() const;
std::shared_ptr<SHAMapItem const> const&
peekItem() const;
bool
setItem(std::shared_ptr<SHAMapItem const> i, TNType type);
std::string
getString(SHAMapNodeID const&) const override;
bool
updateHash() override;
};
// SHAMapAbstractNode
inline SHAMapAbstractNode::SHAMapAbstractNode(TNType type, std::uint32_t seq)
: mType(type), mSeq(seq)
{
}
inline SHAMapAbstractNode::SHAMapAbstractNode(
TNType type,
std::uint32_t seq,
SHAMapHash const& hash)
: mType(type), mHash(hash), mSeq(seq)
{
}
inline std::uint32_t
SHAMapAbstractNode::getSeq() const
{
return mSeq;
}
inline void
SHAMapAbstractNode::setSeq(std::uint32_t s)
{
mSeq = s;
}
inline SHAMapHash const&
SHAMapAbstractNode::getNodeHash() const
{
return mHash;
}
inline SHAMapAbstractNode::TNType
SHAMapAbstractNode::getType() const
{
return mType;
}
inline bool
SHAMapAbstractNode::isLeaf() const
{
return (mType == tnTRANSACTION_NM) || (mType == tnTRANSACTION_MD) ||
(mType == tnACCOUNT_STATE);
}
inline bool
SHAMapAbstractNode::isInner() const
{
return mType == tnINNER;
}
inline bool
SHAMapAbstractNode::isInBounds(SHAMapNodeID const& id) const
{
// Nodes at depth 64 must be leaves
return (!isInner() || (id.getDepth() < 64));
}
// SHAMapInnerNode
inline SHAMapInnerNode::SHAMapInnerNode(std::uint32_t seq)
: SHAMapAbstractNode(tnINNER, seq)
{
}
inline bool
SHAMapInnerNode::isEmptyBranch(int m) const
{
return (mIsBranch & (1 << m)) == 0;
}
inline SHAMapHash const&
SHAMapInnerNode::getChildHash(int m) const
{
assert((m >= 0) && (m < 16) && (getType() == tnINNER));
return mHashes[m];
}
inline bool
SHAMapInnerNode::isFullBelow(std::uint32_t generation) const
{
return mFullBelowGen == generation;
}
inline void
SHAMapInnerNode::setFullBelowGen(std::uint32_t gen)
{
mFullBelowGen = gen;
}
// SHAMapTreeNode
inline bool
SHAMapTreeNode::hasItem() const
{
return bool(mItem);
}
inline std::shared_ptr<SHAMapItem const> const&
SHAMapTreeNode::peekItem() const
{
return mItem;
}
} // namespace ripple
#endif

89
src/ripple/shamap/SHAMapTxLeafNode.h Normal file
View File

@@ -0,0 +1,89 @@
//------------------------------------------------------------------------------
/*
This file is part of rippled: https://github.com/ripple/rippled
Copyright (c) 2012, 2013 Ripple Labs Inc.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#ifndef RIPPLE_SHAMAP_SHAMAPTXLEAFNODE_H_INCLUDED
#define RIPPLE_SHAMAP_SHAMAPTXLEAFNODE_H_INCLUDED
#include <ripple/basics/CountedObject.h>
#include <ripple/protocol/HashPrefix.h>
#include <ripple/protocol/digest.h>
#include <ripple/shamap/SHAMapItem.h>
#include <ripple/shamap/SHAMapLeafNode.h>
#include <ripple/shamap/SHAMapNodeID.h>
namespace ripple {
/** A leaf node for a transaction. No metadata is included. */
class SHAMapTxLeafNode final : public SHAMapLeafNode,
public CountedObject<SHAMapTxLeafNode>
{
public:
SHAMapTxLeafNode(
std::shared_ptr<SHAMapItem const> item,
std::uint32_t cowid)
: SHAMapLeafNode(std::move(item), cowid)
{
updateHash();
}
SHAMapTxLeafNode(
std::shared_ptr<SHAMapItem const> item,
std::uint32_t cowid,
SHAMapHash const& hash)
: SHAMapLeafNode(std::move(item), cowid, hash)
{
}
std::shared_ptr<SHAMapTreeNode>
clone(std::uint32_t cowid) const final override
{
return std::make_shared<SHAMapTxLeafNode>(item_, cowid, hash_);
}
SHAMapNodeType
getType() const final override
{
return SHAMapNodeType::tnTRANSACTION_NM;
}
void
updateHash() final override
{
hash_ = SHAMapHash{sha512Half(
HashPrefix::transactionID, makeSlice(item_->peekData()))};
}
void
serializeForWire(Serializer& s) const final override
{
s.addRaw(item_->peekData());
s.add8(wireTypeTransaction);
}
void
serializeWithPrefix(Serializer& s) const final override
{
s.add32(HashPrefix::transactionID);
s.addRaw(item_->peekData());
}
};
} // namespace ripple
#endif

92
src/ripple/shamap/SHAMapTxPlusMetaLeafNode.h Normal file
View File

@@ -0,0 +1,92 @@
//------------------------------------------------------------------------------
/*
This file is part of rippled: https://github.com/ripple/rippled
Copyright (c) 2012, 2013 Ripple Labs Inc.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#ifndef RIPPLE_SHAMAP_SHAMAPLEAFTXPLUSMETANODE_H_INCLUDED
#define RIPPLE_SHAMAP_SHAMAPLEAFTXPLUSMETANODE_H_INCLUDED
#include <ripple/basics/CountedObject.h>
#include <ripple/protocol/HashPrefix.h>
#include <ripple/protocol/digest.h>
#include <ripple/shamap/SHAMapItem.h>
#include <ripple/shamap/SHAMapLeafNode.h>
#include <ripple/shamap/SHAMapNodeID.h>
namespace ripple {
/** A leaf node for a transaction and its associated metadata. */
class SHAMapTxPlusMetaLeafNode final
: public SHAMapLeafNode,
public CountedObject<SHAMapTxPlusMetaLeafNode>
{
public:
SHAMapTxPlusMetaLeafNode(
std::shared_ptr<SHAMapItem const> item,
std::uint32_t cowid)
: SHAMapLeafNode(std::move(item), cowid)
{
updateHash();
}
SHAMapTxPlusMetaLeafNode(
std::shared_ptr<SHAMapItem const> item,
std::uint32_t cowid,
SHAMapHash const& hash)
: SHAMapLeafNode(std::move(item), cowid, hash)
{
}
std::shared_ptr<SHAMapTreeNode>
clone(std::uint32_t cowid) const override
{
return std::make_shared<SHAMapTxPlusMetaLeafNode>(item_, cowid, hash_);
}
SHAMapNodeType
getType() const override
{
return SHAMapNodeType::tnTRANSACTION_MD;
}
void
updateHash() final override
{
hash_ = SHAMapHash{sha512Half(
HashPrefix::txNode, makeSlice(item_->peekData()), item_->key())};
}
void
serializeForWire(Serializer& s) const final override
{
s.addRaw(item_->peekData());
s.addBitString(item_->key());
s.add8(wireTypeTransactionWithMeta);
}
void
serializeWithPrefix(Serializer& s) const final override
{
s.add32(HashPrefix::txNode);
s.addRaw(item_->peekData());
s.addBitString(item_->key());
}
};
} // namespace ripple
#endif

2
src/ripple/shamap/TreeNodeCache.h
View File

@@ -24,7 +24,7 @@
namespace ripple {
using TreeNodeCache = TaggedCache<uint256, SHAMapAbstractNode>;
using TreeNodeCache = TaggedCache<uint256, SHAMapTreeNode>;
} // namespace ripple

391
src/ripple/shamap/impl/SHAMap.cpp
View File

@@ -19,19 +19,41 @@
#include <ripple/basics/contract.h>
#include <ripple/shamap/SHAMap.h>
#include <ripple/shamap/SHAMapAccountStateLeafNode.h>
#include <ripple/shamap/SHAMapNodeID.h>
#include <ripple/shamap/SHAMapSyncFilter.h>
#include <ripple/shamap/SHAMapTxLeafNode.h>
#include <ripple/shamap/SHAMapTxPlusMetaLeafNode.h>
namespace ripple {
SHAMap::SHAMap(SHAMapType t, Family& f)
: f_(f)
, journal_(f.journal())
, seq_(1)
, state_(SHAMapState::Modifying)
, type_(t)
[[nodiscard]] std::shared_ptr<SHAMapLeafNode>
makeTypedLeaf(
SHAMapNodeType type,
std::shared_ptr<SHAMapItem const> item,
std::uint32_t owner)
{
root_ = std::make_shared<SHAMapInnerNode>(seq_);
if (type == SHAMapNodeType::tnTRANSACTION_NM)
return std::make_shared<SHAMapTxLeafNode>(std::move(item), owner);
if (type == SHAMapNodeType::tnTRANSACTION_MD)
return std::make_shared<SHAMapTxPlusMetaLeafNode>(
std::move(item), owner);
if (type == SHAMapNodeType::tnACCOUNT_STATE)
return std::make_shared<SHAMapAccountStateLeafNode>(
std::move(item), owner);
LogicError(
"Attempt to create leaf node of unknown type " +
std::to_string(
static_cast<std::underlying_type_t<SHAMapNodeType>>(type)));
}
SHAMap::SHAMap(SHAMapType t, Family& f)
: f_(f), journal_(f.journal()), state_(SHAMapState::Modifying), type_(t)
{
root_ = std::make_shared<SHAMapInnerNode>(cowid_);
}
// The `hash` parameter is unused. It is part of the interface so it's clear
@@ -39,18 +61,9 @@ SHAMap::SHAMap(SHAMapType t, Family& f)
// known. The fact that the parameter is unused is an implementation detail that
// should not change the interface.
SHAMap::SHAMap(SHAMapType t, uint256 const& hash, Family& f)
: f_(f)
, journal_(f.journal())
, seq_(1)
, state_(SHAMapState::Synching)
, type_(t)
: f_(f), journal_(f.journal()), state_(SHAMapState::Synching), type_(t)
{
root_ = std::make_shared<SHAMapInnerNode>(seq_);
}
SHAMap::~SHAMap()
{
state_ = SHAMapState::Invalid;
root_ = std::make_shared<SHAMapInnerNode>(cowid_);
}
std::shared_ptr<SHAMap>
@@ -62,7 +75,7 @@ SHAMap::snapShot(bool isMutable) const
if (!isMutable)
newMap.state_ = SHAMapState::Immutable;
newMap.seq_ = seq_ + 1;
newMap.cowid_ = cowid_ + 1;
newMap.ledgerSeq_ = ledgerSeq_;
newMap.root_ = root_;
newMap.backed_ = backed_;
@@ -81,7 +94,7 @@ void
SHAMap::dirtyUp(
SharedPtrNodeStack& stack,
uint256 const& target,
std::shared_ptr<SHAMapAbstractNode> child)
std::shared_ptr<SHAMapTreeNode> child)
{
// walk the tree up from through the inner nodes to the root_
// update hashes and links
@@ -91,7 +104,7 @@ SHAMap::dirtyUp(
assert(
(state_ != SHAMapState::Synching) &&
(state_ != SHAMapState::Immutable));
assert(child && (child->getSeq() == seq_));
assert(child && (child->cowid() == cowid_));
while (!stack.empty())
{
@@ -111,7 +124,7 @@ SHAMap::dirtyUp(
}
}
SHAMapTreeNode*
SHAMapLeafNode*
SHAMap::walkTowardsKey(uint256 const& id, SharedPtrNodeStack* stack) const
{
assert(stack == nullptr || stack->empty());
@@ -134,22 +147,22 @@ SHAMap::walkTowardsKey(uint256 const& id, SharedPtrNodeStack* stack) const
if (stack != nullptr)
stack->push({inNode, nodeID});
return static_cast<SHAMapTreeNode*>(inNode.get());
return static_cast<SHAMapLeafNode*>(inNode.get());
}
SHAMapTreeNode*
SHAMapLeafNode*
SHAMap::findKey(uint256 const& id) const
{
SHAMapTreeNode* leaf = walkTowardsKey(id);
SHAMapLeafNode* leaf = walkTowardsKey(id);
if (leaf && leaf->peekItem()->key() != id)
leaf = nullptr;
return leaf;
}
std::shared_ptr<SHAMapAbstractNode>
std::shared_ptr<SHAMapTreeNode>
SHAMap::fetchNodeFromDB(SHAMapHash const& hash) const
{
std::shared_ptr<SHAMapAbstractNode> node;
std::shared_ptr<SHAMapTreeNode> node;
if (backed_)
{
@@ -158,7 +171,7 @@ SHAMap::fetchNodeFromDB(SHAMapHash const& hash) const
{
try
{
node = SHAMapAbstractNode::makeFromPrefix(
node = SHAMapTreeNode::makeFromPrefix(
makeSlice(nodeObject->getData()), hash);
if (node)
canonicalize(hash, node);
@@ -166,13 +179,13 @@ SHAMap::fetchNodeFromDB(SHAMapHash const& hash) const
catch (std::exception const&)
{
JLOG(journal_.warn()) << "Invalid DB node " << hash;
return std::shared_ptr<SHAMapTreeNode>();
return std::shared_ptr<SHAMapLeafNode>();
}
}
else if (full_)
{
full_ = false;
f_.missingNode(ledgerSeq_);
const_cast<bool&>(full_) = false;
}
}
@@ -180,7 +193,7 @@ SHAMap::fetchNodeFromDB(SHAMapHash const& hash) const
}
// See if a sync filter has a node
std::shared_ptr<SHAMapAbstractNode>
std::shared_ptr<SHAMapTreeNode>
SHAMap::checkFilter(SHAMapHash const& hash, SHAMapSyncFilter* filter) const
{
if (auto nodeData = filter->getNode(hash))
@@ -188,7 +201,7 @@ SHAMap::checkFilter(SHAMapHash const& hash, SHAMapSyncFilter* filter) const
try
{
auto node =
SHAMapAbstractNode::makeFromPrefix(makeSlice(*nodeData), hash);
SHAMapTreeNode::makeFromPrefix(makeSlice(*nodeData), hash);
if (node)
{
filter->gotNode(
@@ -213,10 +226,10 @@ SHAMap::checkFilter(SHAMapHash const& hash, SHAMapSyncFilter* filter) const
// Get a node without throwing
// Used on maps where missing nodes are expected
std::shared_ptr<SHAMapAbstractNode>
std::shared_ptr<SHAMapTreeNode>
SHAMap::fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter* filter) const
{
std::shared_ptr<SHAMapAbstractNode> node = getCache(hash);
auto node = cacheLookup(hash);
if (node)
return node;
@@ -236,10 +249,10 @@ SHAMap::fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter* filter) const
return node;
}
std::shared_ptr<SHAMapAbstractNode>
std::shared_ptr<SHAMapTreeNode>
SHAMap::fetchNodeNT(SHAMapHash const& hash) const
{
auto node = getCache(hash);
auto node = cacheLookup(hash);
if (!node && backed_)
node = fetchNodeFromDB(hash);
@@ -248,7 +261,7 @@ SHAMap::fetchNodeNT(SHAMapHash const& hash) const
}
// Throw if the node is missing
std::shared_ptr<SHAMapAbstractNode>
std::shared_ptr<SHAMapTreeNode>
SHAMap::fetchNode(SHAMapHash const& hash) const
{
auto node = fetchNodeNT(hash);
@@ -259,10 +272,10 @@ SHAMap::fetchNode(SHAMapHash const& hash) const
return node;
}
SHAMapAbstractNode*
SHAMapTreeNode*
SHAMap::descendThrow(SHAMapInnerNode* parent, int branch) const
{
SHAMapAbstractNode* ret = descend(parent, branch);
SHAMapTreeNode* ret = descend(parent, branch);
if (!ret && !parent->isEmptyBranch(branch))
Throw<SHAMapMissingNode>(type_, parent->getChildHash(branch));
@@ -270,11 +283,11 @@ SHAMap::descendThrow(SHAMapInnerNode* parent, int branch) const
return ret;
}
std::shared_ptr<SHAMapAbstractNode>
std::shared_ptr<SHAMapTreeNode>
SHAMap::descendThrow(std::shared_ptr<SHAMapInnerNode> const& parent, int branch)
const
{
std::shared_ptr<SHAMapAbstractNode> ret = descend(parent, branch);
std::shared_ptr<SHAMapTreeNode> ret = descend(parent, branch);
if (!ret && !parent->isEmptyBranch(branch))
Throw<SHAMapMissingNode>(type_, parent->getChildHash(branch));
@@ -282,14 +295,14 @@ SHAMap::descendThrow(std::shared_ptr<SHAMapInnerNode> const& parent, int branch)
return ret;
}
SHAMapAbstractNode*
SHAMapTreeNode*
SHAMap::descend(SHAMapInnerNode* parent, int branch) const
{
SHAMapAbstractNode* ret = parent->getChildPointer(branch);
SHAMapTreeNode* ret = parent->getChildPointer(branch);
if (ret || !backed_)
return ret;
std::shared_ptr<SHAMapAbstractNode> node =
std::shared_ptr<SHAMapTreeNode> node =
fetchNodeNT(parent->getChildHash(branch));
if (!node)
return nullptr;
@@ -298,11 +311,11 @@ SHAMap::descend(SHAMapInnerNode* parent, int branch) const
return node.get();
}
std::shared_ptr<SHAMapAbstractNode>
std::shared_ptr<SHAMapTreeNode>
SHAMap::descend(std::shared_ptr<SHAMapInnerNode> const& parent, int branch)
const
{
std::shared_ptr<SHAMapAbstractNode> node = parent->getChild(branch);
std::shared_ptr<SHAMapTreeNode> node = parent->getChild(branch);
if (node || !backed_)
return node;
@@ -316,18 +329,18 @@ SHAMap::descend(std::shared_ptr<SHAMapInnerNode> const& parent, int branch)
// Gets the node that would be hooked to this branch,
// but doesn't hook it up.
std::shared_ptr<SHAMapAbstractNode>
std::shared_ptr<SHAMapTreeNode>
SHAMap::descendNoStore(
std::shared_ptr<SHAMapInnerNode> const& parent,
int branch) const
{
std::shared_ptr<SHAMapAbstractNode> ret = parent->getChild(branch);
std::shared_ptr<SHAMapTreeNode> ret = parent->getChild(branch);
if (!ret && backed_)
ret = fetchNode(parent->getChildHash(branch));
return ret;
}
std::pair<SHAMapAbstractNode*, SHAMapNodeID>
std::pair<SHAMapTreeNode*, SHAMapNodeID>
SHAMap::descend(
SHAMapInnerNode* parent,
SHAMapNodeID const& parentID,
@@ -338,12 +351,12 @@ SHAMap::descend(
assert((branch >= 0) && (branch < branchFactor));
assert(!parent->isEmptyBranch(branch));
SHAMapAbstractNode* child = parent->getChildPointer(branch);
SHAMapTreeNode* child = parent->getChildPointer(branch);
if (!child)
{
auto const& childHash = parent->getChildHash(branch);
std::shared_ptr<SHAMapAbstractNode> childNode =
std::shared_ptr<SHAMapTreeNode> childNode =
fetchNodeNT(childHash, filter);
if (childNode)
@@ -356,7 +369,7 @@ SHAMap::descend(
return std::make_pair(child, parentID.getChildNodeID(branch));
}
SHAMapAbstractNode*
SHAMapTreeNode*
SHAMap::descendAsync(
SHAMapInnerNode* parent,
int branch,
@@ -365,13 +378,13 @@ SHAMap::descendAsync(
{
pending = false;
SHAMapAbstractNode* ret = parent->getChildPointer(branch);
SHAMapTreeNode* ret = parent->getChildPointer(branch);
if (ret)
return ret;
auto const& hash = parent->getChildHash(branch);
std::shared_ptr<SHAMapAbstractNode> ptr = getCache(hash);
auto ptr = cacheLookup(hash);
if (!ptr)
{
if (filter)
@@ -388,8 +401,8 @@ SHAMap::descendAsync(
if (!obj)
return nullptr;
ptr = SHAMapAbstractNode::makeFromPrefix(
makeSlice(obj->getData()), hash);
ptr =
SHAMapTreeNode::makeFromPrefix(makeSlice(obj->getData()), hash);
if (ptr && backed_)
canonicalize(hash, ptr);
}
@@ -406,28 +419,28 @@ std::shared_ptr<Node>
SHAMap::unshareNode(std::shared_ptr<Node> node, SHAMapNodeID const& nodeID)
{
// make sure the node is suitable for the intended operation (copy on write)
assert(node->getSeq() <= seq_);
if (node->getSeq() != seq_)
assert(node->cowid() <= cowid_);
if (node->cowid() != cowid_)
{
// have a CoW
assert(state_ != SHAMapState::Immutable);
node = std::static_pointer_cast<Node>(node->clone(seq_));
node = std::static_pointer_cast<Node>(node->clone(cowid_));
if (nodeID.isRoot())
root_ = node;
}
return node;
}
SHAMapTreeNode*
SHAMapLeafNode*
SHAMap::firstBelow(
std::shared_ptr<SHAMapAbstractNode> node,
std::shared_ptr<SHAMapTreeNode> node,
SharedPtrNodeStack& stack,
int branch) const
{
// Return the first item at or below this node
if (node->isLeaf())
{
auto n = std::static_pointer_cast<SHAMapTreeNode>(node);
auto n = std::static_pointer_cast<SHAMapLeafNode>(node);
stack.push({node, {leafDepth, n->peekItem()->key()}});
return n.get();
}
@@ -444,7 +457,7 @@ SHAMap::firstBelow(
assert(!stack.empty());
if (node->isLeaf())
{
auto n = std::static_pointer_cast<SHAMapTreeNode>(node);
auto n = std::static_pointer_cast<SHAMapLeafNode>(node);
stack.push({n, {leafDepth, n->peekItem()->key()}});
return n.get();
}
@@ -461,13 +474,13 @@ SHAMap::firstBelow(
static const std::shared_ptr<SHAMapItem const> no_item;
std::shared_ptr<SHAMapItem const> const&
SHAMap::onlyBelow(SHAMapAbstractNode* node) const
SHAMap::onlyBelow(SHAMapTreeNode* node) const
{
// If there is only one item below this node, return it
while (!node->isLeaf())
{
SHAMapAbstractNode* nextNode = nullptr;
SHAMapTreeNode* nextNode = nullptr;
auto inner = static_cast<SHAMapInnerNode*>(node);
for (int i = 0; i < branchFactor; ++i)
{
@@ -491,17 +504,16 @@ SHAMap::onlyBelow(SHAMapAbstractNode* node) const
// An inner node must have at least one leaf
// below it, unless it's the root_
auto leaf = static_cast<SHAMapTreeNode*>(node);
assert(leaf->hasItem() || (leaf == root_.get()));
auto const leaf = static_cast<SHAMapLeafNode const*>(node);
assert(leaf->peekItem() || (leaf == root_.get()));
return leaf->peekItem();
}
SHAMapTreeNode const*
SHAMapLeafNode const*
SHAMap::peekFirstItem(SharedPtrNodeStack& stack) const
{
assert(stack.empty());
SHAMapTreeNode* node = firstBelow(root_, stack);
SHAMapLeafNode* node = firstBelow(root_, stack);
if (!node)
{
while (!stack.empty())
@@ -511,7 +523,7 @@ SHAMap::peekFirstItem(SharedPtrNodeStack& stack) const
return node;
}
SHAMapTreeNode const*
SHAMapLeafNode const*
SHAMap::peekNextItem(uint256 const& id, SharedPtrNodeStack& stack) const
{
assert(!stack.empty());
@@ -543,7 +555,7 @@ SHAMap::peekNextItem(uint256 const& id, SharedPtrNodeStack& stack) const
std::shared_ptr<SHAMapItem const> const&
SHAMap::peekItem(uint256 const& id) const
{
SHAMapTreeNode* leaf = findKey(id);
SHAMapLeafNode* leaf = findKey(id);
if (!leaf)
return no_item;
@@ -551,27 +563,15 @@ SHAMap::peekItem(uint256 const& id) const
return leaf->peekItem();
}
std::shared_ptr<SHAMapItem const> const&
SHAMap::peekItem(uint256 const& id, SHAMapTreeNode::TNType& type) const
{
SHAMapTreeNode* leaf = findKey(id);
if (!leaf)
return no_item;
type = leaf->getType();
return leaf->peekItem();
}
std::shared_ptr<SHAMapItem const> const&
SHAMap::peekItem(uint256 const& id, SHAMapHash& hash) const
{
SHAMapTreeNode* leaf = findKey(id);
SHAMapLeafNode* leaf = findKey(id);
if (!leaf)
return no_item;
hash = leaf->getNodeHash();
hash = leaf->getHash();
return leaf->peekItem();
}
@@ -587,7 +587,7 @@ SHAMap::upper_bound(uint256 const& id) const
auto [node, nodeID] = stack.top();
if (node->isLeaf())
{
auto leaf = static_cast<SHAMapTreeNode*>(node.get());
auto leaf = static_cast<SHAMapLeafNode*>(node.get());
if (leaf->peekItem()->key() > id)
return const_iterator(
this, leaf->peekItem().get(), std::move(stack));
@@ -618,9 +618,7 @@ SHAMap::upper_bound(uint256 const& id) const
bool
SHAMap::hasItem(uint256 const& id) const
{
// does the tree have an item with this ID
SHAMapTreeNode* leaf = findKey(id);
return (leaf != nullptr);
return (findKey(id) != nullptr);
}
bool
@@ -635,17 +633,17 @@ SHAMap::delItem(uint256 const& id)
if (stack.empty())
Throw<SHAMapMissingNode>(type_, id);
auto leaf = std::dynamic_pointer_cast<SHAMapTreeNode>(stack.top().first);
auto leaf = std::dynamic_pointer_cast<SHAMapLeafNode>(stack.top().first);
stack.pop();
if (!leaf || (leaf->peekItem()->key() != id))
return false;
SHAMapTreeNode::TNType type = leaf->getType();
SHAMapNodeType type = leaf->getType();
// What gets attached to the end of the chain
// (For now, nothing, since we deleted the leaf)
std::shared_ptr<SHAMapAbstractNode> prevNode;
std::shared_ptr<SHAMapTreeNode> prevNode;
while (!stack.empty())
{
@@ -682,8 +680,8 @@ SHAMap::delItem(uint256 const& id)
break;
}
}
prevNode = std::make_shared<SHAMapTreeNode>(
item, type, node->getSeq());
prevNode = makeTypedLeaf(type, item, node->cowid());
}
else
{
@@ -702,19 +700,13 @@ SHAMap::delItem(uint256 const& id)
}
bool
SHAMap::addGiveItem(
std::shared_ptr<SHAMapItem const> item,
bool isTransaction,
bool hasMeta)
SHAMap::addGiveItem(SHAMapNodeType type, std::shared_ptr<SHAMapItem const> item)
{
assert(state_ != SHAMapState::Immutable);
assert(type != SHAMapNodeType::tnINNER);
// add the specified item, does not update
uint256 tag = item->key();
SHAMapTreeNode::TNType type = !isTransaction
? SHAMapTreeNode::tnACCOUNT_STATE
: (hasMeta ? SHAMapTreeNode::tnTRANSACTION_MD
: SHAMapTreeNode::tnTRANSACTION_NM);
assert(state_ != SHAMapState::Immutable);
SharedPtrNodeStack stack;
walkTowardsKey(tag, &stack);
@@ -727,7 +719,7 @@ SHAMap::addGiveItem(
if (node->isLeaf())
{
auto leaf = std::static_pointer_cast<SHAMapTreeNode>(node);
auto leaf = std::static_pointer_cast<SHAMapLeafNode>(node);
if (leaf->peekItem()->key() == tag)
return false;
}
@@ -738,21 +730,20 @@ SHAMap::addGiveItem(
auto inner = std::static_pointer_cast<SHAMapInnerNode>(node);
int branch = selectBranch(nodeID, tag);
assert(inner->isEmptyBranch(branch));
auto newNode =
std::make_shared<SHAMapTreeNode>(std::move(item), type, seq_);
auto newNode = makeTypedLeaf(type, std::move(item), cowid_);
inner->setChild(branch, newNode);
}
else
{
// this is a leaf node that has to be made an inner node holding two
// items
auto leaf = std::static_pointer_cast<SHAMapTreeNode>(node);
auto leaf = std::static_pointer_cast<SHAMapLeafNode>(node);
std::shared_ptr<SHAMapItem const> otherItem = leaf->peekItem();
assert(otherItem && (tag != otherItem->key()));
node = std::make_shared<SHAMapInnerNode>(node->getSeq());
node = std::make_shared<SHAMapInnerNode>(node->cowid());
int b1, b2;
unsigned int b1, b2;
while ((b1 = selectBranch(nodeID, tag)) ==
(b2 = selectBranch(nodeID, otherItem->key())))
@@ -762,22 +753,15 @@ SHAMap::addGiveItem(
// we need a new inner node, since both go on same branch at this
// level
nodeID = nodeID.getChildNodeID(b1);
node = std::make_shared<SHAMapInnerNode>(seq_);
node = std::make_shared<SHAMapInnerNode>(cowid_);
}
// we can add the two leaf nodes here
assert(node->isInner());
std::shared_ptr<SHAMapTreeNode> newNode =
std::make_shared<SHAMapTreeNode>(std::move(item), type, seq_);
assert(newNode->isLeaf());
auto inner = std::static_pointer_cast<SHAMapInnerNode>(node);
inner->setChild(b1, newNode);
newNode =
std::make_shared<SHAMapTreeNode>(std::move(otherItem), type, seq_);
assert(newNode->isLeaf());
inner->setChild(b2, newNode);
auto inner = static_cast<SHAMapInnerNode*>(node.get());
inner->setChild(b1, makeTypedLeaf(type, std::move(item), cowid_));
inner->setChild(b2, makeTypedLeaf(type, std::move(otherItem), cowid_));
}
dirtyUp(stack, tag, node);
@@ -785,31 +769,27 @@ SHAMap::addGiveItem(
}
bool
SHAMap::addItem(SHAMapItem&& i, bool isTransaction, bool hasMetaData)
SHAMap::addItem(SHAMapNodeType type, SHAMapItem&& i)
{
return addGiveItem(
std::make_shared<SHAMapItem const>(std::move(i)),
isTransaction,
hasMetaData);
return addGiveItem(type, std::make_shared<SHAMapItem const>(std::move(i)));
}
SHAMapHash
SHAMap::getHash() const
{
auto hash = root_->getNodeHash();
auto hash = root_->getHash();
if (hash.isZero())
{
const_cast<SHAMap&>(*this).unshare();
hash = root_->getNodeHash();
hash = root_->getHash();
}
return hash;
}
bool
SHAMap::updateGiveItem(
std::shared_ptr<SHAMapItem const> item,
bool isTransaction,
bool hasMeta)
SHAMapNodeType type,
std::shared_ptr<SHAMapItem const> item)
{
// can't change the tag but can change the hash
uint256 tag = item->key();
@@ -822,7 +802,7 @@ SHAMap::updateGiveItem(
if (stack.empty())
Throw<SHAMapMissingNode>(type_, tag);
auto node = std::dynamic_pointer_cast<SHAMapTreeNode>(stack.top().first);
auto node = std::dynamic_pointer_cast<SHAMapLeafNode>(stack.top().first);
auto nodeID = stack.top().second;
stack.pop();
@@ -832,26 +812,24 @@ SHAMap::updateGiveItem(
return false;
}
node = unshareNode(std::move(node), nodeID);
if (!node->setItem(
std::move(item),
!isTransaction ? SHAMapTreeNode::tnACCOUNT_STATE
: (hasMeta ? SHAMapTreeNode::tnTRANSACTION_MD
: SHAMapTreeNode::tnTRANSACTION_NM)))
if (node->getType() != type)
{
JLOG(journal_.trace()) << "SHAMap setItem, no change";
return true;
JLOG(journal_.fatal()) << "SHAMap::setItem: cross-type change!";
return false;
}
dirtyUp(stack, tag, node);
node = unshareNode(std::move(node), nodeID);
if (node->setItem(std::move(item)))
dirtyUp(stack, tag, node);
return true;
}
bool
SHAMap::fetchRoot(SHAMapHash const& hash, SHAMapSyncFilter* filter)
{
if (hash == root_->getNodeHash())
if (hash == root_->getHash())
return true;
if (auto stream = journal_.trace())
@@ -875,42 +853,37 @@ SHAMap::fetchRoot(SHAMapHash const& hash, SHAMapSyncFilter* filter)
if (newRoot)
{
root_ = newRoot;
assert(root_->getNodeHash() == hash);
assert(root_->getHash() == hash);
return true;
}
return false;
}
// Replace a node with a shareable node.
//
// This code handles two cases:
//
// 1) An unshared, unshareable node needs to be made shareable
// so immutable SHAMap's can have references to it.
//
// 2) An unshareable node is shared. This happens when you make
// a mutable snapshot of a mutable SHAMap.
std::shared_ptr<SHAMapAbstractNode>
SHAMap::writeNode(
NodeObjectType t,
std::uint32_t seq,
std::shared_ptr<SHAMapAbstractNode> node) const
{
// Node is ours, so we can just make it shareable
assert(node->getSeq() == seq_);
assert(backed_);
node->setSeq(0);
/** Replace a node with a shareable node.
canonicalize(node->getNodeHash(), node);
This code handles two cases:
1) An unshared, unshareable node needs to be made shareable
so immutable SHAMap's can have references to it.
2) An unshareable node is shared. This happens when you make
a mutable snapshot of a mutable SHAMap.
@note The node must have already been unshared by having the caller
first call SHAMapTreeNode::unshare().
*/
std::shared_ptr<SHAMapTreeNode>
SHAMap::writeNode(NodeObjectType t, std::shared_ptr<SHAMapTreeNode> node) const
{
assert(node->cowid() == 0);
assert(backed_);
canonicalize(node->getHash(), node);
Serializer s;
node->serializeWithPrefix(s);
f_.db().store(
t,
std::move(s.modData()),
node->getNodeHash().as_uint256(),
ledgerSeq_);
t, std::move(s.modData()), node->getHash().as_uint256(), ledgerSeq_);
return node;
}
@@ -923,13 +896,13 @@ SHAMap::preFlushNode(std::shared_ptr<Node> node) const
{
// A shared node should never need to be flushed
// because that would imply someone modified it
assert(node->getSeq() != 0);
assert(node->cowid() != 0);
if (node->getSeq() != seq_)
if (node->cowid() != cowid_)
{
// Node is not uniquely ours, so unshare it before
// possibly modifying it
node = std::static_pointer_cast<Node>(node->clone(seq_));
node = std::static_pointer_cast<Node>(node->clone(cowid_));
}
return node;
}
@@ -937,35 +910,36 @@ SHAMap::preFlushNode(std::shared_ptr<Node> node) const
int
SHAMap::unshare()
{
// Don't share nodes wth parent map
return walkSubTree(false, hotUNKNOWN, 0);
}
/** Convert all modified nodes to shared nodes */
// If requested, write them to the node store
int
SHAMap::flushDirty(NodeObjectType t, std::uint32_t seq)
{
return walkSubTree(true, t, seq);
// Don't share nodes with parent map
return walkSubTree(false, hotUNKNOWN);
}
int
SHAMap::walkSubTree(bool doWrite, NodeObjectType t, std::uint32_t seq)
SHAMap::flushDirty(NodeObjectType t)
{
// We only write back if this map is backed.
return walkSubTree(backed_, t);
}
int
SHAMap::walkSubTree(bool doWrite, NodeObjectType t)
{
assert(!doWrite || backed_);
int flushed = 0;
Serializer s;
if (!root_ || (root_->getSeq() == 0))
if (!root_ || (root_->cowid() == 0))
return flushed;
if (root_->isLeaf())
{ // special case -- root_ is leaf
root_ = preFlushNode(std::move(root_));
root_->updateHash();
if (doWrite && backed_)
root_ = writeNode(t, seq, std::move(root_));
else
root_->setSeq(0);
root_->unshare();
if (doWrite)
root_ = writeNode(t, std::move(root_));
return 1;
}
@@ -1002,7 +976,7 @@ SHAMap::walkSubTree(bool doWrite, NodeObjectType t, std::uint32_t seq)
int branch = pos;
auto child = node->getChild(pos++);
if (child && (child->getSeq() != 0))
if (child && (child->cowid() != 0))
{
// This is a node that needs to be flushed
@@ -1025,13 +999,12 @@ SHAMap::walkSubTree(bool doWrite, NodeObjectType t, std::uint32_t seq)
// flush this leaf
++flushed;
assert(node->getSeq() == seq_);
assert(node->cowid() == cowid_);
child->updateHash();
child->unshare();
if (doWrite && backed_)
child = writeNode(t, seq, std::move(child));
else
child->setSeq(0);
if (doWrite)
child = writeNode(t, std::move(child));
node->shareChild(branch, child);
}
@@ -1043,11 +1016,11 @@ SHAMap::walkSubTree(bool doWrite, NodeObjectType t, std::uint32_t seq)
node->updateHashDeep();
// This inner node can now be shared
if (doWrite && backed_)
node->unshare();
if (doWrite)
node = std::static_pointer_cast<SHAMapInnerNode>(
writeNode(t, seq, std::move(node)));
else
node->setSeq(0);
writeNode(t, std::move(node)));
++flushed;
@@ -1059,7 +1032,7 @@ SHAMap::walkSubTree(bool doWrite, NodeObjectType t, std::uint32_t seq)
stack.pop();
// Hook this inner node to its parent
assert(parent->getSeq() == seq_);
assert(parent->cowid() == cowid_);
parent->shareChild(pos, node);
// Continue with parent's next child, if any
@@ -1079,7 +1052,7 @@ SHAMap::dump(bool hash) const
int leafCount = 0;
JLOG(journal_.info()) << " MAP Contains";
std::stack<std::pair<SHAMapAbstractNode*, SHAMapNodeID>> stack;
std::stack<std::pair<SHAMapTreeNode*, SHAMapNodeID>> stack;
stack.push({root_.get(), SHAMapNodeID()});
do
@@ -1090,7 +1063,7 @@ SHAMap::dump(bool hash) const
JLOG(journal_.info()) << node->getString(nodeID);
if (hash)
{
JLOG(journal_.info()) << "Hash: " << node->getNodeHash();
JLOG(journal_.info()) << "Hash: " << node->getHash();
}
if (node->isInner())
@@ -1103,7 +1076,7 @@ SHAMap::dump(bool hash) const
auto child = inner->getChildPointer(i);
if (child)
{
assert(child->getNodeHash() == inner->getChildHash(i));
assert(child->getHash() == inner->getChildHash(i));
stack.push({child, nodeID.getChildNodeID(i)});
}
}
@@ -1116,22 +1089,22 @@ SHAMap::dump(bool hash) const
JLOG(journal_.info()) << leafCount << " resident leaves";
}
std::shared_ptr<SHAMapAbstractNode>
SHAMap::getCache(SHAMapHash const& hash) const
std::shared_ptr<SHAMapTreeNode>
SHAMap::cacheLookup(SHAMapHash const& hash) const
{
auto ret = f_.getTreeNodeCache(ledgerSeq_)->fetch(hash.as_uint256());
assert(!ret || !ret->getSeq());
assert(!ret || !ret->cowid());
return ret;
}
void
SHAMap::canonicalize(
SHAMapHash const& hash,
std::shared_ptr<SHAMapAbstractNode>& node) const
std::shared_ptr<SHAMapTreeNode>& node) const
{
assert(backed_);
assert(node->getSeq() == 0);
assert(node->getNodeHash() == hash);
assert(node->cowid() == 0);
assert(node->getHash() == hash);
f_.getTreeNodeCache(ledgerSeq_)
->canonicalize_replace_client(hash.as_uint256(), node);

23
src/ripple/shamap/impl/SHAMapDelta.cpp
View File

@@ -32,7 +32,7 @@ namespace ripple {
bool
SHAMap::walkBranch(
SHAMapAbstractNode* node,
SHAMapTreeNode* node,
std::shared_ptr<SHAMapItem const> const& otherMapItem,
bool isFirstMap,
Delta& differences,
@@ -40,7 +40,7 @@ SHAMap::walkBranch(
{
// Walk a branch of a SHAMap that's matched by an empty branch or single
// item in the other map
std::stack<SHAMapAbstractNode*, std::vector<SHAMapAbstractNode*>> nodeStack;
std::stack<SHAMapTreeNode*, std::vector<SHAMapTreeNode*>> nodeStack;
nodeStack.push(node);
bool emptyBranch = !otherMapItem;
@@ -61,7 +61,7 @@ SHAMap::walkBranch(
else
{
// This is a leaf node, process its item
auto item = static_cast<SHAMapTreeNode*>(node)->peekItem();
auto item = static_cast<SHAMapLeafNode*>(node)->peekItem();
if (emptyBranch || (item->key() != otherMapItem->key()))
{
@@ -133,7 +133,7 @@ SHAMap::compare(SHAMap const& otherMap, Delta& differences, int maxCount) const
if (getHash() == otherMap.getHash())
return true;
using StackEntry = std::pair<SHAMapAbstractNode*, SHAMapAbstractNode*>;
using StackEntry = std::pair<SHAMapTreeNode*, SHAMapTreeNode*>;
std::stack<StackEntry, std::vector<StackEntry>>
nodeStack; // track nodes we've pushed
@@ -152,8 +152,8 @@ SHAMap::compare(SHAMap const& otherMap, Delta& differences, int maxCount) const
if (ourNode->isLeaf() && otherNode->isLeaf())
{
// two leaves
auto ours = static_cast<SHAMapTreeNode*>(ourNode);
auto other = static_cast<SHAMapTreeNode*>(otherNode);
auto ours = static_cast<SHAMapLeafNode*>(ourNode);
auto other = static_cast<SHAMapLeafNode*>(otherNode);
if (ours->peekItem()->key() == other->peekItem()->key())
{
if (ours->peekItem()->peekData() !=
@@ -188,14 +188,14 @@ SHAMap::compare(SHAMap const& otherMap, Delta& differences, int maxCount) const
else if (ourNode->isInner() && otherNode->isLeaf())
{
auto ours = static_cast<SHAMapInnerNode*>(ourNode);
auto other = static_cast<SHAMapTreeNode*>(otherNode);
auto other = static_cast<SHAMapLeafNode*>(otherNode);
if (!walkBranch(
ours, other->peekItem(), true, differences, maxCount))
return false;
}
else if (ourNode->isLeaf() && otherNode->isInner())
{
auto ours = static_cast<SHAMapTreeNode*>(ourNode);
auto ours = static_cast<SHAMapLeafNode*>(ourNode);
auto other = static_cast<SHAMapInnerNode*>(otherNode);
if (!otherMap.walkBranch(
other, ours->peekItem(), false, differences, maxCount))
@@ -211,7 +211,7 @@ SHAMap::compare(SHAMap const& otherMap, Delta& differences, int maxCount) const
if (other->isEmptyBranch(i))
{
// We have a branch, the other tree does not
SHAMapAbstractNode* iNode = descendThrow(ours, i);
SHAMapTreeNode* iNode = descendThrow(ours, i);
if (!walkBranch(
iNode,
std::shared_ptr<SHAMapItem const>(),
@@ -223,8 +223,7 @@ SHAMap::compare(SHAMap const& otherMap, Delta& differences, int maxCount) const
else if (ours->isEmptyBranch(i))
{
// The other tree has a branch, we do not
SHAMapAbstractNode* iNode =
otherMap.descendThrow(other, i);
SHAMapTreeNode* iNode = otherMap.descendThrow(other, i);
if (!otherMap.walkBranch(
iNode,
std::shared_ptr<SHAMapItem const>(),
@@ -267,7 +266,7 @@ SHAMap::walkMap(std::vector<SHAMapMissingNode>& missingNodes, int maxMissing)
{
if (!node->isEmptyBranch(i))
{
std::shared_ptr<SHAMapAbstractNode> nextNode =
std::shared_ptr<SHAMapTreeNode> nextNode =
descendNoStore(node, i);
if (nextNode)

313
src/ripple/shamap/impl/SHAMapInnerNode.cpp Normal file
View File

@@ -0,0 +1,313 @@
//------------------------------------------------------------------------------
/*
This file is part of rippled: https://github.com/ripple/rippled
Copyright (c) 2012, 2013 Ripple Labs Inc.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include <ripple/shamap/SHAMapInnerNode.h>
#include <ripple/basics/ByteUtilities.h>
#include <ripple/basics/Log.h>
#include <ripple/basics/Slice.h>
#include <ripple/basics/contract.h>
#include <ripple/basics/safe_cast.h>
#include <ripple/beast/core/LexicalCast.h>
#include <ripple/protocol/HashPrefix.h>
#include <ripple/protocol/digest.h>
#include <ripple/shamap/SHAMapTreeNode.h>
#include <openssl/sha.h>
#include <algorithm>
#include <array>
#include <atomic>
#include <iterator>
#include <mutex>
#include <utility>
namespace ripple {
std::mutex SHAMapInnerNode::childLock;
std::shared_ptr<SHAMapTreeNode>
SHAMapInnerNode::clone(std::uint32_t cowid) const
{
auto p = std::make_shared<SHAMapInnerNode>(cowid);
p->hash_ = hash_;
p->mIsBranch = mIsBranch;
p->mFullBelowGen = mFullBelowGen;
p->mHashes = mHashes;
std::lock_guard lock(childLock);
for (int i = 0; i < 16; ++i)
p->mChildren[i] = mChildren[i];
return p;
}
std::shared_ptr<SHAMapTreeNode>
SHAMapInnerNode::makeFullInner(
Slice data,
SHAMapHash const& hash,
bool hashValid)
{
if (data.size() != 512)
Throw<std::runtime_error>("Invalid FI node");
auto ret = std::make_shared<SHAMapInnerNode>(0);
Serializer s(data.data(), data.size());
for (int i = 0; i < 16; ++i)
{
s.getBitString(ret->mHashes[i].as_uint256(), i * 32);
if (ret->mHashes[i].isNonZero())
ret->mIsBranch |= (1 << i);
}
if (hashValid)
ret->hash_ = hash;
else
ret->updateHash();
return ret;
}
std::shared_ptr<SHAMapTreeNode>
SHAMapInnerNode::makeCompressedInner(Slice data)
{
Serializer s(data.data(), data.size());
int len = s.getLength();
auto ret = std::make_shared<SHAMapInnerNode>(0);
for (int i = 0; i < (len / 33); ++i)
{
int pos;
if (!s.get8(pos, 32 + (i * 33)))
Throw<std::runtime_error>("short CI node");
if ((pos < 0) || (pos >= 16))
Throw<std::runtime_error>("invalid CI node");
s.getBitString(ret->mHashes[pos].as_uint256(), i * 33);
if (ret->mHashes[pos].isNonZero())
ret->mIsBranch |= (1 << pos);
}
ret->updateHash();
return ret;
}
void
SHAMapInnerNode::updateHash()
{
uint256 nh;
if (mIsBranch != 0)
{
sha512_half_hasher h;
using beast::hash_append;
hash_append(h, HashPrefix::innerNode);
for (auto const& hh : mHashes)
hash_append(h, hh);
nh = static_cast<typename sha512_half_hasher::result_type>(h);
}
hash_ = SHAMapHash{nh};
}
void
SHAMapInnerNode::updateHashDeep()
{
for (auto pos = 0; pos < 16; ++pos)
{
if (mChildren[pos] != nullptr)
mHashes[pos] = mChildren[pos]->getHash();
}
updateHash();
}
void
SHAMapInnerNode::serializeForWire(Serializer& s) const
{
assert(!isEmpty());
// If the node is sparse, then only send non-empty branches:
if (getBranchCount() < 12)
{
// compressed node
for (int i = 0; i < mHashes.size(); ++i)
{
if (!isEmptyBranch(i))
{
s.addBitString(mHashes[i].as_uint256());
s.add8(i);
}
}
s.add8(wireTypeCompressedInner);
}
else
{
for (auto const& hh : mHashes)
s.addBitString(hh.as_uint256());
s.add8(wireTypeInner);
}
}
void
SHAMapInnerNode::serializeWithPrefix(Serializer& s) const
{
assert(!isEmpty());
s.add32(HashPrefix::innerNode);
for (auto const& hh : mHashes)
s.addBitString(hh.as_uint256());
}
bool
SHAMapInnerNode::isEmpty() const
{
return mIsBranch == 0;
}
int
SHAMapInnerNode::getBranchCount() const
{
int count = 0;
for (int i = 0; i < 16; ++i)
if (!isEmptyBranch(i))
++count;
return count;
}
std::string
SHAMapInnerNode::getString(const SHAMapNodeID& id) const
{
std::string ret = SHAMapTreeNode::getString(id);
for (int i = 0; i < mHashes.size(); ++i)
{
if (!isEmptyBranch(i))
{
ret += "\n";
ret += std::to_string(i);
ret += " = ";
ret += to_string(mHashes[i]);
}
}
return ret;
}
// We are modifying an inner node
void
SHAMapInnerNode::setChild(int m, std::shared_ptr<SHAMapTreeNode> const& child)
{
assert((m >= 0) && (m < 16));
assert(cowid_ != 0);
assert(child.get() != this);
mHashes[m].zero();
hash_.zero();
if (child)
mIsBranch |= (1 << m);
else
mIsBranch &= ~(1 << m);
mChildren[m] = child;
}
// finished modifying, now make shareable
void
SHAMapInnerNode::shareChild(int m, std::shared_ptr<SHAMapTreeNode> const& child)
{
assert((m >= 0) && (m < 16));
assert(cowid_ != 0);
assert(child);
assert(child.get() != this);
mChildren[m] = child;
}
SHAMapTreeNode*
SHAMapInnerNode::getChildPointer(int branch)
{
assert(branch >= 0 && branch < 16);
std::lock_guard lock(childLock);
return mChildren[branch].get();
}
std::shared_ptr<SHAMapTreeNode>
SHAMapInnerNode::getChild(int branch)
{
assert(branch >= 0 && branch < 16);
std::lock_guard lock(childLock);
return mChildren[branch];
}
std::shared_ptr<SHAMapTreeNode>
SHAMapInnerNode::canonicalizeChild(
int branch,
std::shared_ptr<SHAMapTreeNode> node)
{
assert(branch >= 0 && branch < 16);
assert(node);
assert(node->getHash() == mHashes[branch]);
std::lock_guard lock(childLock);
if (mChildren[branch])
{
// There is already a node hooked up, return it
node = mChildren[branch];
}
else
{
// Hook this node up
mChildren[branch] = node;
}
return node;
}
void
SHAMapInnerNode::invariants(bool is_root) const
{
unsigned count = 0;
for (int i = 0; i < 16; ++i)
{
if (mHashes[i].isNonZero())
{
assert((mIsBranch & (1 << i)) != 0);
if (mChildren[i] != nullptr)
mChildren[i]->invariants();
++count;
}
else
{
assert((mIsBranch & (1 << i)) == 0);
}
}
if (!is_root)
{
assert(hash_.isNonZero());
assert(count >= 1);
}
assert((count == 0) ? hash_.isZero() : hash_.isNonZero());
}
} // namespace ripple

94
src/ripple/shamap/impl/SHAMapLeafNode.cpp Normal file
View File

@@ -0,0 +1,94 @@
//------------------------------------------------------------------------------
/*
This file is part of rippled: https://github.com/ripple/rippled
Copyright (c) 2012, 2013 Ripple Labs Inc.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include <ripple/basics/contract.h>
#include <ripple/beast/core/LexicalCast.h>
#include <ripple/shamap/SHAMapLeafNode.h>
namespace ripple {
SHAMapLeafNode::SHAMapLeafNode(
std::shared_ptr<SHAMapItem const> item,
std::uint32_t cowid)
: SHAMapTreeNode(cowid), item_(std::move(item))
{
assert(item_->peekData().size() >= 12);
}
SHAMapLeafNode::SHAMapLeafNode(
std::shared_ptr<SHAMapItem const> item,
std::uint32_t cowid,
SHAMapHash const& hash)
: SHAMapTreeNode(cowid, hash), item_(std::move(item))
{
assert(item_->peekData().size() >= 12);
}
std::shared_ptr<SHAMapItem const> const&
SHAMapLeafNode::peekItem() const
{
return item_;
}
bool
SHAMapLeafNode::setItem(std::shared_ptr<SHAMapItem const> i)
{
assert(cowid_ != 0);
item_ = std::move(i);
auto const oldHash = hash_;
updateHash();
return (oldHash != hash_);
}
std::string
SHAMapLeafNode::getString(const SHAMapNodeID& id) const
{
std::string ret = SHAMapTreeNode::getString(id);
auto const type = getType();
if (type == SHAMapNodeType::tnTRANSACTION_NM)
ret += ",txn\n";
else if (type == SHAMapNodeType::tnTRANSACTION_MD)
ret += ",txn+md\n";
else if (type == SHAMapNodeType::tnACCOUNT_STATE)
ret += ",as\n";
else
ret += ",leaf\n";
ret += " Tag=";
ret += to_string(item_->key());
ret += "\n Hash=";
ret += to_string(hash_);
ret += "/";
ret += std::to_string(item_->size());
return ret;
}
void
SHAMapLeafNode::invariants(bool) const
{
assert(hash_.isNonZero());
assert(item_ != nullptr);
}
} // namespace ripple

108
src/ripple/shamap/impl/SHAMapSync.cpp
View File

@@ -28,16 +28,15 @@ SHAMap::visitLeaves(
std::function<void(std::shared_ptr<SHAMapItem const> const& item)> const&
leafFunction) const
{
visitNodes([&leafFunction](SHAMapAbstractNode& node) {
visitNodes([&leafFunction](SHAMapTreeNode& node) {
if (!node.isInner())
leafFunction(static_cast<SHAMapTreeNode&>(node).peekItem());
leafFunction(static_cast<SHAMapLeafNode&>(node).peekItem());
return true;
});
}
void
SHAMap::visitNodes(
std::function<bool(SHAMapAbstractNode&)> const& function) const
SHAMap::visitNodes(std::function<bool(SHAMapTreeNode&)> const& function) const
{
if (!root_)
return;
@@ -60,7 +59,7 @@ SHAMap::visitNodes(
uint256 childHash;
if (!node->isEmptyBranch(pos))
{
std::shared_ptr<SHAMapAbstractNode> child =
std::shared_ptr<SHAMapTreeNode> child =
descendNoStore(node, pos);
if (!function(*child))
return;
@@ -101,24 +100,24 @@ SHAMap::visitNodes(
void
SHAMap::visitDifferences(
SHAMap const* have,
std::function<bool(SHAMapAbstractNode&)> function) const
std::function<bool(SHAMapTreeNode const&)> function) const
{
// Visit every node in this SHAMap that is not present
// in the specified SHAMap
if (!root_)
return;
if (root_->getNodeHash().isZero())
if (root_->getHash().isZero())
return;
if (have && (root_->getNodeHash() == have->root_->getNodeHash()))
if (have && (root_->getHash() == have->root_->getHash()))
return;
if (root_->isLeaf())
{
auto leaf = std::static_pointer_cast<SHAMapTreeNode>(root_);
auto leaf = std::static_pointer_cast<SHAMapLeafNode>(root_);
if (!have ||
!have->hasLeafNode(leaf->peekItem()->key(), leaf->getNodeHash()))
!have->hasLeafNode(leaf->peekItem()->key(), leaf->getHash()))
function(*root_);
return;
}
@@ -155,7 +154,7 @@ SHAMap::visitDifferences(
else if (
!have ||
!have->hasLeafNode(
static_cast<SHAMapTreeNode*>(next)->peekItem()->key(),
static_cast<SHAMapLeafNode*>(next)->peekItem()->key(),
childHash))
{
if (!function(*next))
@@ -242,7 +241,7 @@ SHAMap::gmn_ProcessNodes(MissingNodes& mn, MissingNodes::StackEntry& se)
if (backed_)
{
f_.getFullBelowCache(ledgerSeq_)
->insert(node->getNodeHash().as_uint256());
->insert(node->getHash().as_uint256());
}
}
@@ -315,7 +314,7 @@ SHAMap::gmn_ProcessDeferredReads(MissingNodes& mn)
std::vector<std::pair<SHAMapNodeID, uint256>>
SHAMap::getMissingNodes(int max, SHAMapSyncFilter* filter)
{
assert(root_->getNodeHash().isNonZero());
assert(root_->getHash().isNonZero());
assert(max > 0);
MissingNodes mn(
@@ -423,23 +422,9 @@ SHAMap::getMissingNodes(int max, SHAMapSyncFilter* filter)
return std::move(mn.missingNodes_);
}
std::vector<uint256>
SHAMap::getNeededHashes(int max, SHAMapSyncFilter* filter)
{
auto ret = getMissingNodes(max, filter);
std::vector<uint256> hashes;
hashes.reserve(ret.size());
for (auto const& n : ret)
hashes.push_back(n.second);
return hashes;
}
bool
SHAMap::getNodeFat(
SHAMapNodeID wanted,
SHAMapNodeID const& wanted,
std::vector<SHAMapNodeID>& nodeIDs,
std::vector<Blob>& rawNodes,
bool fatLeaves,
@@ -476,7 +461,7 @@ SHAMap::getNodeFat(
return false;
}
std::stack<std::tuple<SHAMapAbstractNode*, SHAMapNodeID, int>> stack;
std::stack<std::tuple<SHAMapTreeNode*, SHAMapNodeID, int>> stack;
stack.emplace(node, nodeID, depth);
while (!stack.empty())
@@ -547,16 +532,16 @@ SHAMap::addRootNode(
SHAMapSyncFilter* filter)
{
// we already have a root_ node
if (root_->getNodeHash().isNonZero())
if (root_->getHash().isNonZero())
{
JLOG(journal_.trace()) << "got root node, already have one";
assert(root_->getNodeHash() == hash);
assert(root_->getHash() == hash);
return SHAMapAddNode::duplicate();
}
assert(seq_ >= 1);
auto node = SHAMapAbstractNode::makeFromWire(rootNode);
if (!node || node->getNodeHash() != hash)
assert(cowid_ >= 1);
auto node = SHAMapTreeNode::makeFromWire(rootNode);
if (!node || node->getHash() != hash)
return SHAMapAddNode::invalid();
if (backed_)
@@ -573,7 +558,7 @@ SHAMap::addRootNode(
root_->serializeWithPrefix(s);
filter->gotNode(
false,
root_->getNodeHash(),
root_->getHash(),
ledgerSeq_,
std::move(s.modData()),
root_->getType());
@@ -597,7 +582,7 @@ SHAMap::addKnownNode(
}
auto const generation = f_.getFullBelowCache(ledgerSeq_)->getGeneration();
auto newNode = SHAMapAbstractNode::makeFromWire(rawNode);
auto newNode = SHAMapTreeNode::makeFromWire(rawNode);
SHAMapNodeID iNodeID;
auto iNode = root_.get();
@@ -626,13 +611,17 @@ SHAMap::addKnownNode(
if (iNode == nullptr)
{
if (!newNode || childHash != newNode->getNodeHash())
if (!newNode || childHash != newNode->getHash())
{
JLOG(journal_.warn()) << "Corrupt node received";
return SHAMapAddNode::invalid();
}
if (!newNode->isInBounds(iNodeID))
// Inner nodes must be at a level strictly less than 64
// but leaf nodes (while notionally at level 64) can be
// at any depth up to and including 64:
if ((iNodeID.getDepth() > leafDepth) ||
(newNode->isInner() && iNodeID.getDepth() == leafDepth))
{
// Map is provably invalid
state_ = SHAMapState::Invalid;
@@ -678,7 +667,7 @@ bool
SHAMap::deepCompare(SHAMap& other) const
{
// Intended for debug/test only
std::stack<std::pair<SHAMapAbstractNode*, SHAMapAbstractNode*>> stack;
std::stack<std::pair<SHAMapTreeNode*, SHAMapTreeNode*>> stack;
stack.push({root_.get(), other.root_.get()});
@@ -692,7 +681,7 @@ SHAMap::deepCompare(SHAMap& other) const
JLOG(journal_.info()) << "unable to fetch node";
return false;
}
else if (otherNode->getNodeHash() != node->getNodeHash())
else if (otherNode->getHash() != node->getHash())
{
JLOG(journal_.warn()) << "node hash mismatch";
return false;
@@ -702,9 +691,9 @@ SHAMap::deepCompare(SHAMap& other) const
{
if (!otherNode->isLeaf())
return false;
auto& nodePeek = static_cast<SHAMapTreeNode*>(node)->peekItem();
auto& nodePeek = static_cast<SHAMapLeafNode*>(node)->peekItem();
auto& otherNodePeek =
static_cast<SHAMapTreeNode*>(otherNode)->peekItem();
static_cast<SHAMapLeafNode*>(otherNode)->peekItem();
if (nodePeek->key() != otherNodePeek->key())
return false;
if (nodePeek->peekData() != otherNodePeek->peekData())
@@ -765,7 +754,7 @@ SHAMap::hasInnerNode(
nodeID = nodeID.getChildNodeID(branch);
}
return (node->isInner()) && (node->getNodeHash() == targetNodeHash);
return (node->isInner()) && (node->getHash() == targetNodeHash);
}
/** Does this map have this leaf node?
@@ -777,7 +766,7 @@ SHAMap::hasLeafNode(uint256 const& tag, SHAMapHash const& targetNodeHash) const
SHAMapNodeID nodeID;
if (!node->isInner()) // only one leaf node in the tree
return node->getNodeHash() == targetNodeHash;
return node->getHash() == targetNodeHash;
do
{
@@ -798,35 +787,4 @@ SHAMap::hasLeafNode(uint256 const& tag, SHAMapHash const& targetNodeHash) const
// already
}
/**
@param have A pointer to the map that the recipient already has (if any).
@param includeLeaves True if leaf nodes should be included.
@param max The maximum number of nodes to return.
@param func The functor to call for each node added to the FetchPack.
Note: a caller should set includeLeaves to false for transaction trees.
There's no point in including the leaves of transaction trees.
*/
void
SHAMap::getFetchPack(
SHAMap const* have,
bool includeLeaves,
int max,
std::function<void(SHAMapHash const&, const Blob&)> func) const
{
visitDifferences(
have, [includeLeaves, &max, &func](SHAMapAbstractNode& smn) -> bool {
if (includeLeaves || smn.isInner())
{
Serializer s;
smn.serializeWithPrefix(s);
func(smn.getNodeHash(), s.peekData());
if (--max <= 0)
return false;
}
return true;
});
}
} // namespace ripple

521
src/ripple/shamap/impl/SHAMapTreeNode.cpp
View File

@@ -24,69 +24,21 @@
#include <ripple/beast/core/LexicalCast.h>
#include <ripple/protocol/HashPrefix.h>
#include <ripple/protocol/digest.h>
#include <ripple/shamap/SHAMapAccountStateLeafNode.h>
#include <ripple/shamap/SHAMapInnerNode.h>
#include <ripple/shamap/SHAMapLeafNode.h>
#include <ripple/shamap/SHAMapTreeNode.h>
#include <ripple/shamap/SHAMapTxLeafNode.h>
#include <ripple/shamap/SHAMapTxPlusMetaLeafNode.h>
#include <mutex>
#include <openssl/sha.h>
namespace ripple {
// These are wire-protocol identifiers used during serialization to encode the
// type of a node. They should not be arbitrarily be changed.
static constexpr unsigned char const wireTypeTransaction = 0;
static constexpr unsigned char const wireTypeAccountState = 1;
static constexpr unsigned char const wireTypeInner = 2;
static constexpr unsigned char const wireTypeCompressedInner = 3;
static constexpr unsigned char const wireTypeTransactionWithMeta = 4;
std::mutex SHAMapInnerNode::childLock;
SHAMapAbstractNode::~SHAMapAbstractNode() = default;
std::shared_ptr<SHAMapAbstractNode>
SHAMapInnerNode::clone(std::uint32_t seq) const
{
auto p = std::make_shared<SHAMapInnerNode>(seq);
p->mHash = mHash;
p->mIsBranch = mIsBranch;
p->mFullBelowGen = mFullBelowGen;
p->mHashes = mHashes;
std::lock_guard lock(childLock);
for (int i = 0; i < 16; ++i)
p->mChildren[i] = mChildren[i];
return p;
}
std::shared_ptr<SHAMapAbstractNode>
SHAMapTreeNode::clone(std::uint32_t seq) const
{
return std::make_shared<SHAMapTreeNode>(mItem, mType, seq, mHash);
}
SHAMapTreeNode::SHAMapTreeNode(
std::shared_ptr<SHAMapItem const> item,
TNType type,
std::uint32_t seq)
: SHAMapAbstractNode(type, seq), mItem(std::move(item))
{
assert(mItem->peekData().size() >= 12);
updateHash();
}
SHAMapTreeNode::SHAMapTreeNode(
std::shared_ptr<SHAMapItem const> item,
TNType type,
std::uint32_t seq,
SHAMapHash const& hash)
: SHAMapAbstractNode(type, seq, hash), mItem(std::move(item))
{
assert(mItem->peekData().size() >= 12);
}
std::shared_ptr<SHAMapAbstractNode>
SHAMapAbstractNode::makeTransaction(
std::shared_ptr<SHAMapTreeNode>
SHAMapTreeNode::makeTransaction(
Slice data,
std::uint32_t seq,
SHAMapHash const& hash,
bool hashValid)
{
@@ -97,17 +49,14 @@ SHAMapAbstractNode::makeTransaction(
sha512Half(HashPrefix::transactionID, data), s);
if (hashValid)
return std::make_shared<SHAMapTreeNode>(
std::move(item), tnTRANSACTION_NM, seq, hash);
return std::make_shared<SHAMapTxLeafNode>(std::move(item), 0, hash);
return std::make_shared<SHAMapTreeNode>(
std::move(item), tnTRANSACTION_NM, seq);
return std::make_shared<SHAMapTxLeafNode>(std::move(item), 0);
}
std::shared_ptr<SHAMapAbstractNode>
SHAMapAbstractNode::makeTransactionWithMeta(
std::shared_ptr<SHAMapTreeNode>
SHAMapTreeNode::makeTransactionWithMeta(
Slice data,
std::uint32_t seq,
SHAMapHash const& hash,
bool hashValid)
{
@@ -128,17 +77,15 @@ SHAMapAbstractNode::makeTransactionWithMeta(
auto item = std::make_shared<SHAMapItem const>(tag, s.peekData());
if (hashValid)
return std::make_shared<SHAMapTreeNode>(
std::move(item), tnTRANSACTION_MD, seq, hash);
return std::make_shared<SHAMapTxPlusMetaLeafNode>(
std::move(item), 0, hash);
return std::make_shared<SHAMapTreeNode>(
std::move(item), tnTRANSACTION_MD, seq);
return std::make_shared<SHAMapTxPlusMetaLeafNode>(std::move(item), 0);
}
std::shared_ptr<SHAMapAbstractNode>
SHAMapAbstractNode::makeAccountState(
std::shared_ptr<SHAMapTreeNode>
SHAMapTreeNode::makeAccountState(
Slice data,
std::uint32_t seq,
SHAMapHash const& hash,
bool hashValid)
{
@@ -162,74 +109,14 @@ SHAMapAbstractNode::makeAccountState(
auto item = std::make_shared<SHAMapItem const>(tag, s.peekData());
if (hashValid)
return std::make_shared<SHAMapTreeNode>(
std::move(item), tnACCOUNT_STATE, seq, hash);
return std::make_shared<SHAMapAccountStateLeafNode>(
std::move(item), 0, hash);
return std::make_shared<SHAMapTreeNode>(
std::move(item), tnACCOUNT_STATE, seq);
return std::make_shared<SHAMapAccountStateLeafNode>(std::move(item), 0);
}
std::shared_ptr<SHAMapAbstractNode>
SHAMapInnerNode::makeFullInner(
Slice data,
std::uint32_t seq,
SHAMapHash const& hash,
bool hashValid)
{
if (data.size() != 512)
Throw<std::runtime_error>("Invalid FI node");
auto ret = std::make_shared<SHAMapInnerNode>(seq);
Serializer s(data.data(), data.size());
for (int i = 0; i < 16; ++i)
{
s.getBitString(ret->mHashes[i].as_uint256(), i * 32);
if (ret->mHashes[i].isNonZero())
ret->mIsBranch |= (1 << i);
}
if (hashValid)
ret->mHash = hash;
else
ret->updateHash();
return ret;
}
std::shared_ptr<SHAMapAbstractNode>
SHAMapInnerNode::makeCompressedInner(Slice data, std::uint32_t seq)
{
Serializer s(data.data(), data.size());
int len = s.getLength();
auto ret = std::make_shared<SHAMapInnerNode>(seq);
for (int i = 0; i < (len / 33); ++i)
{
int pos;
if (!s.get8(pos, 32 + (i * 33)))
Throw<std::runtime_error>("short CI node");
if ((pos < 0) || (pos >= 16))
Throw<std::runtime_error>("invalid CI node");
s.getBitString(ret->mHashes[pos].as_uint256(), i * 33);
if (ret->mHashes[pos].isNonZero())
ret->mIsBranch |= (1 << pos);
}
ret->updateHash();
return ret;
}
std::shared_ptr<SHAMapAbstractNode>
SHAMapAbstractNode::makeFromWire(Slice rawNode)
std::shared_ptr<SHAMapTreeNode>
SHAMapTreeNode::makeFromWire(Slice rawNode)
{
if (rawNode.empty())
return {};
@@ -241,29 +128,27 @@ SHAMapAbstractNode::makeFromWire(Slice rawNode)
bool const hashValid = false;
SHAMapHash const hash;
std::uint32_t const seq = 0;
if (type == wireTypeTransaction)
return makeTransaction(rawNode, seq, hash, hashValid);
return makeTransaction(rawNode, hash, hashValid);
if (type == wireTypeAccountState)
return makeAccountState(rawNode, seq, hash, hashValid);
return makeAccountState(rawNode, hash, hashValid);
if (type == wireTypeInner)
return SHAMapInnerNode::makeFullInner(rawNode, seq, hash, hashValid);
return SHAMapInnerNode::makeFullInner(rawNode, hash, hashValid);
if (type == wireTypeCompressedInner)
return SHAMapInnerNode::makeCompressedInner(rawNode, seq);
return SHAMapInnerNode::makeCompressedInner(rawNode);
if (type == wireTypeTransactionWithMeta)
return makeTransactionWithMeta(rawNode, seq, hash, hashValid);
return makeTransactionWithMeta(rawNode, hash, hashValid);
Throw<std::runtime_error>(
"wire: Unknown type (" + std::to_string(type) + ")");
}
std::shared_ptr<SHAMapAbstractNode>
SHAMapAbstractNode::makeFromPrefix(Slice rawNode, SHAMapHash const& hash)
std::shared_ptr<SHAMapTreeNode>
SHAMapTreeNode::makeFromPrefix(Slice rawNode, SHAMapHash const& hash)
{
if (rawNode.size() < 4)
Throw<std::runtime_error>("prefix: short node");
@@ -279,19 +164,18 @@ SHAMapAbstractNode::makeFromPrefix(Slice rawNode, SHAMapHash const& hash)
rawNode.remove_prefix(4);
bool const hashValid = true;
std::uint32_t const seq = 0;
if (type == HashPrefix::transactionID)
return makeTransaction(rawNode, seq, hash, hashValid);
return makeTransaction(rawNode, hash, hashValid);
if (type == HashPrefix::leafNode)
return makeAccountState(rawNode, seq, hash, hashValid);
return makeAccountState(rawNode, hash, hashValid);
if (type == HashPrefix::innerNode)
return SHAMapInnerNode::makeFullInner(rawNode, seq, hash, hashValid);
return SHAMapInnerNode::makeFullInner(rawNode, hash, hashValid);
if (type == HashPrefix::txNode)
return makeTransactionWithMeta(rawNode, seq, hash, hashValid);
return makeTransactionWithMeta(rawNode, hash, hashValid);
Throw<std::runtime_error>(
"prefix: unknown type (" +
@@ -299,349 +183,10 @@ SHAMapAbstractNode::makeFromPrefix(Slice rawNode, SHAMapHash const& hash)
")");
}
bool
SHAMapInnerNode::updateHash()
{
uint256 nh;
if (mIsBranch != 0)
{
sha512_half_hasher h;
using beast::hash_append;
hash_append(h, HashPrefix::innerNode);
for (auto const& hh : mHashes)
hash_append(h, hh);
nh = static_cast<typename sha512_half_hasher::result_type>(h);
}
if (nh == mHash.as_uint256())
return false;
mHash = SHAMapHash{nh};
return true;
}
void
SHAMapInnerNode::updateHashDeep()
{
for (auto pos = 0; pos < 16; ++pos)
{
if (mChildren[pos] != nullptr)
mHashes[pos] = mChildren[pos]->getNodeHash();
}
updateHash();
}
bool
SHAMapTreeNode::updateHash()
{
uint256 nh;
if (mType == tnTRANSACTION_NM)
{
nh =
sha512Half(HashPrefix::transactionID, makeSlice(mItem->peekData()));
}
else if (mType == tnACCOUNT_STATE)
{
nh = sha512Half(
HashPrefix::leafNode, makeSlice(mItem->peekData()), mItem->key());
}
else if (mType == tnTRANSACTION_MD)
{
nh = sha512Half(
HashPrefix::txNode, makeSlice(mItem->peekData()), mItem->key());
}
else
assert(false);
if (nh == mHash.as_uint256())
return false;
mHash = SHAMapHash{nh};
return true;
}
void
SHAMapInnerNode::serializeForWire(Serializer& s) const
{
assert(mType == tnINNER);
assert(!isEmpty());
// If the node is sparse, then only send non-empty branches:
if (getBranchCount() < 12)
{
// compressed node
for (int i = 0; i < mHashes.size(); ++i)
{
if (!isEmptyBranch(i))
{
s.addBitString(mHashes[i].as_uint256());
s.add8(i);
}
}
s.add8(wireTypeCompressedInner);
}
else
{
for (auto const& hh : mHashes)
s.addBitString(hh.as_uint256());
s.add8(wireTypeInner);
}
}
void
SHAMapInnerNode::serializeWithPrefix(Serializer& s) const
{
assert(mType == tnINNER);
assert(!isEmpty());
s.add32(HashPrefix::innerNode);
for (auto const& hh : mHashes)
s.addBitString(hh.as_uint256());
}
void
SHAMapTreeNode::serializeForWire(Serializer& s) const
{
if (mType == tnACCOUNT_STATE)
{
s.addRaw(mItem->peekData());
s.addBitString(mItem->key());
s.add8(wireTypeAccountState);
}
else if (mType == tnTRANSACTION_NM)
{
s.addRaw(mItem->peekData());
s.add8(wireTypeTransaction);
}
else if (mType == tnTRANSACTION_MD)
{
s.addRaw(mItem->peekData());
s.addBitString(mItem->key());
s.add8(wireTypeTransactionWithMeta);
}
}
void
SHAMapTreeNode::serializeWithPrefix(Serializer& s) const
{
if (mType == tnACCOUNT_STATE)
{
s.add32(HashPrefix::leafNode);
s.addRaw(mItem->peekData());
s.addBitString(mItem->key());
}
else if (mType == tnTRANSACTION_NM)
{
s.add32(HashPrefix::transactionID);
s.addRaw(mItem->peekData());
}
else if (mType == tnTRANSACTION_MD)
{
s.add32(HashPrefix::txNode);
s.addRaw(mItem->peekData());
s.addBitString(mItem->key());
}
}
bool
SHAMapTreeNode::setItem(std::shared_ptr<SHAMapItem const> i, TNType type)
{
mType = type;
mItem = std::move(i);
assert(isLeaf());
assert(mSeq != 0);
return updateHash();
}
bool
SHAMapInnerNode::isEmpty() const
{
return mIsBranch == 0;
}
int
SHAMapInnerNode::getBranchCount() const
{
assert(isInner());
int count = 0;
for (int i = 0; i < 16; ++i)
if (!isEmptyBranch(i))
++count;
return count;
}
std::string
SHAMapAbstractNode::getString(const SHAMapNodeID& id) const
SHAMapTreeNode::getString(const SHAMapNodeID& id) const
{
return to_string(id);
}
std::string
SHAMapInnerNode::getString(const SHAMapNodeID& id) const
{
std::string ret = SHAMapAbstractNode::getString(id);
for (int i = 0; i < mHashes.size(); ++i)
{
if (!isEmptyBranch(i))
{
ret += "\n";
ret += std::to_string(i);
ret += " = ";
ret += to_string(mHashes[i]);
}
}
return ret;
}
std::string
SHAMapTreeNode::getString(const SHAMapNodeID& id) const
{
std::string ret = SHAMapAbstractNode::getString(id);
if (mType == tnTRANSACTION_NM)
ret += ",txn\n";
else if (mType == tnTRANSACTION_MD)
ret += ",txn+md\n";
else if (mType == tnACCOUNT_STATE)
ret += ",as\n";
else
ret += ",leaf\n";
ret += " Tag=";
ret += to_string(peekItem()->key());
ret += "\n Hash=";
ret += to_string(mHash);
ret += "/";
ret += std::to_string(mItem->size());
return ret;
}
// We are modifying an inner node
void
SHAMapInnerNode::setChild(
int m,
std::shared_ptr<SHAMapAbstractNode> const& child)
{
assert((m >= 0) && (m < 16));
assert(mType == tnINNER);
assert(mSeq != 0);
assert(child.get() != this);
mHashes[m].zero();
mHash.zero();
if (child)
mIsBranch |= (1 << m);
else
mIsBranch &= ~(1 << m);
mChildren[m] = child;
}
// finished modifying, now make shareable
void
SHAMapInnerNode::shareChild(
int m,
std::shared_ptr<SHAMapAbstractNode> const& child)
{
assert((m >= 0) && (m < 16));
assert(mType == tnINNER);
assert(mSeq != 0);
assert(child);
assert(child.get() != this);
mChildren[m] = child;
}
SHAMapAbstractNode*
SHAMapInnerNode::getChildPointer(int branch)
{
assert(branch >= 0 && branch < 16);
assert(isInner());
std::lock_guard lock(childLock);
return mChildren[branch].get();
}
std::shared_ptr<SHAMapAbstractNode>
SHAMapInnerNode::getChild(int branch)
{
assert(branch >= 0 && branch < 16);
assert(isInner());
std::lock_guard lock(childLock);
return mChildren[branch];
}
std::shared_ptr<SHAMapAbstractNode>
SHAMapInnerNode::canonicalizeChild(
int branch,
std::shared_ptr<SHAMapAbstractNode> node)
{
assert(branch >= 0 && branch < 16);
assert(isInner());
assert(node);
assert(node->getNodeHash() == mHashes[branch]);
std::lock_guard lock(childLock);
if (mChildren[branch])
{
// There is already a node hooked up, return it
node = mChildren[branch];
}
else
{
// Hook this node up
mChildren[branch] = node;
}
return node;
}
uint256 const&
SHAMapInnerNode::key() const
{
Throw<std::logic_error>("SHAMapInnerNode::key() should never be called");
static uint256 x;
return x;
}
uint256 const&
SHAMapTreeNode::key() const
{
return mItem->key();
}
void
SHAMapInnerNode::invariants(bool is_root) const
{
assert(mType == tnINNER);
unsigned count = 0;
for (int i = 0; i < 16; ++i)
{
if (mHashes[i].isNonZero())
{
assert((mIsBranch & (1 << i)) != 0);
if (mChildren[i] != nullptr)
mChildren[i]->invariants();
++count;
}
else
{
assert((mIsBranch & (1 << i)) == 0);
}
}
if (!is_root)
{
assert(mHash.isNonZero());
assert(count >= 1);
}
assert((count == 0) ? mHash.isZero() : mHash.isNonZero());
}
void
SHAMapTreeNode::invariants(bool) const
{
assert(mType >= tnTRANSACTION_NM);
assert(mHash.isNonZero());
assert(mItem != nullptr);
}
} // namespace ripple

4
src/test/app/LedgerHistory_test.cpp
View File

@@ -72,8 +72,8 @@ public:
res->updateSkipList();
{
res->stateMap().flushDirty(hotACCOUNT_NODE, res->info().seq);
res->txMap().flushDirty(hotTRANSACTION_NODE, res->info().seq);
res->stateMap().flushDirty(hotACCOUNT_NODE);
res->txMap().flushDirty(hotTRANSACTION_NODE);
}
res->unshare();

40
src/test/nodestore/DatabaseShard_test.cpp
View File

@@ -283,15 +283,14 @@ class DatabaseShard_test : public TestBase
}
// Store the state map
auto visitAcc = [&](SHAMapAbstractNode& node) {
auto visitAcc = [&](SHAMapTreeNode const& node) {
Serializer s;
node.serializeWithPrefix(s);
db.store(
node.getType() == SHAMapAbstractNode::TNType::tnINNER
? hotUNKNOWN
: hotACCOUNT_NODE,
node.getType() == SHAMapNodeType::tnINNER ? hotUNKNOWN
: hotACCOUNT_NODE,
std::move(s.modData()),
node.getNodeHash().as_uint256(),
node.getHash().as_uint256(),
ledger.info().seq);
return true;
};
@@ -311,15 +310,14 @@ class DatabaseShard_test : public TestBase
}
// Store the transaction map
auto visitTx = [&](SHAMapAbstractNode& node) {
auto visitTx = [&](SHAMapTreeNode& node) {
Serializer s;
node.serializeWithPrefix(s);
db.store(
node.getType() == SHAMapAbstractNode::TNType::tnINNER
? hotUNKNOWN
: hotTRANSACTION_NODE,
node.getType() == SHAMapNodeType::tnINNER ? hotUNKNOWN
: hotTRANSACTION_NODE,
std::move(s.modData()),
node.getNodeHash().as_uint256(),
node.getHash().as_uint256(),
ledger.info().seq);
return true;
};
@@ -357,20 +355,19 @@ class DatabaseShard_test : public TestBase
LedgerFill{*fetched, LedgerFill::full | LedgerFill::binary}));
// walk shamap and validate each node
auto fcompAcc = [&](SHAMapAbstractNode& node) -> bool {
auto fcompAcc = [&](SHAMapTreeNode& node) -> bool {
Serializer s;
node.serializeWithPrefix(s);
auto nSrc{NodeObject::createObject(
node.getType() == SHAMapAbstractNode::TNType::tnINNER
? hotUNKNOWN
: hotACCOUNT_NODE,
node.getType() == SHAMapNodeType::tnINNER ? hotUNKNOWN
: hotACCOUNT_NODE,
std::move(s.modData()),
node.getNodeHash().as_uint256())};
node.getHash().as_uint256())};
if (!BEAST_EXPECT(nSrc))
return false;
auto nDst = db.fetchNodeObject(
node.getNodeHash().as_uint256(), ledger.info().seq);
node.getHash().as_uint256(), ledger.info().seq);
if (!BEAST_EXPECT(nDst))
return false;
@@ -381,20 +378,19 @@ class DatabaseShard_test : public TestBase
if (ledger.stateMap().getHash().isNonZero())
ledger.stateMap().snapShot(false)->visitNodes(fcompAcc);
auto fcompTx = [&](SHAMapAbstractNode& node) -> bool {
auto fcompTx = [&](SHAMapTreeNode& node) -> bool {
Serializer s;
node.serializeWithPrefix(s);
auto nSrc{NodeObject::createObject(
node.getType() == SHAMapAbstractNode::TNType::tnINNER
? hotUNKNOWN
: hotTRANSACTION_NODE,
node.getType() == SHAMapNodeType::tnINNER ? hotUNKNOWN
: hotTRANSACTION_NODE,
std::move(s.modData()),
node.getNodeHash().as_uint256())};
node.getHash().as_uint256())};
if (!BEAST_EXPECT(nSrc))
return false;
auto nDst = db.fetchNodeObject(
node.getNodeHash().as_uint256(), ledger.info().seq);
node.getHash().as_uint256(), ledger.info().seq);
if (!BEAST_EXPECT(nDst))
return false;

5
src/test/shamap/FetchPack_test.cpp
View File

@@ -65,7 +65,7 @@ public:
SHAMapHash const& nodeHash,
std::uint32_t ledgerSeq,
Blob&& nodeData,
SHAMapTreeNode::TNType type) const override
SHAMapNodeType type) const override
{
}
@@ -100,7 +100,8 @@ public:
while (n--)
{
std::shared_ptr<SHAMapItem> item(make_random_item(r));
auto const result(t.addItem(std::move(*item), false, false));
auto const result(
t.addItem(SHAMapNodeType::tnACCOUNT_STATE, std::move(*item)));
assert(result);
(void)result;
}

5
src/test/shamap/SHAMapSync_test.cpp
View File

@@ -59,7 +59,7 @@ public:
std::shared_ptr<SHAMapItem> item = makeRandomAS();
items.push_back(item->key());
if (!map.addItem(std::move(*item), false, false))
if (!map.addItem(SHAMapNodeType::tnACCOUNT_STATE, std::move(*item)))
{
log << "Unable to add item to map\n";
return false;
@@ -98,7 +98,8 @@ public:
int items = 10000;
for (int i = 0; i < items; ++i)
{
source.addItem(std::move(*makeRandomAS()), false, false);
source.addItem(
SHAMapNodeType::tnACCOUNT_STATE, std::move(*makeRandomAS()));
if (i % 100 == 0)
source.invariants();
}

43
src/test/shamap/SHAMap_test.cpp
View File

@@ -64,12 +64,12 @@ static_assert(std::is_copy_assignable<SHAMapHash>{}, "");
static_assert(std::is_move_constructible<SHAMapHash>{}, "");
static_assert(std::is_move_assignable<SHAMapHash>{}, "");
static_assert(!std::is_nothrow_destructible<SHAMapAbstractNode>{}, "");
static_assert(!std::is_default_constructible<SHAMapAbstractNode>{}, "");
static_assert(!std::is_copy_constructible<SHAMapAbstractNode>{}, "");
static_assert(!std::is_copy_assignable<SHAMapAbstractNode>{}, "");
static_assert(!std::is_move_constructible<SHAMapAbstractNode>{}, "");
static_assert(!std::is_move_assignable<SHAMapAbstractNode>{}, "");
static_assert(std::is_nothrow_destructible<SHAMapTreeNode>{}, "");
static_assert(!std::is_default_constructible<SHAMapTreeNode>{}, "");
static_assert(!std::is_copy_constructible<SHAMapTreeNode>{}, "");
static_assert(!std::is_copy_assignable<SHAMapTreeNode>{}, "");
static_assert(!std::is_move_constructible<SHAMapTreeNode>{}, "");
static_assert(!std::is_move_assignable<SHAMapTreeNode>{}, "");
static_assert(std::is_nothrow_destructible<SHAMapInnerNode>{}, "");
static_assert(!std::is_default_constructible<SHAMapInnerNode>{}, "");
@@ -78,12 +78,12 @@ static_assert(!std::is_copy_assignable<SHAMapInnerNode>{}, "");
static_assert(!std::is_move_constructible<SHAMapInnerNode>{}, "");
static_assert(!std::is_move_assignable<SHAMapInnerNode>{}, "");
static_assert(std::is_nothrow_destructible<SHAMapTreeNode>{}, "");
static_assert(!std::is_default_constructible<SHAMapTreeNode>{}, "");
static_assert(!std::is_copy_constructible<SHAMapTreeNode>{}, "");
static_assert(!std::is_copy_assignable<SHAMapTreeNode>{}, "");
static_assert(!std::is_move_constructible<SHAMapTreeNode>{}, "");
static_assert(!std::is_move_assignable<SHAMapTreeNode>{}, "");
static_assert(std::is_nothrow_destructible<SHAMapLeafNode>{}, "");
static_assert(!std::is_default_constructible<SHAMapLeafNode>{}, "");
static_assert(!std::is_copy_constructible<SHAMapLeafNode>{}, "");
static_assert(!std::is_copy_assignable<SHAMapLeafNode>{}, "");
static_assert(!std::is_move_constructible<SHAMapLeafNode>{}, "");
static_assert(!std::is_move_assignable<SHAMapLeafNode>{}, "");
#endif
inline bool
@@ -161,9 +161,13 @@ public:
SHAMapItem i1(h1, IntToVUC(1)), i2(h2, IntToVUC(2)),
i3(h3, IntToVUC(3)), i4(h4, IntToVUC(4)), i5(h5, IntToVUC(5));
unexpected(!sMap.addItem(SHAMapItem{i2}, true, false), "no add");
unexpected(
!sMap.addItem(SHAMapNodeType::tnTRANSACTION_NM, SHAMapItem{i2}),
"no add");
sMap.invariants();
unexpected(!sMap.addItem(SHAMapItem{i1}, true, false), "no add");
unexpected(
!sMap.addItem(SHAMapNodeType::tnTRANSACTION_NM, SHAMapItem{i1}),
"no add");
sMap.invariants();
auto i = sMap.begin();
@@ -173,11 +177,11 @@ public:
unexpected(i == e || (*i != i2), "bad traverse");
++i;
unexpected(i != e, "bad traverse");
sMap.addItem(SHAMapItem{i4}, true, false);
sMap.addItem(SHAMapNodeType::tnTRANSACTION_NM, SHAMapItem{i4});
sMap.invariants();
sMap.delItem(i2.key());
sMap.invariants();
sMap.addItem(SHAMapItem{i3}, true, false);
sMap.addItem(SHAMapNodeType::tnTRANSACTION_NM, SHAMapItem{i3});
sMap.invariants();
i = sMap.begin();
e = sMap.end();
@@ -282,7 +286,8 @@ public:
for (int k = 0; k < keys.size(); ++k)
{
SHAMapItem item(keys[k], IntToVUC(k));
BEAST_EXPECT(map.addItem(std::move(item), true, false));
BEAST_EXPECT(map.addItem(
SHAMapNodeType::tnTRANSACTION_NM, std::move(item)));
BEAST_EXPECT(map.getHash().as_uint256() == hashes[k]);
map.invariants();
}
@@ -333,7 +338,9 @@ public:
map.setUnbacked();
for (auto const& k : keys)
{
map.addItem(SHAMapItem{k, IntToVUC(0)}, true, false);
map.addItem(
SHAMapNodeType::tnTRANSACTION_NM,
SHAMapItem{k, IntToVUC(0)});
map.invariants();
}