ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2025-12-06 17:27:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rework deferred node logic and async fetch behavior

Browse Source 
This comment explains this patch and the associated patches
that should be folded into it. This paragraph should be removed
when the patches are folded after review.

This change significantly improves ledger sync and fetch
times while reducing memory consumption. The change affects
the code from that begins with SHAMap::getMissingNodes and runs
through to Database::threadEntry.

The existing code issues a number of async fetches which are then
handed off to the Database's pool of read threads to execute.
The results of each read are placed in the Database's positive
and negative caches. The caller waits for all reads to complete
and then retrieves the results out of these caches.

Among other issues, this means that the results of the first read
cannot be processed until the last read completes. Additionally,
all the results must sit in memory.

This patch changes the behavior so that each read operation has a
completion handler associated with it. The completion of the read
calls the handler, allowing the results of each read to be
processed as it completes. As this was the only reason the
negative and positive caches were needed, they can now be removed.

The read generation code is also no longer needed and is removed.
The batch fetch logic was never implemented or supported and is
removed.
This commit is contained in:
JoelKatz
2020-11-16 22:38:31 -08:00
committed by Nik Bougalis
parent 7b192945eb
commit 02ccdeb94e
28 changed files with 275 additions and 750 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
src/ripple/nodestore/impl/DatabaseNodeImp.cpp
View File

@@ -32,42 +32,14 @@ DatabaseNodeImp::store(
std::uint32_t)
{
auto nObj = NodeObject::createObject(type, std::move(data), hash);
pCache_->canonicalize_replace_cache(hash, nObj);
backend_->store(nObj);
nCache_->erase(hash);
storeStats(1, nObj->getData().size());
}
bool
DatabaseNodeImp::asyncFetch(
uint256 const& hash,
std::uint32_t ledgerSeq,
std::shared_ptr<NodeObject>& nodeObject)
{
// See if the object is in cache
nodeObject = pCache_->fetch(hash);
if (nodeObject || nCache_->touch_if_exists(hash))
return true;
// Otherwise post a read
Database::asyncFetch(hash, ledgerSeq);
return false;
}
void
DatabaseNodeImp::tune(int size, std::chrono::seconds age)
{
pCache_->setTargetSize(size);
pCache_->setTargetAge(age);
nCache_->setTargetSize(size);
nCache_->setTargetAge(age);
}
void
DatabaseNodeImp::sweep()
{
pCache_->sweep();
nCache_->sweep();
// nothing to do
}
std::shared_ptr<NodeObject>
@@ -76,60 +48,38 @@ DatabaseNodeImp::fetchNodeObject(
std::uint32_t,
FetchReport& fetchReport)
{
// See if the node object exists in the cache
auto nodeObject{pCache_->fetch(hash)};
if (!nodeObject && !nCache_->touch_if_exists(hash))
std::shared_ptr<NodeObject> nodeObject;
Status status;
try
{
// Try the backend
fetchReport.wentToDisk = true;
Status status;
try
{
status = backend_->fetch(hash.data(), &nodeObject);
}
catch (std::exception const& e)
{
JLOG(j_.fatal()) << "Exception, " << e.what();
Rethrow();
}
switch (status)
{
case ok:
++fetchHitCount_;
if (nodeObject)
fetchSz_ += nodeObject->getData().size();
break;
case notFound:
break;
case dataCorrupt:
JLOG(j_.fatal()) << "Corrupt NodeObject #" << hash;
break;
default:
JLOG(j_.warn()) << "Unknown status=" << status;
break;
}
if (!nodeObject)
{
// Just in case a write occurred
nodeObject = pCache_->fetch(hash);
if (!nodeObject)
// We give up
nCache_->insert(hash);
}
else
{
fetchReport.wasFound = true;
// Ensure all threads get the same object
pCache_->canonicalize_replace_client(hash, nodeObject);
// Since this was a 'hard' fetch, we will log it
JLOG(j_.trace()) << "HOS: " << hash << " fetch: in shard db";
}
status = backend_->fetch(hash.data(), &nodeObject);
}
catch (std::exception const& e)
{
JLOG(j_.fatal()) << "Exception, " << e.what();
Rethrow();
}
switch (status)
{
case ok:
++fetchHitCount_;
if (nodeObject)
fetchSz_ += nodeObject->getData().size();
break;
case notFound:
break;
case dataCorrupt:
JLOG(j_.fatal()) << "Corrupt NodeObject #" << hash;
break;
default:
JLOG(j_.warn()) << "Unknown status=" << status;
break;
}
if (nodeObject)
fetchReport.wasFound = true;
return nodeObject;
}