ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2025-12-01 08:25:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve shard finalize memory utilization

Browse Source
This commit is contained in:
Miguel Portilla
2020-08-07 17:04:39 -04:00
committed by Nik Bougalis
parent 4ae2f06be4
commit ab4102a632
3 changed files with 29 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
src/ripple/nodestore/impl/Shard.cpp
View File

@@ -228,7 +228,17 @@ Shard::open(Scheduler& scheduler, nudb::context& ctx)
std::string("exception ") + e.what() + " in function " + __func__);
}
setBackendCache(lock);
// Set backend caches
{
auto const size{config.getValueFor(SizedItem::nodeCacheSize, 0)};
auto const age{std::chrono::seconds{
config.getValueFor(SizedItem::nodeCacheAge, 0)}};
auto const name{"shard " + std::to_string(index_)};
pCache_ = std::make_shared<PCache>(name, size, age, stopwatch(), j_);
nCache_ = std::make_shared<NCache>(name, stopwatch(), size, age);
}
if (!initSQLite(lock))
return fail({});
@@ -306,7 +316,6 @@ Shard::store(std::shared_ptr<Ledger const> const& ledger)
return false;
backendComplete_ = true;
setBackendCache(lock);
}
JLOG(j_.debug()) << "shard " << index_ << " stored ledger sequence " << seq
@@ -523,6 +532,15 @@ Shard::finalize(
std::shared_ptr<Ledger> ledger;
std::shared_ptr<Ledger const> next;
auto const lastLedgerHash{hash};
auto& shardFamily{*app_.getShardFamily()};
auto const fullBelowCache{shardFamily.getFullBelowCache(lastSeq_)};
auto const treeNodeCache{shardFamily.getTreeNodeCache(lastSeq_)};
// Reset caches to reduce memory usage
pCache_->reset();
nCache_->reset();
fullBelowCache->reset();
treeNodeCache->reset();
// Start with the last ledger in the shard and walk backwards from
// child to parent until we reach the first ledger
@@ -539,7 +557,7 @@ Shard::finalize(
ledger = std::make_shared<Ledger>(
deserializePrefixedHeader(makeSlice(nObj->getData())),
app_.config(),
*app_.getShardFamily());
shardFamily);
if (ledger->info().seq != seq)
return fail("invalid ledger sequence");
if (ledger->info().hash != hash)
@@ -573,6 +591,11 @@ Shard::finalize(
hash = ledger->info().parentHash;
next = std::move(ledger);
--seq;
pCache_->reset();
nCache_->reset();
fullBelowCache->reset();
treeNodeCache->reset();
}
JLOG(j_.debug()) << "shard " << index_ << " is valid";
@@ -621,9 +644,7 @@ Shard::finalize(
try
{
backend_->store(nObj);
std::lock_guard lock(mutex_);
final_ = true;
// Remove the acquire SQLite database if present
if (acquireInfo_)
@@ -634,6 +655,7 @@ Shard::finalize(
return fail("failed to initialize SQLite databases");
setFileStats(lock);
final_ = true;
}
catch (std::exception const& e)
{
@@ -644,40 +666,6 @@ Shard::finalize(
return true;
}
void
Shard::setBackendCache(std::lock_guard<std::recursive_mutex> const&)
{
// Complete shards use the smallest cache and
// fastest expiration to reduce memory consumption.
// An incomplete shard is set according to configuration.
Config const& config{app_.config()};
if (!pCache_)
{
auto const name{"shard " + std::to_string(index_)};
auto const sz{config.getValueFor(
SizedItem::nodeCacheSize,
backendComplete_ ? boost::optional<std::size_t>(0) : boost::none)};
auto const age{std::chrono::seconds{config.getValueFor(
SizedItem::nodeCacheAge,
backendComplete_ ? boost::optional<std::size_t>(0) : boost::none)}};
pCache_ = std::make_shared<PCache>(name, sz, age, stopwatch(), j_);
nCache_ = std::make_shared<NCache>(name, stopwatch(), sz, age);
}
else
{
auto const sz{config.getValueFor(SizedItem::nodeCacheSize, 0)};
pCache_->setTargetSize(sz);
nCache_->setTargetSize(sz);
auto const age{std::chrono::seconds{
config.getValueFor(SizedItem::nodeCacheAge, 0)}};
pCache_->setTargetAge(age);
nCache_->setTargetAge(age);
}
}
bool
Shard::initSQLite(std::lock_guard<std::recursive_mutex> const&)
{