mirror of
https://github.com/Xahau/xahaud.git
synced 2025-12-06 17:27:52 +00:00
Provide sensible default values for nodestore cache:
The nodestore includes a built-in cache to reduce the disk I/O
load but, by default, this cache was not initialized unless it
was explicitly configured by the server operator.
This commit introduces sensible defaults based on the server's
configured node size.
It remains possible to completely disable the cache if desired
by explicitly configuring it the cache size and age parameters
to 0:
[node_db]
...
cache_size = 0
cache_age = 0
This commit is contained in:
@@ -168,7 +168,23 @@ SHAMapStoreImp::SHAMapStoreImp(
|
|||||||
std::unique_ptr<NodeStore::Database>
|
std::unique_ptr<NodeStore::Database>
|
||||||
SHAMapStoreImp::makeNodeStore(std::int32_t readThreads)
|
SHAMapStoreImp::makeNodeStore(std::int32_t readThreads)
|
||||||
{
|
{
|
||||||
|
auto nscfg = app_.config().section(ConfigSection::nodeDatabase());
|
||||||
|
|
||||||
|
// Provide default values:
|
||||||
|
if (!nscfg.exists("cache_size"))
|
||||||
|
nscfg.set(
|
||||||
|
"cache_size",
|
||||||
|
std::to_string(app_.config().getValueFor(
|
||||||
|
SizedItem::treeCacheSize, std::nullopt)));
|
||||||
|
|
||||||
|
if (!nscfg.exists("cache_age"))
|
||||||
|
nscfg.set(
|
||||||
|
"cache_age",
|
||||||
|
std::to_string(app_.config().getValueFor(
|
||||||
|
SizedItem::treeCacheAge, std::nullopt)));
|
||||||
|
|
||||||
std::unique_ptr<NodeStore::Database> db;
|
std::unique_ptr<NodeStore::Database> db;
|
||||||
|
|
||||||
if (deleteInterval_)
|
if (deleteInterval_)
|
||||||
{
|
{
|
||||||
if (app_.config().reporting())
|
if (app_.config().reporting())
|
||||||
@@ -187,13 +203,14 @@ SHAMapStoreImp::makeNodeStore(std::int32_t readThreads)
|
|||||||
state_db_.setState(state);
|
state_db_.setState(state);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create NodeStore with two backends to allow online deletion of data
|
// Create NodeStore with two backends to allow online deletion of
|
||||||
|
// data
|
||||||
auto dbr = std::make_unique<NodeStore::DatabaseRotatingImp>(
|
auto dbr = std::make_unique<NodeStore::DatabaseRotatingImp>(
|
||||||
scheduler_,
|
scheduler_,
|
||||||
readThreads,
|
readThreads,
|
||||||
std::move(writableBackend),
|
std::move(writableBackend),
|
||||||
std::move(archiveBackend),
|
std::move(archiveBackend),
|
||||||
app_.config().section(ConfigSection::nodeDatabase()),
|
nscfg,
|
||||||
app_.logs().journal(nodeStoreName_));
|
app_.logs().journal(nodeStoreName_));
|
||||||
fdRequired_ += dbr->fdRequired();
|
fdRequired_ += dbr->fdRequired();
|
||||||
dbRotating_ = dbr.get();
|
dbRotating_ = dbr.get();
|
||||||
@@ -206,7 +223,7 @@ SHAMapStoreImp::makeNodeStore(std::int32_t readThreads)
|
|||||||
app_.config().getValueFor(SizedItem::burstSize, std::nullopt)),
|
app_.config().getValueFor(SizedItem::burstSize, std::nullopt)),
|
||||||
scheduler_,
|
scheduler_,
|
||||||
readThreads,
|
readThreads,
|
||||||
app_.config().section(ConfigSection::nodeDatabase()),
|
nscfg,
|
||||||
app_.logs().journal(nodeStoreName_));
|
app_.logs().journal(nodeStoreName_));
|
||||||
fdRequired_ += db->fdRequired();
|
fdRequired_ += db->fdRequired();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -31,9 +31,9 @@ DatabaseNodeImp::store(
|
|||||||
uint256 const& hash,
|
uint256 const& hash,
|
||||||
std::uint32_t)
|
std::uint32_t)
|
||||||
{
|
{
|
||||||
auto nObj = NodeObject::createObject(type, std::move(data), hash);
|
storeStats(1, data.size());
|
||||||
backend_->store(nObj);
|
|
||||||
storeStats(1, nObj->getData().size());
|
backend_->store(NodeObject::createObject(type, std::move(data), hash));
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
@@ -50,12 +50,14 @@ DatabaseNodeImp::fetchNodeObject(
|
|||||||
FetchReport& fetchReport,
|
FetchReport& fetchReport,
|
||||||
bool duplicate)
|
bool duplicate)
|
||||||
{
|
{
|
||||||
std::shared_ptr<NodeObject> nodeObject{
|
std::shared_ptr<NodeObject> nodeObject =
|
||||||
cache_ ? cache_->fetch(hash) : nullptr};
|
cache_ ? cache_->fetch(hash) : nullptr;
|
||||||
|
|
||||||
if (!nodeObject)
|
if (!nodeObject)
|
||||||
{
|
{
|
||||||
JLOG(j_.trace())
|
JLOG(j_.trace()) << "fetchNodeObject " << hash << ": record not "
|
||||||
<< "DatabaseNodeImp::fetchNodeObject - record not in cache";
|
<< (cache_ ? "cached" : "found");
|
||||||
|
|
||||||
Status status;
|
Status status;
|
||||||
|
|
||||||
try
|
try
|
||||||
@@ -64,7 +66,9 @@ DatabaseNodeImp::fetchNodeObject(
|
|||||||
}
|
}
|
||||||
catch (std::exception const& e)
|
catch (std::exception const& e)
|
||||||
{
|
{
|
||||||
JLOG(j_.fatal()) << "Exception, " << e.what();
|
JLOG(j_.fatal())
|
||||||
|
<< "fetchNodeObject " << hash
|
||||||
|
<< ": Exception fetching from backend: " << e.what();
|
||||||
Rethrow();
|
Rethrow();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -77,17 +81,20 @@ DatabaseNodeImp::fetchNodeObject(
|
|||||||
case notFound:
|
case notFound:
|
||||||
break;
|
break;
|
||||||
case dataCorrupt:
|
case dataCorrupt:
|
||||||
JLOG(j_.fatal()) << "Corrupt NodeObject #" << hash;
|
JLOG(j_.fatal()) << "fetchNodeObject " << hash
|
||||||
|
<< ": nodestore data is corrupted";
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
JLOG(j_.warn()) << "Unknown status=" << status;
|
JLOG(j_.warn())
|
||||||
|
<< "fetchNodeObject " << hash
|
||||||
|
<< ": backend returns unknown result " << status;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
JLOG(j_.trace())
|
JLOG(j_.trace()) << "fetchNodeObject " << hash
|
||||||
<< "DatabaseNodeImp::fetchNodeObject - record in cache";
|
<< ": record found in cache";
|
||||||
}
|
}
|
||||||
|
|
||||||
if (nodeObject)
|
if (nodeObject)
|
||||||
@@ -126,7 +133,7 @@ DatabaseNodeImp::fetchBatch(std::vector<uint256> const& hashes)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
JLOG(j_.debug()) << "DatabaseNodeImp::fetchBatch - cache hits = "
|
JLOG(j_.debug()) << "fetchBatch - cache hits = "
|
||||||
<< (hashes.size() - cacheMisses.size())
|
<< (hashes.size() - cacheMisses.size())
|
||||||
<< " - cache misses = " << cacheMisses.size();
|
<< " - cache misses = " << cacheMisses.size();
|
||||||
auto dbResults = backend_->fetchBatch(cacheMisses).first;
|
auto dbResults = backend_->fetchBatch(cacheMisses).first;
|
||||||
@@ -147,7 +154,7 @@ DatabaseNodeImp::fetchBatch(std::vector<uint256> const& hashes)
|
|||||||
else
|
else
|
||||||
{
|
{
|
||||||
JLOG(j_.error())
|
JLOG(j_.error())
|
||||||
<< "DatabaseNodeImp::fetchBatch - "
|
<< "fetchBatch - "
|
||||||
<< "record not found in db or cache. hash = " << strHex(hash);
|
<< "record not found in db or cache. hash = " << strHex(hash);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -45,6 +45,7 @@ public:
|
|||||||
, backend_(std::move(backend))
|
, backend_(std::move(backend))
|
||||||
{
|
{
|
||||||
std::optional<int> cacheSize, cacheAge;
|
std::optional<int> cacheSize, cacheAge;
|
||||||
|
|
||||||
if (config.exists("cache_size"))
|
if (config.exists("cache_size"))
|
||||||
{
|
{
|
||||||
cacheSize = get<int>(config, "cache_size");
|
cacheSize = get<int>(config, "cache_size");
|
||||||
@@ -54,6 +55,7 @@ public:
|
|||||||
"Specified negative value for cache_size");
|
"Specified negative value for cache_size");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config.exists("cache_age"))
|
if (config.exists("cache_age"))
|
||||||
{
|
{
|
||||||
cacheAge = get<int>(config, "cache_age");
|
cacheAge = get<int>(config, "cache_age");
|
||||||
@@ -63,19 +65,17 @@ public:
|
|||||||
"Specified negative value for cache_age");
|
"Specified negative value for cache_age");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (cacheSize || cacheAge)
|
|
||||||
|
if (cacheSize != 0 || cacheAge != 0)
|
||||||
{
|
{
|
||||||
if (!cacheSize || *cacheSize == 0)
|
|
||||||
cacheSize = 16384;
|
|
||||||
if (!cacheAge || *cacheAge == 0)
|
|
||||||
cacheAge = 5;
|
|
||||||
cache_ = std::make_shared<TaggedCache<uint256, NodeObject>>(
|
cache_ = std::make_shared<TaggedCache<uint256, NodeObject>>(
|
||||||
"DatabaseNodeImp",
|
"DatabaseNodeImp",
|
||||||
cacheSize.value(),
|
cacheSize.value_or(0),
|
||||||
std::chrono::minutes{cacheAge.value()},
|
std::chrono::minutes(cacheAge.value_or(0)),
|
||||||
stopwatch(),
|
stopwatch(),
|
||||||
j);
|
j);
|
||||||
}
|
}
|
||||||
|
|
||||||
assert(backend_);
|
assert(backend_);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user