ARCHIVE/xahaud
3
0
Fork 0
mirror of https://github.com/Xahau/xahaud.git synced 2025-11-04 10:45:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(SQLite): allow configurable database pragma values (#5135)

Browse Source 
Make page_size and journal_size_limit configurable values in rippled.cfg
This commit is contained in:
Denis Angell
2024-09-20 23:26:21 +02:00
committed by tequ
parent 088c1deaf5
commit 2227a382d6
8 changed files with 167 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
cfg/rippled-example.cfg
View File

@@ -1119,6 +1119,20 @@
# This setting may not be combined with the
# "safety_level" setting.
#
# page_size Valid values: integer (MUST be power of 2 between 512 and 65536)
# The default is 4096 bytes. This setting determines
# the size of a page in the transaction.db file.
# See https://www.sqlite.org/pragma.html#pragma_page_size
# for more details about the available options.
#
# journal_size_limit Valid values: integer
# The default is 1582080. This setting limits
# the size of the journal for transaction.db file. When the limit is
# reached, older entries will be deleted.
# See https://www.sqlite.org/pragma.html#pragma_journal_size_limit
# for more details about the available options.
#
#
#-------------------------------------------------------------------------------
#
# 7. Diagnostics

109
src/test/nodestore/Database_test.cpp
View File

@@ -439,6 +439,115 @@ public:
BEAST_EXPECT(found);
}
}
{
// N/A: Default values
Env env(*this);
auto const s = setup_DatabaseCon(env.app().config());
if (BEAST_EXPECT(s.txPragma.size() == 4))
{
BEAST_EXPECT(s.txPragma.at(0) == "PRAGMA page_size=4096;");
BEAST_EXPECT(
s.txPragma.at(1) == "PRAGMA journal_size_limit=1582080;");
BEAST_EXPECT(
s.txPragma.at(2) == "PRAGMA max_page_count=4294967294;");
BEAST_EXPECT(
s.txPragma.at(3) == "PRAGMA mmap_size=17179869184;");
}
}
{
// Success: Valid values
Env env = [&]() {
auto p = test::jtx::envconfig();
{
auto& section = p->section("sqlite");
section.set("page_size", "512");
section.set("journal_size_limit", "2582080");
}
return Env(*this, std::move(p));
}();
auto const s = setup_DatabaseCon(env.app().config());
if (BEAST_EXPECT(s.txPragma.size() == 4))
{
BEAST_EXPECT(s.txPragma.at(0) == "PRAGMA page_size=512;");
BEAST_EXPECT(
s.txPragma.at(1) == "PRAGMA journal_size_limit=2582080;");
BEAST_EXPECT(
s.txPragma.at(2) == "PRAGMA max_page_count=4294967294;");
BEAST_EXPECT(
s.txPragma.at(3) == "PRAGMA mmap_size=17179869184;");
}
}
{
// Error: Invalid values
auto const expected =
"Invalid page_size. Must be between 512 and 65536.";
bool found = false;
auto p = test::jtx::envconfig();
{
auto& section = p->section("sqlite");
section.set("page_size", "256");
}
try
{
Env env(
*this,
std::move(p),
std::make_unique<CheckMessageLogs>(expected, &found),
beast::severities::kWarning);
fail();
}
catch (...)
{
BEAST_EXPECT(found);
}
}
{
// Error: Invalid values
auto const expected =
"Invalid page_size. Must be between 512 and 65536.";
bool found = false;
auto p = test::jtx::envconfig();
{
auto& section = p->section("sqlite");
section.set("page_size", "131072");
}
try
{
Env env(
*this,
std::move(p),
std::make_unique<CheckMessageLogs>(expected, &found),
beast::severities::kWarning);
fail();
}
catch (...)
{
BEAST_EXPECT(found);
}
}
{
// Error: Invalid values
auto const expected = "Invalid page_size. Must be a power of 2.";
bool found = false;
auto p = test::jtx::envconfig();
{
auto& section = p->section("sqlite");
section.set("page_size", "513");
}
try
{
Env env(
*this,
std::move(p),
std::make_unique<CheckMessageLogs>(expected, &found),
beast::severities::kWarning);
fail();
}
catch (...)
{
BEAST_EXPECT(found);
}
}
}
//--------------------------------------------------------------------------

22
src/xrpld/app/main/DBInit.h
View File

@@ -42,9 +42,6 @@ inline constexpr std::uint32_t SQLITE_TUNING_CUTOFF = 10'000'000;
// Ledger database holds ledgers and ledger confirmations
inline constexpr auto LgrDBName{"ledger.db"};
inline constexpr std::array<char const*, 1> LgrDBPragma{
{"PRAGMA journal_size_limit=1582080;"}};
inline constexpr std::array<char const*, 5> LgrDBInit{
{"BEGIN TRANSACTION;",
@@ -72,25 +69,6 @@ inline constexpr std::array<char const*, 5> LgrDBInit{
// Transaction database holds transactions and public keys
inline constexpr auto TxDBName{"transaction.db"};
// In C++17 omitting the explicit template parameters caused
// a crash
inline constexpr std::array<char const*, 4> TxDBPragma
{
"PRAGMA page_size=4096;", "PRAGMA journal_size_limit=1582080;",
"PRAGMA max_page_count=4294967294;",
#if (ULONG_MAX > UINT_MAX) && !defined(NO_SQLITE_MMAP)
"PRAGMA mmap_size=17179869184;"
#else
// Provide an explicit `no-op` SQL statement
// in order to keep the size of the array
// constant regardless of the preprocessor
// condition evaluation
"PRAGMA sqlite_noop_statement;"
#endif
};
inline constexpr std::array<char const*, 8> TxDBInit{
{"BEGIN TRANSACTION;",

4
src/xrpld/app/rdb/backend/detail/Node.cpp
View File

@@ -72,7 +72,7 @@ makeLedgerDBs(
{
// ledger database
auto lgr{std::make_unique<DatabaseCon>(
setup, LgrDBName, LgrDBPragma, LgrDBInit, checkpointerSetup, j)};
setup, LgrDBName, setup.lgrPragma, LgrDBInit, checkpointerSetup, j)};
lgr->getSession() << boost::str(
boost::format("PRAGMA cache_size=-%d;") %
kilobytes(config.getValueFor(SizedItem::lgrDBCache)));
@@ -81,7 +81,7 @@ makeLedgerDBs(
{
// transaction database
auto tx{std::make_unique<DatabaseCon>(
setup, TxDBName, TxDBPragma, TxDBInit, checkpointerSetup, j)};
setup, TxDBName, setup.txPragma, TxDBInit, checkpointerSetup, j)};
tx->getSession() << boost::str(
boost::format("PRAGMA cache_size=-%d;") %
kilobytes(config.getValueFor(SizedItem::txnDBCache)));

4
src/xrpld/app/rdb/detail/Vacuum.cpp
View File

@@ -40,8 +40,8 @@ doVacuumDB(DatabaseCon::Setup const& setup, beast::Journal j)
return false;
}
auto txnDB =
std::make_unique<DatabaseCon>(setup, TxDBName, TxDBPragma, TxDBInit, j);
auto txnDB = std::make_unique<DatabaseCon>(
setup, TxDBName, setup.txPragma, TxDBInit, j);
auto& session = txnDB->getSession();
std::uint32_t pageSize;

4
src/xrpld/app/rdb/detail/Wallet.cpp
View File

@@ -27,7 +27,7 @@ makeWalletDB(DatabaseCon::Setup const& setup, beast::Journal j)
{
// wallet database
return std::make_unique<DatabaseCon>(
setup, WalletDBName, std::array<char const*, 0>(), WalletDBInit, j);
setup, WalletDBName, std::array<std::string, 0>(), WalletDBInit, j);
}
std::unique_ptr<DatabaseCon>
@@ -38,7 +38,7 @@ makeTestWalletDB(
{
// wallet database
return std::make_unique<DatabaseCon>(
setup, dbname.data(), std::array<char const*, 0>(), WalletDBInit, j);
setup, dbname.data(), std::array<std::string, 0>(), WalletDBInit, j);
}
void

12
src/xrpld/core/DatabaseCon.h
View File

@@ -102,6 +102,8 @@ public:
}
static std::unique_ptr<std::vector<std::string> const> globalPragma;
std::array<std::string, 4> txPragma;
std::array<std::string, 1> lgrPragma;
};
struct CheckpointerSetup
@@ -114,7 +116,7 @@ public:
DatabaseCon(
Setup const& setup,
std::string const& dbName,
std::array<char const*, N> const& pragma,
std::array<std::string, N> const& pragma,
std::array<char const*, M> const& initSQL,
beast::Journal journal)
// Use temporary files or regular DB files?
@@ -136,7 +138,7 @@ public:
DatabaseCon(
Setup const& setup,
std::string const& dbName,
std::array<char const*, N> const& pragma,
std::array<std::string, N> const& pragma,
std::array<char const*, M> const& initSQL,
CheckpointerSetup const& checkpointerSetup,
beast::Journal journal)
@@ -149,7 +151,7 @@ public:
DatabaseCon(
boost::filesystem::path const& dataDir,
std::string const& dbName,
std::array<char const*, N> const& pragma,
std::array<std::string, N> const& pragma,
std::array<char const*, M> const& initSQL,
beast::Journal journal)
: DatabaseCon(dataDir / dbName, nullptr, pragma, initSQL, journal)
@@ -161,7 +163,7 @@ public:
DatabaseCon(
boost::filesystem::path const& dataDir,
std::string const& dbName,
std::array<char const*, N> const& pragma,
std::array<std::string, N> const& pragma,
std::array<char const*, M> const& initSQL,
CheckpointerSetup const& checkpointerSetup,
beast::Journal journal)
@@ -199,7 +201,7 @@ private:
DatabaseCon(
boost::filesystem::path const& pPath,
std::vector<std::string> const* commonPragma,
std::array<char const*, N> const& pragma,
std::array<std::string, N> const& pragma,
std::array<char const*, M> const& initSQL,
beast::Journal journal)
: session_(std::make_shared<soci::session>()), j_(journal)

32
src/xrpld/core/detail/DatabaseCon.cpp
View File

@@ -175,7 +175,7 @@ setup_DatabaseCon(Config const& c, std::optional<beast::Journal> j)
}
{
//#synchronous Valid values : off, normal, full, extra
// #synchronous Valid values : off, normal, full, extra
if (set(synchronous, "synchronous", sqlite) &&
!safety_level.empty())
{
@@ -237,6 +237,36 @@ setup_DatabaseCon(Config const& c, std::optional<beast::Journal> j)
}
setup.useGlobalPragma = true;
auto setPragma =
[](std::string& pragma, std::string const& key, int64_t value) {
pragma = "PRAGMA " + key + "=" + std::to_string(value) + ";";
};
// Lgr Pragma
setPragma(setup.lgrPragma[0], "journal_size_limit", 1582080);
// TX Pragma
int64_t page_size = 4096;
int64_t journal_size_limit = 1582080;
if (c.exists("sqlite"))
{
auto& s = c.section("sqlite");
set(journal_size_limit, "journal_size_limit", s);
set(page_size, "page_size", s);
if (page_size < 512 || page_size > 65536)
Throw<std::runtime_error>(
"Invalid page_size. Must be between 512 and 65536.");
if (page_size & (page_size - 1))
Throw<std::runtime_error>(
"Invalid page_size. Must be a power of 2.");
}
setPragma(setup.txPragma[0], "page_size", page_size);
setPragma(setup.txPragma[1], "journal_size_limit", journal_size_limit);
setPragma(setup.txPragma[2], "max_page_count", 4294967294);
setPragma(setup.txPragma[3], "mmap_size", 17179869184);
return setup;
}