From 433feade5d4073b574e4de767329208a6b514601 Mon Sep 17 00:00:00 2001 From: Nik Bougalis Date: Sun, 11 Apr 2021 21:57:40 -0700 Subject: [PATCH] Automatically determine the node size: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `[node_size]` configuration parameter is used to tune various parameters based on the hardware that the code is running on. The parameter can take five distinct values: `tiny`, `small`, `medium`, `large` and `huge`. The default value in the code is `tiny` but the default configuration file sets the value to `medium`. This commit attempts to detect the amount of RAM on the system and adjusts the node size default value based on the amount of RAM and the number of hardware execution threads on the system. The decision matrix currently used is: | | 1 | 2 or 3 | ≥ 4 | |:-------:|:----:|:------:|:------:| | > ~8GB | tiny | tiny | tiny | | > ~12GB | tiny | small | small | | > ~16GB | tiny | small | medium | | > ~24GB | tiny | small | large | | > ~32GB | tiny | small | huge | Some systems exclude memory reserved by the the hardware, the kernel or the underlying hypervisor so the automatic detection code may end up determining the node_size to be one less than "appropriate" given the above table. The detection algorithm is simplistic and does not take into account other relevant factors. Therefore, for production-quality servers it is recommended that server operators examine the system holistically and determine what the appropriate size is instead of relying on the automatic detection code. To aid server operators, the node size will now be reported in the `server_info` API as `node_size` when the command is invoked in 'admin' mode. --- cfg/rippled-example.cfg | 19 +++- src/ripple/app/main/Main.cpp | 6 +- src/ripple/app/misc/NetworkOPs.cpp | 19 ++++ src/ripple/core/Config.h | 15 +-- src/ripple/core/ConfigSections.h | 1 - src/ripple/core/impl/Config.cpp | 166 +++++++++++++++++++++++------ src/ripple/protocol/jss.h | 1 + 7 files changed, 179 insertions(+), 48 deletions(-) diff --git a/cfg/rippled-example.cfg b/cfg/rippled-example.cfg index be9f22815..aa30d242b 100644 --- a/cfg/rippled-example.cfg +++ b/cfg/rippled-example.cfg @@ -1426,8 +1426,20 @@ # Tunes the servers based on the expected load and available memory. Legal # sizes are "tiny", "small", "medium", "large", and "huge". We recommend # you start at the default and raise the setting if you have extra memory. -# If no value is specified, the code assumes the proper size is "tiny". The -# default configuration file explicitly specifies "medium" as the size. +# +# The code attempts to automatically determine the appropriate size for +# this parameter based on the amount of RAM and the number of execution +# cores availabe to the server. The current decision matrix is: +# +# | | Cores | +# |---------|------------------------| +# | RAM | 1 | 2 or 3 | ≥ 4 | +# |---------|------|--------|--------| +# | < ~8GB | tiny | tiny | tiny | +# | < ~12GB | tiny | small | small | +# | < ~16GB | tiny | small | medium | +# | < ~24GB | tiny | small | large | +# | < ~32GB | tiny | small | huge | # # [signing_support] # @@ -1598,9 +1610,6 @@ protocol = ws #------------------------------------------------------------------------------- -[node_size] -medium - # This is primary persistent datastore for rippled. This includes transaction # metadata, account states, and ledger headers. Helpful information can be # found at https://xrpl.org/capacity-planning.html#node-db-type diff --git a/src/ripple/app/main/Main.cpp b/src/ripple/app/main/Main.cpp index 7f1ce1f41..1fe02f660 100644 --- a/src/ripple/app/main/Main.cpp +++ b/src/ripple/app/main/Main.cpp @@ -746,8 +746,6 @@ run(int argc, char** argv) } // namespace ripple -// Must be outside the namespace for obvious reasons -// int main(int argc, char** argv) { @@ -771,7 +769,5 @@ main(int argc, char** argv) atexit(&google::protobuf::ShutdownProtobufLibrary); - auto const result(ripple::run(argc, argv)); - - return result; + return ripple::run(argc, argv); } diff --git a/src/ripple/app/misc/NetworkOPs.cpp b/src/ripple/app/misc/NetworkOPs.cpp index 460943710..a1c8db3b6 100644 --- a/src/ripple/app/misc/NetworkOPs.cpp +++ b/src/ripple/app/misc/NetworkOPs.cpp @@ -2213,6 +2213,25 @@ NetworkOPsImp::getServerInfo(bool human, bool admin, bool counters) if (admin) { + switch (app_.config().NODE_SIZE) + { + case 0: + info[jss::node_size] = "tiny"; + break; + case 1: + info[jss::node_size] = "small"; + break; + case 2: + info[jss::node_size] = "medium"; + break; + case 3: + info[jss::node_size] = "large"; + break; + case 4: + info[jss::node_size] = "huge"; + break; + } + auto when = app_.validators().expires(); if (!human) diff --git a/src/ripple/core/Config.h b/src/ripple/core/Config.h index 08615395e..146a40a79 100644 --- a/src/ripple/core/Config.h +++ b/src/ripple/core/Config.h @@ -56,7 +56,8 @@ enum class SizedItem : std::size_t { txnDBCache, lgrDBCache, openFinalLimit, - burstSize + burstSize, + ramSizeGB }; // This entire derived class is deprecated. @@ -116,6 +117,9 @@ private: */ bool signingEnabled_ = false; + // The amount of RAM, in bytes, that we detected on this system. + std::uint64_t const ramSize_; + public: bool doImport = false; bool nodeToShard = false; @@ -156,8 +160,6 @@ public: std::size_t PEERS_OUT_MAX = 0; std::size_t PEERS_IN_MAX = 0; - std::chrono::seconds WEBSOCKET_PING_FREQ = std::chrono::minutes{5}; - // Path searching int PATH_SEARCH_OLD = 7; int PATH_SEARCH = 7; @@ -176,6 +178,9 @@ public: std::uint32_t LEDGER_HISTORY = 256; std::uint32_t FETCH_DEPTH = 1000000000; + // Tunable that adjusts various parameters, typically associated + // with hardware parameters (RAM size and CPU cores). The default + // is 'tiny'. std::size_t NODE_SIZE = 0; bool SSL_VERIFY = true; @@ -230,9 +235,7 @@ public: bool BETA_RPC_API = false; public: - Config() : j_{beast::Journal::getNullSink()} - { - } + Config(); /* Be very careful to make sure these bool params are in the right order. */ diff --git a/src/ripple/core/ConfigSections.h b/src/ripple/core/ConfigSections.h index 9b87dccd4..1c06e75d9 100644 --- a/src/ripple/core/ConfigSections.h +++ b/src/ripple/core/ConfigSections.h @@ -88,7 +88,6 @@ struct ConfigSection #define SECTION_SERVER_DOMAIN "server_domain" #define SECTION_VALIDATORS_FILE "validators_file" #define SECTION_VALIDATION_SEED "validation_seed" -#define SECTION_WEBSOCKET_PING_FREQ "websocket_ping_frequency" #define SECTION_VALIDATOR_KEYS "validator_keys" #define SECTION_VALIDATOR_KEY_REVOCATION "validator_key_revocation" #define SECTION_VALIDATOR_LIST_KEYS "validator_list_keys" diff --git a/src/ripple/core/impl/Config.cpp b/src/ripple/core/impl/Config.cpp index 120b0384b..ee07d6c4f 100644 --- a/src/ripple/core/impl/Config.cpp +++ b/src/ripple/core/impl/Config.cpp @@ -29,36 +29,106 @@ #include #include #include -#include #include +#include #include #include #include -#include +#include #include #include +#include + +#if BOOST_OS_WINDOWS +#include + +namespace ripple { +namespace detail { + +[[nodiscard]] std::uint64_t +getMemorySize() +{ + if (MEMORYSTATUSEX msx{sizeof(MEMORYSTATUSEX)}; GlobalMemoryStatusEx(&msx)) + return static_cast(msx.ullTotalPhys); + + return 0; +} + +} // namespace detail +} // namespace ripple +#endif + +#if BOOST_OS_LINUX +#include + +namespace ripple { +namespace detail { + +[[nodiscard]] std::uint64_t +getMemorySize() +{ + struct sysinfo si; + + if (sysinfo(&si) == 0) + return static_cast(si.totalram); + + return 0; +} + +} // namespace detail +} // namespace ripple + +#endif + +#if BOOST_OS_MACOS +#include +#include + +namespace ripple { +namespace detail { + +[[nodiscard]] std::uint64_t +getMemorySize() +{ + int mib[] = {CTL_HW, HW_MEMSIZE}; + std::int64_t ram = 0; + size_t size = sizeof(ram); + + if (sysctl(mib, 2, &ram, &size, NULL, 0) == 0) + return static_cast(ram); + + return 0; +} + +} // namespace detail +} // namespace ripple +#endif namespace ripple { +// clang-format off // The configurable node sizes are "tiny", "small", "medium", "large", "huge" -inline constexpr std::array>, 11> - sizedItems{{ - // FIXME: We should document each of these items, explaining exactly - // what - // they control and whether there exists an explicit config - // option that can be used to override the default. - {SizedItem::sweepInterval, {{10, 30, 60, 90, 120}}}, - {SizedItem::treeCacheSize, {{128000, 256000, 512000, 768000, 2048000}}}, - {SizedItem::treeCacheAge, {{30, 60, 90, 120, 900}}}, - {SizedItem::ledgerSize, {{32, 128, 256, 384, 768}}}, - {SizedItem::ledgerAge, {{30, 90, 180, 240, 900}}}, - {SizedItem::ledgerFetch, {{2, 3, 4, 5, 8}}}, - {SizedItem::hashNodeDBCache, {{4, 12, 24, 64, 128}}}, - {SizedItem::txnDBCache, {{4, 12, 24, 64, 128}}}, - {SizedItem::lgrDBCache, {{4, 8, 16, 32, 128}}}, - {SizedItem::openFinalLimit, {{8, 16, 32, 64, 128}}}, - {SizedItem::burstSize, {{4, 8, 16, 32, 48}}}, - }}; +inline constexpr std::array>, 12> +sizedItems +{{ + // FIXME: We should document each of these items, explaining exactly + // what they control and whether there exists an explicit + // config option that can be used to override the default. + + // tiny small medium large huge + {SizedItem::sweepInterval, {{ 10, 30, 60, 90, 120 }}}, + {SizedItem::treeCacheSize, {{ 128000, 256000, 512000, 768000, 2048000 }}}, + {SizedItem::treeCacheAge, {{ 30, 60, 90, 120, 900 }}}, + {SizedItem::ledgerSize, {{ 32, 128, 256, 384, 768 }}}, + {SizedItem::ledgerAge, {{ 30, 90, 180, 240, 900 }}}, + {SizedItem::ledgerFetch, {{ 2, 3, 4, 5, 8 }}}, + {SizedItem::hashNodeDBCache, {{ 4, 12, 24, 64, 128 }}}, + {SizedItem::txnDBCache, {{ 4, 12, 24, 64, 128 }}}, + {SizedItem::lgrDBCache, {{ 4, 8, 16, 32, 128 }}}, + {SizedItem::openFinalLimit, {{ 8, 16, 32, 64, 128 }}}, + {SizedItem::burstSize, {{ 4, 8, 16, 32, 48 }}}, + {SizedItem::ramSizeGB, {{ 8, 12, 16, 24, 32 }}}, +}}; // Ensure that the order of entries in the table corresponds to the // order of entries in the enum: @@ -77,6 +147,7 @@ static_assert( return true; }(), "Mismatch between sized item enum & array indices"); +// clang-format on // // TODO: Check permissions on config file before using it. @@ -180,14 +251,12 @@ char const* const Config::configFileName = "rippled.cfg"; char const* const Config::databaseDirName = "db"; char const* const Config::validatorsFileName = "validators.txt"; -static std::string +[[nodiscard]] static std::string getEnvVar(char const* name) { std::string value; - auto const v = getenv(name); - - if (v != nullptr) + if (auto const v = std::getenv(name); v != nullptr) value = v; return value; @@ -195,12 +264,51 @@ getEnvVar(char const* name) constexpr FeeUnit32 Config::TRANSACTION_FEE_BASE; +Config::Config() + : j_(beast::Journal::getNullSink()), ramSize_(detail::getMemorySize()) +{ +} + void Config::setupControl(bool bQuiet, bool bSilent, bool bStandalone) { + assert(NODE_SIZE == 0); + QUIET = bQuiet || bSilent; SILENT = bSilent; RUN_STANDALONE = bStandalone; + + // We try to autodetect the appropriate node size by checking available + // RAM and CPU resources. We default to "tiny" for standalone mode. + if (!bStandalone) + { + // First, check against 'minimum' RAM requirements per node size: + auto const& threshold = + sizedItems[std::underlying_type_t(SizedItem::ramSizeGB)]; + + auto ns = std::find_if( + threshold.second.begin(), + threshold.second.end(), + [this](std::size_t limit) { + return (ramSize_ / (1024 * 1024 * 1024)) < limit; + }); + + if (ns != threshold.second.end()) + NODE_SIZE = std::distance(threshold.second.begin(), ns); + + // Adjust the size based on the number of hardware threads of + // execution available to us: + if (auto const hc = std::thread::hardware_concurrency()) + { + if (hc == 1) + NODE_SIZE = 0; + + if (hc < 4) + NODE_SIZE = std::min(NODE_SIZE, 1); + } + } + + assert(NODE_SIZE <= 4); } void @@ -244,9 +352,9 @@ Config::setup( // Construct XDG config and data home. // http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html - std::string strHome = getEnvVar("HOME"); - std::string strXdgConfigHome = getEnvVar("XDG_CONFIG_HOME"); - std::string strXdgDataHome = getEnvVar("XDG_DATA_HOME"); + auto const strHome = getEnvVar("HOME"); + auto strXdgConfigHome = getEnvVar("XDG_CONFIG_HOME"); + auto strXdgDataHome = getEnvVar("XDG_DATA_HOME"); if (boost::filesystem::exists(CONFIG_FILE) // Can we figure out XDG dirs? @@ -436,10 +544,6 @@ Config::loadFromString(std::string const& fileContents) if (getSingleSection(secConfig, SECTION_ELB_SUPPORT, strTemp, j_)) ELB_SUPPORT = beast::lexicalCastThrow(strTemp); - if (getSingleSection(secConfig, SECTION_WEBSOCKET_PING_FREQ, strTemp, j_)) - WEBSOCKET_PING_FREQ = - std::chrono::seconds{beast::lexicalCastThrow(strTemp)}; - getSingleSection(secConfig, SECTION_SSL_VERIFY_FILE, SSL_VERIFY_FILE, j_); getSingleSection(secConfig, SECTION_SSL_VERIFY_DIR, SSL_VERIFY_DIR, j_); diff --git a/src/ripple/protocol/jss.h b/src/ripple/protocol/jss.h index f70a6004f..b9b782a6a 100644 --- a/src/ripple/protocol/jss.h +++ b/src/ripple/protocol/jss.h @@ -395,6 +395,7 @@ JSS(node_read_retries); // out: GetCounts JSS(node_reads_hit); // out: GetCounts JSS(node_reads_total); // out: GetCounts JSS(node_reads_duration_us); // out: GetCounts +JSS(node_size); // out: server_info JSS(nodestore); // out: GetCounts JSS(node_writes); // out: GetCounts JSS(node_written_bytes); // out: GetCounts