From 68b8ffdb638d07937f841f7217edeb25efdb3b5d Mon Sep 17 00:00:00 2001 From: Mark Travis Date: Thu, 8 Jun 2017 10:00:29 -0700 Subject: [PATCH] Improve automatic tuning of thread pool: The job queue can automatically tune the number of threads that it creates based on the number of processors or processor cores that are available. The existing tuning was very conservative, limiting the maximum number of threads to only 6. Adjust the new algorithm to allow a larger number of threads and allow server administrators to override the value in the config file. --- doc/rippled-example.cfg | 8 ++++++++ src/ripple/app/main/Application.cpp | 4 +++- src/ripple/core/Config.h | 3 +++ src/ripple/core/ConfigSections.h | 1 + src/ripple/core/JobQueue.h | 3 ++- src/ripple/core/impl/Config.cpp | 3 +++ src/ripple/core/impl/JobQueue.cpp | 19 ++++++++++++++----- 7 files changed, 34 insertions(+), 7 deletions(-) diff --git a/doc/rippled-example.cfg b/doc/rippled-example.cfg index b9cb9898cd..c645802343 100644 --- a/doc/rippled-example.cfg +++ b/doc/rippled-example.cfg @@ -690,6 +690,14 @@ # # # +# [workers] +# +# Configures the number of threads for processing work submitted by peers +# and clients. If not specified, then the value is automatically determined +# by factors including the number of system processors and whether this +# node is a validator. +# +# #------------------------------------------------------------------------------- # # 4. HTTPS Client diff --git a/src/ripple/app/main/Application.cpp b/src/ripple/app/main/Application.cpp index 89f475f229..7f3675a7be 100644 --- a/src/ripple/app/main/Application.cpp +++ b/src/ripple/app/main/Application.cpp @@ -966,7 +966,9 @@ private: bool ApplicationImp::setup() { // VFALCO NOTE: 0 means use heuristics to determine the thread count. - m_jobQueue->setThreadCount (0, config_->standalone()); + m_jobQueue->setThreadCount (config_->WORKERS, config_->standalone(), + config_->exists (SECTION_VALIDATOR_TOKEN) || + config_->exists (SECTION_VALIDATION_SEED)); // We want to intercept and wait for CTRL-C to terminate the process m_signals.add (SIGINT); diff --git a/src/ripple/core/Config.h b/src/ripple/core/Config.h index fb10daab38..8be4e41242 100644 --- a/src/ripple/core/Config.h +++ b/src/ripple/core/Config.h @@ -166,6 +166,9 @@ public: std::string SSL_VERIFY_FILE; std::string SSL_VERIFY_DIR; + // Thread pool configuration + std::size_t WORKERS = 0; + // These override the command line client settings boost::optional rpc_ip; boost::optional rpc_port; diff --git a/src/ripple/core/ConfigSections.h b/src/ripple/core/ConfigSections.h index 638a53403a..29031f21a3 100644 --- a/src/ripple/core/ConfigSections.h +++ b/src/ripple/core/ConfigSections.h @@ -69,6 +69,7 @@ struct ConfigSection #define SECTION_VALIDATORS "validators" #define SECTION_VALIDATOR_TOKEN "validator_token" #define SECTION_VETO_AMENDMENTS "veto_amendments" +#define SECTION_WORKERS "workers" } // ripple diff --git a/src/ripple/core/JobQueue.h b/src/ripple/core/JobQueue.h index 693d088cc1..cf2a873197 100644 --- a/src/ripple/core/JobQueue.h +++ b/src/ripple/core/JobQueue.h @@ -163,7 +163,8 @@ public: /** Set the number of thread serving the job queue to precisely this number. */ - void setThreadCount (int c, bool const standaloneMode); + void setThreadCount (int c, bool const standaloneMode, + bool const validator=true); /** Return a scoped LoadEvent. */ diff --git a/src/ripple/core/impl/Config.cpp b/src/ripple/core/impl/Config.cpp index 2deebae6f6..6e61c518da 100644 --- a/src/ripple/core/impl/Config.cpp +++ b/src/ripple/core/impl/Config.cpp @@ -413,6 +413,9 @@ void Config::loadFromString (std::string const& fileContents) if (getSingleSection (secConfig, SECTION_DEBUG_LOGFILE, strTemp, j_)) DEBUG_LOGFILE = strTemp; + if (getSingleSection (secConfig, SECTION_WORKERS, strTemp, j_)) + WORKERS = beast::lexicalCastThrow (strTemp); + // Do not load trusted validator configuration for standalone mode if (! RUN_STANDALONE) { diff --git a/src/ripple/core/impl/JobQueue.cpp b/src/ripple/core/impl/JobQueue.cpp index cc49adbd13..ffb919a437 100644 --- a/src/ripple/core/impl/JobQueue.cpp +++ b/src/ripple/core/impl/JobQueue.cpp @@ -152,7 +152,8 @@ JobQueue::getJobCountGE (JobType t) const } void -JobQueue::setThreadCount (int c, bool const standaloneMode) +JobQueue::setThreadCount (int c, bool const standaloneMode, + bool const validator) { if (standaloneMode) { @@ -161,10 +162,18 @@ JobQueue::setThreadCount (int c, bool const standaloneMode) else if (c == 0) { c = static_cast(std::thread::hardware_concurrency()); - c = 2 + std::min (c, 4); // I/O will bottleneck - - JLOG(m_journal.info()) << "Auto-tuning to " << c << - " validation/transaction/proposal threads"; + if (validator) + c = 2 + std::min(c, 4); // I/O will bottleneck + else + c *= 2; // Tested to improve stability under high RPC load. + JLOG (m_journal.info()) << "Auto-tuning to " << c << + " validation/transaction/proposal threads for " << + (validator ? "" : "non-") << "validator."; + } + else + { + JLOG (m_journal.info()) << "Configured " << c << + " validation/transaction/proposal threads."; } m_workers.setNumberOfThreads (c);