Improve automatic tuning of thread pool:

The job queue can automatically tune the number of threads that
it creates based on the number of processors or processor cores
that are available.

The existing tuning was very conservative, limiting the maximum
number of threads to only 6.

Adjust the new algorithm to allow a larger number of threads and
allow server administrators to override the value in the config
file.

doc/rippled-example.cfg

@@ -690,6 +690,14 @@
 #
 #
 #
+# [workers]
+#
+#   Configures the number of threads for processing work submitted by peers
+#   and clients. If not specified, then the value is automatically determined
+#   by factors including the number of system processors and whether this
+#   node is a validator.
+#
+#
 #-------------------------------------------------------------------------------
 #
 # 4. HTTPS Client

src/ripple/app/main/Application.cpp

@@ -966,7 +966,9 @@ private:
 bool ApplicationImp::setup()
 {
     // VFALCO NOTE: 0 means use heuristics to determine the thread count.
-    m_jobQueue->setThreadCount (0, config_->standalone());
+    m_jobQueue->setThreadCount (config_->WORKERS, config_->standalone(),
+                                config_->exists (SECTION_VALIDATOR_TOKEN) ||
+                                config_->exists (SECTION_VALIDATION_SEED));
 
     // We want to intercept and wait for CTRL-C to terminate the process
     m_signals.add (SIGINT);

src/ripple/core/Config.h

@@ -166,6 +166,9 @@ public:
     std::string                 SSL_VERIFY_FILE;
     std::string                 SSL_VERIFY_DIR;
 
+    // Thread pool configuration
+    std::size_t                 WORKERS = 0;
+
     // These override the command line client settings
     boost::optional<boost::asio::ip::address_v4> rpc_ip;
     boost::optional<std::uint16_t> rpc_port;

src/ripple/core/ConfigSections.h

@@ -69,6 +69,7 @@ struct ConfigSection
 #define SECTION_VALIDATORS              "validators"
 #define SECTION_VALIDATOR_TOKEN         "validator_token"
 #define SECTION_VETO_AMENDMENTS         "veto_amendments"
+#define SECTION_WORKERS                 "workers"
 
 } // ripple
 

src/ripple/core/JobQueue.h

@@ -163,7 +163,8 @@ public:
 
     /** Set the number of thread serving the job queue to precisely this number.
     */
-    void setThreadCount (int c, bool const standaloneMode);
+    void setThreadCount (int c, bool const standaloneMode,
+                         bool const validator=true);
 
     /** Return a scoped LoadEvent.
     */

src/ripple/core/impl/Config.cpp

@@ -413,6 +413,9 @@ void Config::loadFromString (std::string const& fileContents)
     if (getSingleSection (secConfig, SECTION_DEBUG_LOGFILE, strTemp, j_))
         DEBUG_LOGFILE       = strTemp;
 
+    if (getSingleSection (secConfig, SECTION_WORKERS, strTemp, j_))
+        WORKERS      = beast::lexicalCastThrow <std::size_t> (strTemp);
+
     // Do not load trusted validator configuration for standalone mode
     if (! RUN_STANDALONE)
     {

src/ripple/core/impl/JobQueue.cpp

@@ -152,7 +152,8 @@ JobQueue::getJobCountGE (JobType t) const
 }
 
 void
-JobQueue::setThreadCount (int c, bool const standaloneMode)
+JobQueue::setThreadCount (int c, bool const standaloneMode,
+                          bool const validator)
 {
     if (standaloneMode)
     {
@@ -161,10 +162,18 @@ JobQueue::setThreadCount (int c, bool const standaloneMode)
     else if (c == 0)
     {
         c = static_cast<int>(std::thread::hardware_concurrency());
+        if (validator)
             c = 2 + std::min(c, 4); // I/O will bottleneck
-
+        else
+            c *= 2; // Tested to improve stability under high RPC load.
         JLOG (m_journal.info()) << "Auto-tuning to " << c <<
-                            " validation/transaction/proposal threads";
+                            " validation/transaction/proposal threads for " <<
+                            (validator ? "" : "non-") << "validator.";
+    }
+    else
+    {
+        JLOG (m_journal.info()) << "Configured " << c <<
+                            " validation/transaction/proposal threads.";
     }
 
     m_workers.setNumberOfThreads (c);