Performance logging and counters:

* Tally and duration counters for Job Queue tasks and RPC calls
    optionally rendered by server_info and server_state, and
    optionally printed to a distinct log file.
    - Tally each Job Queue task as it is queued, starts, and
      finishes running. Track total duration queued and running.
    - Tally each RPC call as it starts and either finishes
      successfully or throws an exception. Track total running
      duration for each.
  * Track currently executing Job Queue tasks and RPC methods
    along with durations.
  * Json-formatted performance log file written by a dedicated
    thread, for above-described data.
  * New optional parameter, "counters", for server_info and
    server_state. If set, render Job Queue and RPC call counters
    as well as currently executing tasks.
  * New configuration section, "[perf]", to optionally control
    performance logging to a file.
  * Support optional sub-second periods when rendering human-readable
    time points.

src/ripple/app/ledger/impl/LedgerToJson.cpp

@@ -20,6 +20,7 @@
 #include <ripple/app/ledger/LedgerToJson.h>
 #include <ripple/app/misc/TxQ.h>
 #include <ripple/basics/base_uint.h>
+#include <ripple/basics/date.h>
 
 namespace ripple {
 

src/ripple/app/main/Application.cpp

@@ -46,6 +46,7 @@
 #include <ripple/app/tx/apply.h>
 #include <ripple/basics/ResolverAsio.h>
 #include <ripple/basics/Sustain.h>
+#include <ripple/basics/PerfLog.h>
 #include <ripple/json/json_reader.h>
 #include <ripple/nodestore/DummyScheduler.h>
 #include <ripple/overlay/Cluster.h>
@@ -269,7 +270,7 @@ private:
         void operator() (Duration const& elapsed)
         {
             using namespace std::chrono;
-            auto const lastSample = ceil<milliseconds>(elapsed);
+            auto const lastSample = date::ceil<milliseconds>(elapsed);
 
             lastSample_ = lastSample;
 
@@ -306,6 +307,7 @@ public:
     std::unique_ptr<TimeKeeper> timeKeeper_;
 
     beast::Journal m_journal;
+    std::unique_ptr<perf::PerfLog> perfLog_;
     Application::MutexType m_masterMutex;
 
     // Required by the SHAMapStore
@@ -399,6 +401,11 @@ public:
 
         , m_journal (logs_->journal("Application"))
 
+        // PerfLog must be started before any other threads are launched.
+        , perfLog_ (perf::make_PerfLog(
+            perf::setup_PerfLog(config_->section("perf"), config_->CONFIG_DIR),
+            *this, logs_->journal("PerfLog"), [this] () { signalStop(); }))
+
         , m_txMaster (*this)
 
         , m_nodeStoreScheduler (*this)
@@ -425,7 +432,7 @@ public:
         //
         , m_jobQueue (std::make_unique<JobQueue>(
             m_collectorManager->group ("jobq"), m_nodeStoreScheduler,
-            logs_->journal("JobQueue"), *logs_))
+            logs_->journal("JobQueue"), *logs_, *perfLog_))
 
         //
         // Anything which calls addJob must be a descendant of the JobQueue
@@ -488,7 +495,8 @@ public:
             get_io_service (), *validators_, logs_->journal("ValidatorSite")))
 
         , serverHandler_ (make_ServerHandler (*this, *m_networkOPs, get_io_service (),
-            *m_jobQueue, *m_networkOPs, *m_resourceManager, *m_collectorManager))
+            *m_jobQueue, *m_networkOPs, *m_resourceManager,
+            *m_collectorManager))
 
         , mFeeTrack (std::make_unique<LoadFeeTrack>(logs_->journal("LoadManager")))
 
@@ -653,6 +661,11 @@ public:
         return m_txMaster;
     }
 
+    perf::PerfLog& getPerfLog () override
+    {
+        return *perfLog_;
+    }
+
     NodeCache& getTempNodeCache () override
     {
         return m_tempNodeCache;

src/ripple/app/main/Application.h

@@ -35,6 +35,7 @@ namespace ripple {
 namespace unl { class Manager; }
 namespace Resource { class Manager; }
 namespace NodeStore { class Database; class DatabaseShard; }
+namespace perf { class PerfLog; }
 
 // VFALCO TODO Fix forward declares required for header dependency loops
 class AmendmentTable;
@@ -152,6 +153,7 @@ public:
     virtual NetworkOPs&             getOPs () = 0;
     virtual OrderBookDB&            getOrderBookDB () = 0;
     virtual TransactionMaster&      getMasterTransaction () = 0;
+    virtual perf::PerfLog&          getPerfLog () = 0;
 
     virtual
     std::pair<PublicKey, SecretKey> const&

src/ripple/app/main/Main.cpp

@@ -166,7 +166,8 @@ void printHelp (const po::options_description& desc)
            "     ripple ...\n"
            "     ripple_path_find <json> [<ledger>]\n"
            "     version\n"
-           "     server_info\n"
+           "     server_info [counters]\n"
+           "     server_state [counters]\n"
            "     sign <private_key> <tx_json> [offline]\n"
            "     sign_for <signer_address> <signer_private_key> <tx_json> [offline]\n"
            "     stop\n"

src/ripple/app/misc/NetworkOPs.cpp

@@ -40,6 +40,7 @@
 #include <ripple/app/misc/impl/AccountTxPaging.h>
 #include <ripple/app/tx/apply.h>
 #include <ripple/basics/mulDiv.h>
+#include <ripple/basics/PerfLog.h>
 #include <ripple/basics/UptimeTimer.h>
 #include <ripple/core/ConfigSections.h>
 #include <ripple/crypto/csprng.h>
@@ -357,7 +358,7 @@ public:
     void consensusViewChange () override;
 
     Json::Value getConsensusInfo () override;
-    Json::Value getServerInfo (bool human, bool admin) override;
+    Json::Value getServerInfo (bool human, bool admin, bool counters) override;
     void clearLedgerFetch () override;
     Json::Value getLedgerFetchInfo () override;
     std::uint32_t acceptLedger (
@@ -2078,7 +2079,7 @@ Json::Value NetworkOPsImp::getConsensusInfo ()
     return mConsensus.getJson (true);
 }
 
-Json::Value NetworkOPsImp::getServerInfo (bool human, bool admin)
+Json::Value NetworkOPsImp::getServerInfo (bool human, bool admin, bool counters)
 {
     Json::Value info = Json::objectValue;
 
@@ -2090,6 +2091,9 @@ Json::Value NetworkOPsImp::getServerInfo (bool human, bool admin)
 
     info [jss::server_state] = strOperatingMode ();
 
+    info [jss::time] = to_string(date::floor<std::chrono::microseconds>(
+        std::chrono::system_clock::now()));
+
     if (needNetworkLedger_)
         info[jss::network_ledger] = "waiting";
 
@@ -2136,6 +2140,12 @@ Json::Value NetworkOPsImp::getServerInfo (bool human, bool admin)
         }
     }
 
+    if (counters)
+    {
+        info[jss::counters] = app_.getPerfLog().countersJson();
+        info[jss::current_activities] = app_.getPerfLog().currentJson();
+    }
+
     info[jss::pubkey_node] = toBase58 (
         TokenType::NodePublic,
         app_.nodeIdentity().first);

src/ripple/app/misc/NetworkOPs.h

@@ -177,7 +177,8 @@ public:
     virtual void consensusViewChange () = 0;
 
     virtual Json::Value getConsensusInfo () = 0;
-    virtual Json::Value getServerInfo (bool human, bool admin) = 0;
+    virtual Json::Value getServerInfo (
+        bool human, bool admin, bool counters) = 0;
     virtual void clearLedgerFetch () = 0;
     virtual Json::Value getLedgerFetchInfo () = 0;
 

src/ripple/app/misc/impl/ValidatorList.cpp

@@ -18,6 +18,7 @@
 //==============================================================================
 
 #include <ripple/app/misc/ValidatorList.h>
+#include <ripple/basics/date.h>
 #include <ripple/basics/Slice.h>
 #include <ripple/basics/StringUtilities.h>
 #include <ripple/json/json_reader.h>
@@ -490,9 +491,13 @@ ValidatorList::getJson() const
     if (auto when = expires())
     {
         if (*when == TimeKeeper::time_point::max())
+        {
             res[jss::validator_list_expires] = "never";
+        }
         else
+        {
             res[jss::validator_list_expires] = to_string(*when);
+        }
     }
     else
         res[jss::validator_list_expires] = "unknown";