Performance logging and counters:

* Tally and duration counters for Job Queue tasks and RPC calls
    optionally rendered by server_info and server_state, and
    optionally printed to a distinct log file.
    - Tally each Job Queue task as it is queued, starts, and
      finishes running. Track total duration queued and running.
    - Tally each RPC call as it starts and either finishes
      successfully or throws an exception. Track total running
      duration for each.
  * Track currently executing Job Queue tasks and RPC methods
    along with durations.
  * Json-formatted performance log file written by a dedicated
    thread, for above-described data.
  * New optional parameter, "counters", for server_info and
    server_state. If set, render Job Queue and RPC call counters
    as well as currently executing tasks.
  * New configuration section, "[perf]", to optionally control
    performance logging to a file.
  * Support optional sub-second periods when rendering human-readable
    time points.

src/ripple/core/impl/JobQueue.cpp

@@ -19,18 +19,21 @@
 
 #include <ripple/core/JobQueue.h>
 #include <ripple/basics/contract.h>
+#include <ripple/basics/PerfLog.h>
 
 namespace ripple {
 
 JobQueue::JobQueue (beast::insight::Collector::ptr const& collector,
-    Stoppable& parent, beast::Journal journal, Logs& logs)
+    Stoppable& parent, beast::Journal journal, Logs& logs,
+    perf::PerfLog& perfLog)
     : Stoppable ("JobQueue", parent)
     , m_journal (journal)
     , m_lastJob (0)
-    , m_invalidJobData (getJobTypes ().getInvalid (), collector, logs)
+    , m_invalidJobData (JobTypes::instance().getInvalid (), collector, logs)
     , m_processCount (0)
-    , m_workers (*this, "JobQueue", 0)
+    , m_workers (*this, perfLog, "JobQueue", 0)
     , m_cancelCallback (std::bind (&Stoppable::isStopping, this))
+    , perfLog_ (perfLog)
     , m_collector (collector)
 {
     hook = m_collector->make_hook (std::bind (&JobQueue::collect, this));
@@ -39,7 +42,7 @@ JobQueue::JobQueue (beast::insight::Collector::ptr const& collector,
     {
         std::lock_guard <std::mutex> lock (m_mutex);
 
-        for (auto const& x : getJobTypes ())
+        for (auto const& x : JobTypes::instance())
         {
             JobTypeInfo const& jt = x.second;
 
@@ -329,6 +332,7 @@ JobQueue::queueJob (Job const& job, std::lock_guard <std::mutex> const& lock)
     JobType const type (job.getType ());
     assert (type != jtINVALID);
     assert (m_jobSet.find (job) != m_jobSet.end ());
+    perfLog_.jobQueue(type);
 
     JobTypeData& data (getJobTypeData (type));
 
@@ -398,34 +402,13 @@ JobQueue::finishJob (JobType type)
     --data.running;
 }
 
-template <class Rep, class Period>
-void JobQueue::on_dequeue (JobType type,
-    std::chrono::duration <Rep, Period> const& value)
-{
-    using namespace std::chrono;
-    auto const ms = ceil<milliseconds>(value);
-
-    if (ms >= 10ms)
-        getJobTypeData (type).dequeue.notify (ms);
-}
-
-template <class Rep, class Period>
-void JobQueue::on_execute (JobType type,
-    std::chrono::duration <Rep, Period> const& value)
-{
-    using namespace std::chrono;
-    auto const ms (ceil <milliseconds> (value));
-
-    if (ms >= 10ms)
-        getJobTypeData (type).execute.notify (ms);
-}
-
 void
-JobQueue::processTask ()
+JobQueue::processTask (int instance)
 {
     JobType type;
 
     {
+        using namespace std::chrono;
         Job::clock_type::time_point const start_time (
             Job::clock_type::now());
         {
@@ -438,10 +421,18 @@ JobQueue::processTask ()
             type = job.getType();
             JobTypeData& data(getJobTypeData(type));
             JLOG(m_journal.trace()) << "Doing " << data.name () << " job";
-            on_dequeue (job.getType (), start_time - job.queue_time ());
+            auto const us = date::ceil<microseconds>(
+                start_time - job.queue_time());
+            perfLog_.jobStart(type, us, start_time, instance);
+            if (us >= 10ms)
+                getJobTypeData(type).dequeue.notify(us);
             job.doJob ();
         }
-        on_execute(type, Job::clock_type::now() - start_time);
+        auto const us (
+            date::ceil<microseconds>(Job::clock_type::now() - start_time));
+        perfLog_.jobFinish(type, us, instance);
+        if (us >= 10ms)
+            getJobTypeData(type).execute.notify(us);
     }
 
     {
@@ -462,7 +453,7 @@ JobQueue::processTask ()
 int
 JobQueue::getJobLimit (JobType type)
 {
-    JobTypeInfo const& j (getJobTypes ().get (type));
+    JobTypeInfo const& j (JobTypes::instance().get (type));
     assert (j.type () != jtINVALID);
 
     return j.limit ();