Merge branch 'pratik/otel-phase10-workload-validation' into pratik/perf-test-otel-on

include/xrpl/core/Job.h

@@ -109,6 +109,20 @@ public:
     [[nodiscard]] JobType
     getType() const;
 
+    /**
+     * Returns the job name supplied to JobQueue::addJob.
+     *
+     * Several job types have more than one producer (for example both
+     * RcvGetLedger and RcvGetObjByHash run as JtLedgerReq), so the name is
+     * the only thing that tells them apart. It is read by the JobQueue
+     * PerfLog hooks and exported as the `handler` metric label.
+     *
+     * Empty for the default and index-only constructors, which carry no
+     * name; callers must treat an empty name as "unknown".
+     */
+    [[nodiscard]] std::string const&
+    getName() const;
+
     /**
      * Returns the time when the job was queued.
      */

include/xrpl/core/JobQueue.h

@@ -334,7 +334,9 @@ private:
     //  any.
     //
     // Invariants:
-    //  <none>
+    //  The calling thread owns the JobLock. This function mutates the
+    //  deferred and running counts, which mutex_ guards; collect() reads
+    //  them under the same lock.
     void
     finishJob(JobType type);
 

include/xrpl/core/JobTypeData.h

@@ -3,6 +3,7 @@
 #include <xrpl/basics/Log.h>
 #include <xrpl/beast/insight/Collector.h>
 #include <xrpl/beast/insight/Event.h>
+#include <xrpl/beast/insight/Gauge.h>
 #include <xrpl/core/Job.h>
 #include <xrpl/core/JobTypeInfo.h>
 #include <xrpl/core/LoadMonitor.h>
@@ -20,6 +21,23 @@ private:
     beast::insight::Collector::ptr collector_;
 
 public:
+    /**
+     * Metric-name suffixes appended to `JobTypeInfo::name()`.
+     *
+     * The constructor builds the instrument names from these. Public so
+     * tests can assert on the exported names without repeating the
+     * literals. The collector is the `"jobq"` group, so
+     * `GroupImp::makeName()` prefixes `jobq.` and `OTelCollector`
+     * lowercases and turns `.` into `_`: `ledgerRequest` +
+     * `kSuffixDeferred` exports as `jobq_ledgerrequest_deferred`.
+     */
+    /** @{ */
+    static constexpr char kSuffixWaiting[] = "_waiting";
+    static constexpr char kSuffixRunning[] = "_running";
+    static constexpr char kSuffixDeferred[] = "_deferred";
+    static constexpr char kSuffixQueued[] = "_q";
+    /** @} */
+
     /* The job category which we represent */
     JobTypeInfo const& info;
 
@@ -36,6 +54,32 @@ public:
     beast::insight::Event dequeue;
     beast::insight::Event execute;
 
+    /**
+     * Saturation gauges, published by `JobQueue::collect()`.
+     *
+     * Each mirrors the same-named counter above so per-job-type queue
+     * pressure is visible in metrics. Without them the only exported queue
+     * signal is the process-wide `jobq_job_count`, which cannot attribute
+     * pressure to a job type.
+     *
+     * `waitingGauge` is the backlog not yet started, `runningGauge` is the
+     * in-flight count, and `deferredGauge` is the count held back by this
+     * type's concurrency limit. `deferredGauge` is the leading indicator:
+     * `JobQueue::addJob()` never rejects, so a capped type under pressure
+     * shows up as latency only after the fact, whereas a non-zero deferred
+     * reading precedes it.
+     *
+     * Created only for non-special types (see the constructor). A default
+     * constructed `beast::insight::Gauge` holds a null impl and every
+     * mutator is a no-op, so a special type's gauge is safe to assign to
+     * but publishes nothing.
+     */
+    /** @{ */
+    beast::insight::Gauge waitingGauge;
+    beast::insight::Gauge runningGauge;
+    beast::insight::Gauge deferredGauge;
+    /** @} */
+
     JobTypeData(
         JobTypeInfo const& info,
         beast::insight::Collector::ptr collector,
@@ -45,10 +89,17 @@ public:
     {
         load_.setTargetLatency(info.getAverageLatency(), info.getPeakLatency());
 
+        // Special types have limit_ == 0 and bypass the limit logic
+        // entirely, so their `deferred` is always 0. Excluding them here
+        // matches the existing dequeue/execute rule.
         if (!info.special())
         {
-            dequeue = collector_->makeEvent(info.name() + "_q");
+            dequeue = collector_->makeEvent(info.name() + kSuffixQueued);
             execute = collector_->makeEvent(info.name());
+
+            waitingGauge = collector_->makeGauge(info.name() + kSuffixWaiting);
+            runningGauge = collector_->makeGauge(info.name() + kSuffixRunning);
+            deferredGauge = collector_->makeGauge(info.name() + kSuffixDeferred);
         }
     }
 

include/xrpl/core/PerfLog.h

@@ -92,30 +92,41 @@ public:
      * Log queued job
      *
      * @param type Job type
+     * @param name Job name as given to JobQueue::addJob. Distinguishes the
+     * several producers that share one job type. May be empty.
      */
     virtual void
-    jobQueue(JobType const type) = 0;
+    jobQueue(JobType const type, std::string const& name) = 0;
 
     /**
      * Log job executing
      *
      * @param type Job type
+     * @param name Job name as given to JobQueue::addJob. Distinguishes the
+     * several producers that share one job type. May be empty.
      * @param dur Duration enqueued in microseconds
      * @param startTime Time that execution began
      * @param instance JobQueue worker thread instance
      */
     virtual void
-    jobStart(JobType const type, microseconds dur, steady_time_point startTime, int instance) = 0;
+    jobStart(
+        JobType const type,
+        std::string const& name,
+        microseconds dur,
+        steady_time_point startTime,
+        int instance) = 0;
 
     /**
      * Log job finishing
      *
      * @param type Job type
+     * @param name Job name as given to JobQueue::addJob. Distinguishes the
+     * several producers that share one job type. May be empty.
      * @param dur Duration running in microseconds
      * @param instance Jobqueue worker thread instance
      */
     virtual void
-    jobFinish(JobType const type, microseconds dur, int instance) = 0;
+    jobFinish(JobType const type, std::string const& name, microseconds dur, int instance) = 0;
 
     /**
      * Render performance counters in Json

include/xrpl/telemetry/GetObjectMetricNames.h

@@ -0,0 +1,160 @@
+#pragma once
+
+// cspell:ignore ISTOGRAM
+// The all-caps macro name XRPL_METRIC_HISTOGRAM_RECORD trips cspell's
+// compound-word splitter, which emits the subword "ISTOGRAM"; ignore it here.
+
+/**
+ * Metric names, label keys, label values, and descriptions for the
+ * `TMGetObjectByHash` request path.
+ *
+ * These constants are shared across two modules, which is why they live in a
+ * header rather than in either translation unit's unnamed namespace:
+ *
+ *   GetObjectMetricNames.h
+ *          |
+ *          +--> PeerImp.cpp          (XRPL_METRIC_* call sites: records the
+ *          |                          instruments)
+ *          |
+ *          +--> MetricsRegistry.cpp  (addMicrosecondHistogramView: registers
+ *                                     the explicit bucket boundaries for
+ *                                     kGetObjectLookupUs)
+ *
+ * `kGetObjectLookupUs` in particular is referenced from both sites. A
+ * copy-pasted literal would let the two drift, and a drifted name silently
+ * drops the bucket override -- the histogram would fall back to the SDK
+ * default boundaries, which top out at 10,000 (10 ms), so every quantile
+ * would saturate. This mirrors the reason the existing
+ * `kJobQueuedDurationUs` / `kJobRunningDurationUs` / `kRpcMethodDurationUs`
+ * constants exist in MetricsRegistry.cpp; those three are referenced only
+ * within that one file, so they stay file-local.
+ *
+ * Placed under `include/xrpl/telemetry/` because the two consumers sit in
+ * different levelization modules: PeerImp.cpp is `xrpld.overlay` and
+ * MetricsRegistry.cpp is `xrpld.telemetry`. Both are allowed to depend on
+ * `xrpl.telemetry` (see `xrpld.overlay > xrpl.telemetry` and
+ * `xrpld.telemetry > xrpl.telemetry` in the levelization ordering results), so
+ * `include/xrpl/` is the one level both can reach. Keeping the constants in
+ * `src/xrpld/telemetry/` would have required overlay to include a private
+ * xrpld header from another module, flipping a levelization edge. Both sites
+ * include this file as `<xrpl/telemetry/GetObjectMetricNames.h>`.
+ *
+ * Example usage -- recording a histogram:
+ * @code
+ * XRPL_METRIC_HISTOGRAM_RECORD(
+ *     app_, kGetObjectRequestObjects, kGetObjectRequestObjectsDesc, requested);
+ * @endcode
+ *
+ * Example usage -- edge case: the same instrument recorded under two
+ * different label values, which is why the label key and both values are
+ * constants rather than literals:
+ * @code
+ * XRPL_METRIC_COUNTER_ADD_LABELED(
+ *     app_, kGetObjectLookupsTotal, kGetObjectLookupsTotalDesc, hits,
+ *     {{kLabelResult, std::string(kResultHit)}});
+ * XRPL_METRIC_COUNTER_ADD_LABELED(
+ *     app_, kGetObjectLookupsTotal, kGetObjectLookupsTotalDesc, misses,
+ *     {{kLabelResult, std::string(kResultMiss)}});
+ * @endcode
+ *
+ * @note These are `constexpr char[]`, not `constexpr std::string_view`. The
+ * OTel C++ API takes `nostd::string_view`, which on this build is OTel's own
+ * type (the build defines only OPENTELEMETRY_ABI_VERSION_NO, not
+ * OPENTELEMETRY_STL_VERSION, so `nostd::string_view` is not an alias for
+ * `std::string_view`). It converts from `char const*` and from
+ * `std::string`, but has no converting constructor from `std::string_view`,
+ * so a `string_view` constant would not compile at the call sites. This also
+ * matches the existing convention in MetricsRegistry.cpp.
+ *
+ * @note Header-only constants with no runtime state, so there is nothing to
+ * synchronize -- safe to include from any thread context.
+ */
+
+namespace xrpl::telemetry {
+
+// ===== Metric names ==========================================================
+
+/**
+ * Requests refused by `onMessage(TMGetObjectByHash)` before any NodeStore
+ * access, split by the `reason` label.
+ */
+inline constexpr char kGetObjectRejectedTotal[] = "getobject_rejected_total";
+
+/**
+ * Distribution of the object count requested per message.
+ */
+inline constexpr char kGetObjectRequestObjects[] = "getobject_request_objects";
+
+/**
+ * Wall time spent in the NodeStore fetch loop, in microseconds.
+ *
+ * Referenced twice: here at the record site, and by
+ * `addMicrosecondHistogramView()` in MetricsRegistry.cpp. Both must use this
+ * one constant.
+ */
+inline constexpr char kGetObjectLookupUs[] = "getobject_lookup_us";
+
+/**
+ * NodeStore lookups performed, split by the `result` label.
+ */
+inline constexpr char kGetObjectLookupsTotal[] = "getobject_lookups_total";
+
+/**
+ * Distribution of the dynamic resource charge applied per request.
+ */
+inline constexpr char kGetObjectCharge[] = "getobject_charge";
+
+// ===== Label keys ============================================================
+
+/**
+ * Label key distinguishing a hit from a miss on `kGetObjectLookupsTotal`.
+ */
+inline constexpr char kLabelResult[] = "result";
+
+/**
+ * Label key naming which gate refused the request.
+ */
+inline constexpr char kLabelReason[] = "reason";
+
+// ===== Label values ==========================================================
+
+/**
+ * `kLabelResult` value: the object was found in the NodeStore.
+ */
+inline constexpr char kResultHit[] = "hit";
+
+/**
+ * `kLabelResult` value: the object was not found.
+ */
+inline constexpr char kResultMiss[] = "miss";
+
+/**
+ * `kLabelReason` value: more objects requested than the handler accepts.
+ */
+inline constexpr char kReasonOversize[] = "oversize";
+
+/**
+ * `kLabelReason` value: the ledger hash was not uint256-sized.
+ */
+inline constexpr char kReasonMalformedLedgerHash[] = "malformed_ledgerhash";
+
+// ===== Instrument descriptions ===============================================
+
+/** @{ */
+inline constexpr char kGetObjectRejectedTotalDesc[] =
+    "TMGetObjectByHash requests refused before any NodeStore access, by reason";
+
+inline constexpr char kGetObjectRequestObjectsDesc[] =
+    "Objects requested per TMGetObjectByHash message";
+
+inline constexpr char kGetObjectLookupUsDesc[] =
+    "Time spent in the TMGetObjectByHash NodeStore fetch loop, in microseconds";
+
+inline constexpr char kGetObjectLookupsTotalDesc[] =
+    "TMGetObjectByHash NodeStore lookups, by hit/miss result";
+
+inline constexpr char kGetObjectChargeDesc[] =
+    "Dynamic resource charge applied per TMGetObjectByHash request";
+/** @} */
+
+}  // namespace xrpl::telemetry