diff --git a/.cspell.config.yaml b/.cspell.config.yaml index f8781dc990..3a3a122e2f 100644 --- a/.cspell.config.yaml +++ b/.cspell.config.yaml @@ -146,6 +146,7 @@ words: - hwrap - ifndef - inequation + - initialiser - insuf - insuff - invasively @@ -286,6 +287,11 @@ words: - rustfmt - rustup - sahyadri + - sanitisation + - sanitise + - sanitised + - sanitiser + - sanitising - Satoshi - scons - Schnorr @@ -305,6 +311,7 @@ words: - sles - soci - socidb + - speciality - sponsee - sponsees - SRPMS @@ -365,6 +372,7 @@ words: - unsquelch - unsquelched - unsquelching + - unstored - unvalidated - unveto - unvetoed @@ -375,6 +383,7 @@ words: - vfalco - vinnie - wasmi + - werror - wextra - wptr - writeme diff --git a/.github/scripts/otel-naming/check_otel_naming.py b/.github/scripts/otel-naming/check_otel_naming.py index 2847397afd..685d7489c4 100644 --- a/.github/scripts/otel-naming/check_otel_naming.py +++ b/.github/scripts/otel-naming/check_otel_naming.py @@ -914,8 +914,17 @@ def metric_constants(root: Path) -> Tuple[Set[str], Set[str], Set[str]]: comment cannot seed the authoritative set (same reasoning as `strip_comments` for L1 spans). - A constant in none of the three namespaces is ignored rather than guessed - at, keeping the derivation conservative in the same direction as L1.""" + Two header styles are recognised, because both are in use: + + * Namespaced: constants sit inside `namespace metric` / `label` / `lval`, + and the enclosing namespace decides the bucket. + * Flat `k`-prefixed: a header with no such namespaces names the role in the + identifier instead -- `kLabelFoo` is a label key, `kResultFoo` and + `kReasonFoo` are label values. Used by the per-subsystem headers. + + A constant that neither sits in one of the three namespaces nor carries a + recognised prefix is ignored rather than guessed at, keeping the derivation + conservative in the same direction as L1.""" names: Set[str] = set() keys: Set[str] = set() values: Set[str] = set() @@ -929,6 +938,14 @@ def metric_constants(root: Path) -> Tuple[Set[str], Set[str], Set[str]]: for block in namespace_spans(text, ns): for m in METRIC_CONST_DEF.finditer(block): bucket.add(m.group(2)) + # Flat style: classify by identifier prefix. Only constants outside the + # namespaced blocks reach here, so a namespaced header is unaffected. + for m in METRIC_CONST_DEF.finditer(text): + ident, value = m.group(1), m.group(2) + if ident.startswith("kLabel"): + keys.add(value) + elif ident.startswith(("kResult", "kReason")): + values.add(value) return names, keys, values diff --git a/.github/scripts/otel-naming/test_check_otel_naming.py b/.github/scripts/otel-naming/test_check_otel_naming.py index 451d4eff97..6a61ae2c1e 100644 --- a/.github/scripts/otel-naming/test_check_otel_naming.py +++ b/.github/scripts/otel-naming/test_check_otel_naming.py @@ -1359,7 +1359,7 @@ class InstrumentKindClassification(unittest.TestCase): class MetricPrefixFamilies(unittest.TestCase): def test_first_segment_is_the_family(self): self.assertEqual( - chk.metric_prefixes({"sync_state", "jobq_backlog", "unl_quorum"}), + chk.metric_prefixes({"sync_state", "jobq_saturation", "unl_quorum"}), {"sync_", "jobq_", "unl_"}, ) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 778babf012..e88183a285 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -432,7 +432,7 @@ the C++ layers; a CI check validates the layers that cannot reference a constant 3. A duration carries its unit as the suffix — `_us`, `_ms` or `_seconds`. The unit belongs in the name because the OTel `unit` argument is not surfaced on the Prometheus metric name. -4. A gauge that snapshots current state takes no suffix (`jobq_backlog`, +4. A gauge that snapshots current state takes no suffix (`jobq_saturation`, `sync_state`), and never `_total`. 5. Label keys are `lower_snake_case` and must have **bounded** cardinality. A multi-series gauge discriminates its readings with the `metric` label rather diff --git a/OpenTelemetryPlan/09-data-collection-reference.md b/OpenTelemetryPlan/09-data-collection-reference.md index 2c7c0e00b7..544a06a6d7 100644 --- a/OpenTelemetryPlan/09-data-collection-reference.md +++ b/OpenTelemetryPlan/09-data-collection-reference.md @@ -50,6 +50,7 @@ graph LR style A fill:#4a90d9,color:#fff,stroke:#2a6db5 style B fill:#4a90d9,color:#fff,stroke:#2a6db5 + style C fill:#4a90d9,color:#fff,stroke:#2a6db5 style R1 fill:#5cb85c,color:#fff,stroke:#3d8b3d style BP fill:#449d44,color:#fff,stroke:#2d6e2d style SM fill:#449d44,color:#fff,stroke:#2d6e2d @@ -63,10 +64,17 @@ graph LR style viz fill:#1a2d33,color:#ccc,stroke:#5bc0de ``` -There are two independent telemetry pipelines entering a single **OTel Collector** via the same OTLP receiver: +There are three independent telemetry pipelines entering a single **OTel Collector** via the same OTLP receiver — nodes **A**, **B**, and **C** in the diagram above: -1. **OpenTelemetry Traces** — Distributed spans with attributes, exported via OTLP/HTTP (:4318) to the collector's **OTLP Receiver**. The **Batch Processor** groups spans (1s timeout, batch size 100) before forwarding to trace backends. The **SpanMetrics Connector** derives RED metrics (rate, errors, duration) from every span and feeds them into the metrics pipeline. -2. **beast::insight OTel Metrics** — System-level gauges, counters, and histograms exported natively via OTLP/HTTP (:4318) to the same **OTLP Receiver**. These are batched and exported to Prometheus alongside span-derived metrics. The StatsD UDP transport has been replaced by native OTLP; `server=statsd` remains available as a fallback. +1. **OpenTelemetry Traces** (**A**) — Distributed spans with attributes, exported via OTLP/HTTP (:4318) to the collector's **OTLP Receiver**. The **Batch Processor** groups spans (1s timeout, batch size 100) before forwarding to trace backends. The **SpanMetrics Connector** derives RED metrics (rate, errors, duration) from every span and feeds them into the metrics pipeline. +2. **beast::insight OTel Metrics** (**B**) — System-level gauges, counters, and histograms exported natively via OTLP/HTTP (:4318) to the same **OTLP Receiver**. These are batched and exported to Prometheus alongside span-derived metrics. The StatsD UDP transport has been replaced by native OTLP; `server=statsd` remains available as a fallback. +3. **MetricsRegistry OTel SDK Metrics** (**C**) — Counters, histograms, and observable gauges registered directly with the OTel Metrics SDK, exported via OTLP/HTTP (:4318). This pipeline owns its own `MeterProvider` and reader, separate from **B**'s, so its export cadence is independent — see [§2.5](#25-per-job-type-queue-gauges) for why that matters when comparing the two. + +A third, narrower metrics path exists for instruments created at their call site through the +`XRPL_METRIC_*` macros. These use the OTel Metrics SDK directly and reach the collector's OTLP +receiver rather than the StatsD receiver, so their names carry no `xrpld_` prefix. See +[§2a](#2a-call-site-otel-metrics-metricsregistry). Code in `libxrpl` cannot use these macros and +always goes through `beast::insight` instead. **Trace backend** — The collector exports traces via OTLP/gRPC to: @@ -629,6 +637,73 @@ For each of the 45+ overlay traffic categories (defined in `TrafficCount.h`), fo **Grafana dashboards**: _Network Traffic_ (`network-traffic`), _Overlay Traffic Detail_ (`overlay-traffic-detail`), _Ledger Data & Sync_ (`ledger-data-sync`) +### 2.5 Per-Job-Type Queue Gauges + +Three gauge families give per-job-type queue pressure. Before them the only +exported queue signal was the process-wide `jobq_job_count` +([§2.1](#21-gauges)), which cannot attribute pressure to a job type. + +| Prometheus Metric | Description | +| ------------------------- | --------------------------------------------------- | +| `jobq__waiting` | Backlog for this type: enqueued but not yet started | +| `jobq__running` | Currently executing for this type | +| `jobq__deferred` | Held back by this type's concurrency limit | + +The gauge members live on `JobTypeData` (`include/xrpl/core/JobTypeData.h`) and +are published by `JobQueue::collect()` +(`src/libxrpl/core/detail/JobQueue.cpp`), which reads the same +`waiting`/`running`/`deferred` counters under the mutex that guards them. Values +are clamped at zero before publication, because the gauge value type is unsigned +and an unclamped negative would wrap to ~1.8e19 and swamp every panel reading +the family. + +**Name derivation.** The collector is the `"jobq"` group +(`src/xrpld/app/main/Application.cpp`), so `GroupImp::makeName()` +(`src/libxrpl/beast/insight/Groups.cpp`) joins with a `.`, then +`OTelCollectorImp::formatName()` +(`src/libxrpl/beast/insight/OTelCollector.cpp`) lowercases and maps `.` to +`_`. The exported +name for `JtLedgerReq`, whose `JobTypeInfo` name is `ledgerRequest`, is +therefore `jobq_ledgerrequest_deferred` — bare and lowercase, with no `xrpld` +prefix. The same chain produces `jobq_job_count` from the gauge registered as +`job_count`. + +**Coverage.** Emitted for the **35** job types that are not special. `JobTypes` +defines 46 entries plus the `invalid` sentinel +(`include/xrpl/core/JobTypes.h`); `JobTypeInfo::special()` is `limit_ == 0`, and +11 of the 46 have `limit == 0`, so `JobTypeData`'s constructor creates gauges for +the remaining 35. A special type's gauge stays default-constructed, and a +default `beast::insight::Gauge` holds a null impl whose mutators are no-ops, so +assigning to it publishes nothing. + +**Why `deferred` is the metric to alert on.** `JobQueue::addJob()` never +rejects — it defers. A capped type under pressure therefore surfaces as latency +only after the fact, whereas a non-zero `deferred` reading precedes it. The +types where this bites are the ones with a low concurrency limit +(`JobTypes.h`): `JtPack` = 1 and `JtUpdatePf` = 1, `JtLedgerReq` = 3 and +`JtLedgerData` = 3, `JtTxnData` = 5. + +> **Sampling caveat.** These are sampled, not integrated. The values are read +> when the SDK's periodic reader invokes the observable callbacks, which run the +> collector hooks; the export interval is 1000 ms +> (`export_interval_millis` in `src/libxrpl/telemetry/Telemetry.cpp:441`) and +> hook invocation is debounced to at most once per 500 ms. A spike shorter than +> the interval can be missed entirely, so read these as pressure indicators +> rather than as exact peak depths. + +**Pipeline note.** Unlike the Phase 9 `job_*` counters and histograms, this +family flows through `beast::insight` → `OTelCollector`, **not** the +`XRPL_METRIC_*` macros. `JobQueue.cpp` is in `libxrpl` and those macros are +`xrpld`-only. The two are distinct pipelines — arrows **B** and **C** in the +[Data Flow Overview](#data-flow-overview) — each with its own `MeterProvider`, +reader, and OTLP/HTTP exporter, even though both request a meter named +`xrpld` / `1.0.0`. `OTelCollector` takes its meter from the **global** provider, +which `Telemetry` publishes and reads every 1000 ms; `MetricsRegistry` builds a +private provider it does not publish, read every 10000 ms +(`src/xrpld/telemetry/MetricsRegistry.cpp`). So `jobq__*` and +`job_*_total` reach Prometheus on different cadences and should not be assumed +sampled at the same instant. + --- ## 3. Grafana Dashboard Reference @@ -942,6 +1017,21 @@ async callbacks for new categories. | `nodestore_state{metric="read_threads_running"}` | Gauge | `metric` | Active read threads | | `nodestore_state{metric="read_threads_total"}` | Gauge | `metric` | Total read threads configured | +#### Job Queue and GetObject Additions + +Three further additions are catalogued with the families they extend rather than +repeated here: + +- A `handler` label on the five `job_*` instruments, so producers sharing one + job type stay individually attributable — see + [Per-Job-Type Metrics](#per-job-type-metrics-synchronous-countershistogram). +- Five `getobject_*` instruments covering the `TMGetObjectByHash` request path — + see [GetObject Request Path](#getobject-request-path-synchronous-countershistograms). +- Three per-job-type queue gauge families (`jobq__waiting` / + `_running` / `_deferred`). These travel the `beast::insight` pipeline, not the + OTel SDK one, so they are documented in + [§2.5](#25-per-job-type-queue-gauges). + ### New Grafana Dashboards (Phase 9) | Dashboard | UID | Data Source | Key Panels | @@ -988,10 +1078,23 @@ docker/telemetry/workload/benchmark.sh --xrpld .build/xrpld --duration 300 | Span attributes | per-span assertion | Per-span attribute assertion | `expected_spans.json` | | Legacy beast::insight families | ~270 (≈224 traffic) | Prometheus `__name__` query | `expected_metrics.json` | | Native MetricsRegistry | 35 instruments | Prometheus query | `expected_metrics.json` | +| Call-site `XRPL_METRIC_*` | 7 instruments | Prometheus query | `expected_metrics.json` | +| Per-job-type gauges | 105 (35 types × 3) | Prometheus `__name__` query | `expected_metrics.json` | | SpanMetrics RED | 4 per span | Prometheus query | `expected_metrics.json` | | Grafana dashboards | 15 | Dashboard API "no data" check | `expected_metrics.json` | | Log-trace links | Present | Loki query + Tempo reverse check | — | +The two added rows are the families that do not originate as `MetricsRegistry` +members. **Call-site** instruments are declared by the `XRPL_METRIC_*` macros +(7 distinct names: `rpc_in_flight_requests`, `ledgers_closed_total`, and the +five `getobject_*`). Two of those seven are workload-gated in a way that makes a +zero reading uninformative: `getobject_rejected_total` needs a non-conforming +request, and the `getobject_*` family as a whole needs an inbound +`TMGetObjectByHash`. **Per-job-type gauges** are the `beast::insight` families +from [§2.5](#25-per-job-type-queue-gauges); all 105 should be present on any +running node, but `_deferred` reads zero unless a capped type is actually +saturated. + ### Performance Overhead Targets | Metric | Target | Measurement Method | @@ -1131,13 +1234,124 @@ reserved for monotonic counters). #### Per-Job-Type Metrics (Synchronous Counters/Histogram) -| Prometheus Metric | Type | Labels | Description | -| -------------------- | --------- | ------------------- | --------------------------------- | -| `job_queued_total` | Counter | `job_type=""` | Jobs enqueued | -| `job_started_total` | Counter | `job_type=""` | Jobs started | -| `job_finished_total` | Counter | `job_type=""` | Jobs completed | -| `job_queued_us` | Histogram | `job_type=""` | Queue wait time distribution (us) | -| `job_running_us` | Histogram | `job_type=""` | Execution time distribution (us) | +| Prometheus Metric | Type | Labels | Description | +| -------------------- | --------- | --------------------------------------- | --------------------------------- | +| `job_queued_total` | Counter | `job_type=""`, `handler=""` | Jobs enqueued | +| `job_started_total` | Counter | `job_type=""`, `handler=""` | Jobs started | +| `job_finished_total` | Counter | `job_type=""`, `handler=""` | Jobs completed | +| `job_queued_us` | Histogram | `job_type=""`, `handler=""` | Queue wait time distribution (us) | +| `job_running_us` | Histogram | `job_type=""`, `handler=""` | Execution time distribution (us) | + +All five are recorded from `PerfLogImp` (`jobQueue()`, `jobStart()`, +`jobFinish()`) through `MetricsRegistry::recordJobQueued/Started/Finished`. +A counter and its paired histogram always carry the identical label set, so +the two can be joined in a query. + +**The `handler` label.** `job_type` alone cannot attribute load to a producer, +because several producers share one job type: `RcvGetLedger` and +`RcvGetObjByHash` both run as `JtLedgerReq`, and before this label they were +indistinguishable. `handler` is the `addJob` name, so each producer gets its +own series. + +The name is not used raw. `MetricsRegistry::sanitiseHandler()` keeps it only +when it is non-empty **and** every character is an ASCII letter; anything else +becomes the constant `MetricsRegistry::kHandlerOther`, `"other"`. The rule +exists because two job names embed a ledger sequence — `"Pub" + seq` in +`LedgerPersistence.cpp` and `"OB" + seq` in `OrderBookDBImpl.cpp` — which raw +would mint a fresh series per ledger. Both always contain digits, so both +always fold to `other` by construction. The label domain is therefore a +function of the string literals in the source and cannot grow at runtime; +a name added later that fails the rule degrades to `other` rather than +becoming unbounded. + +Current cardinality: **44 values** — 43 names pass through unchanged, plus +`other`. Five production names fail the letters-only rule and fold into +`other`: + +| Job Name | Job Type | Why it folds | +| ------------- | ------------------- | ------------ | +| `GetConsL1` | `JtAdvance` | digits | +| `GetConsL2` | `JtAdvance` | digits | +| `gRPC-Client` | `JtRpc` | hyphen | +| `RPC-Client` | `JtClientRpc` | hyphen | +| `WS-Client` | `JtClientWebsocket` | hyphen | + +> **`handler="other"` is a mixed bucket, not one producer.** It aggregates the +> five names above plus both dynamic names, so a rate or quantile on it is a +> sum across unrelated work. `GetConsL1` and `GetConsL2` are the sharpest case: +> they are two distinct `JtAdvance` producers that land in the same bucket and +> are mutually inseparable. Filter by `job_type` alongside `handler` to narrow +> it, and read `handler="other"` series as an aggregate only. + +#### GetObject Request Path (Synchronous Counters/Histograms) + +Instruments for the `TMGetObjectByHash` peer request path. Names, label keys, +and label values are the `constexpr` constants in +`include/xrpl/telemetry/GetObjectMetricNames.h`. All five are declared at their +call sites in `src/xrpld/overlay/detail/PeerImp.cpp` via the `XRPL_METRIC_*` +macros, not as `MetricsRegistry` members: the two rejection counters in +`onMessage(TMGetObjectByHash)`, the other three in the +`recordGetObjectMetrics()` helper. + +| Prometheus Metric | Type | Labels | Description | +| --------------------------- | --------- | ----------------------------------------------- | ------------------------------------------------------------ | +| `getobject_lookup_us` | Histogram | (none) | Wall time in the NodeStore fetch loop (us), once per request | +| `getobject_request_objects` | Histogram | (none) | Objects requested per message | +| `getobject_lookups_total` | Counter | `result="hit"` \| `"miss"` | NodeStore lookups, added once per request with batch totals | +| `getobject_rejected_total` | Counter | `reason="oversize"` \| `"malformed_ledgerhash"` | Requests refused before any NodeStore access | +| `getobject_charge` | Histogram | (none) | Dynamic component of the differential resource charge | + +**Per request, not per object.** `getobject_lookup_us` times the whole fetch +loop once, and `getobject_lookups_total` adds the batch hit and miss totals in +two calls. Incrementing per object on a loop bounded by +`Tuning::kHardMaxReplyNodes` (12288) would cost measurably and add no +information the batch totals do not already carry. + +**All three histograms need an explicit bucket view.** The SDK's default +histogram boundaries top out at 10000. Every one of these three exceeds that, so +without a view their top quantiles would all read as a flat 10000. Six views are +registered in `src/xrpld/telemetry/MetricsRegistry.cpp`, and three of the six are +for this family: + +| Instrument | View helper | Boundaries | +| --------------------------- | ------------------------------- | ------------------------------------------------------ | +| `getobject_lookup_us` | `addMicrosecondHistogramView()` | The shared µs ladder, 100 µs to 60 s (16 buckets) | +| `getobject_request_objects` | `addHistogramView()`, own set | `1, 2, 4, 8, 16, 64, 256, 1024, 4096, 12288` | +| `getobject_charge` | `addHistogramView()`, own set | `0, 100, 500, 1000, 5000, 10000, 25000, 50000, 100000` | + +The other three views are `addMicrosecondHistogramView()` on `job_queued_us`, +`job_running_us`, and `rpc_method_us` — four µs-ladder views plus these two +custom sets. + +**Why the latter two do not use the µs ladder.** They are not durations. The µs +ladder's buckets are chosen for time (sub-millisecond jobs through multi-second +stalls), so applying it to a count or a charge would place almost every +observation in one or two buckets and make the distribution unreadable. +`addHistogramView()` exists to take caller-supplied boundaries for exactly this +case. + +- **Counts** run 1 to `Tuning::kHardMaxReplyNodes` (12288). The low end is + fine-grained because the honest sync path asks for at most 8 objects, so the + interesting distinction is between a normal request and a large one. The upper + bounds follow the charge size bands — `kBandSmallMax` (64) and + `kBandMediumMax` (1024) — up to the hard cap, so a bucket boundary coincides + with each price change. +- **Charges** run 0 to roughly 99k for a full-size all-miss request. Two of the + boundaries are the resource thresholds that decide a peer's fate: + `Resource::kWarningThreshold` (5000) and `Resource::kDropThreshold` (25000), + both in `include/xrpl/resource/detail/Tuning.h`. Placing bucket edges exactly + there lets a panel read off how close real charges run to a warning or a drop, + rather than interpolating across an edge. + +`getobject_lookup_us` is named by a shared constant rather than a literal because +it is referenced from both the record site and the view registration; a drifted +spelling would silently drop the override. + +> **A zero `getobject_rejected_total` does not prove the counter works.** Both +> gates it counts (`reason="oversize"`, `reason="malformed_ledgerhash"`) fire +> only on non-conforming requests, so on a healthy network the expected reading +> is zero. Validate it with a deliberately malformed request, not by looking for +> a series. #### Counted Object Instances (Observable Gauge — `object_count`) @@ -1177,11 +1391,28 @@ rate(rpc_method_errored_total{method="server_info"}[5m]) # Job queue wait time p95 histogram_quantile(0.95, sum by (le) (rate(job_queued_us_bucket[5m]))) +# Job run time p95 split by producer, for one job type +histogram_quantile(0.95, sum by (le, handler) (rate(job_running_us_bucket{job_type="ledgerRequest"}[5m]))) + # TxQ utilization percentage txq_metrics{metric="txq_count"} / txq_metrics{metric="txq_max_size"} # High load factor alert candidate load_factor_metrics{metric="load_factor"} > 5 + +# Job types currently hitting their concurrency limit (backpressure). +# Scoped to one node: unscoped, this aggregates every node on the stack. +max by (__name__) ({__name__=~"jobq_.*_deferred", service_instance_id="$node"}) > 0 + +# GetObject NodeStore hit ratio +sum(rate(getobject_lookups_total{result="hit"}[5m])) + / sum(rate(getobject_lookups_total[5m])) + +# GetObject fetch-loop p95 (microseconds) +histogram_quantile(0.95, sum by (le) (rate(getobject_lookup_us_bucket[5m]))) + +# GetObject requests refused, by reason +sum by (reason) (rate(getobject_rejected_total[5m])) ``` ### Phase 7+: External Dashboard Parity Metrics @@ -1332,12 +1563,13 @@ counters), observed from an existing cumulative source each collection cycle: ## 6. Known Issues -| Issue | Impact | Status | -| ------------------------------------------------------------------ | ------------------------------------------------ | -------------------------------------------------------------------- | -| `warn` and `drop` metrics use non-standard StatsD `\|m` meter type | Metrics silently dropped by OTel StatsD receiver | Phase 6 Task 6.1 — needs `\|m` → `\|c` change in StatsDCollector.cpp | -| `jobq_job_count` may not emit in standalone mode | Missing from Prometheus in some test configs | Requires active job queue activity | -| `rpc_requests` depends on `[insight]` config | Zero series if StatsD not configured | Requires `[insight] server=statsd` in xrpld.cfg | -| Peer tracing enabled by default | `peer.*` spans emit unless `trace_peer=0` | High volume — set `trace_peer=0` to opt out on busy mainnet nodes | +| Issue | Impact | Status | +| ------------------------------------------------------------------ | ------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------- | +| `warn` and `drop` metrics use non-standard StatsD `\|m` meter type | Metrics silently dropped by OTel StatsD receiver | Phase 6 Task 6.1 — needs `\|m` → `\|c` change in StatsDCollector.cpp | +| `jobq_job_count` may not emit in standalone mode | Missing from Prometheus in some test configs | Requires active job queue activity | +| `rpc_requests` depends on `[insight]` config | Zero series if StatsD not configured | Requires `[insight] server=statsd` in xrpld.cfg | +| Peer tracing enabled by default | `peer.*` spans emit unless `trace_peer=0` | High volume — set `trace_peer=0` to opt out on busy mainnet nodes | +| `handler="other"` mixes several producers | Cannot separate `GetConsL1` from `GetConsL2` | By design — the cardinality bound; see [§Per-Job-Type Metrics](#per-job-type-metrics-synchronous-countershistogram) | --- @@ -1463,8 +1695,7 @@ no panel (it is read in Tempo instead). | `sync_acquire_no_progress_total` | counter | `InboundLedger.cpp` — `InboundLedger::onTimer` | Acquire Stall Rate (no progress) | Acquire timeouts where not one new node arrived since the previous timeout, from the `progress_` flag that was previously log-only. Fires on the 3 s acquire timer, never per node. A sustained rate together with a flat missing-node count is the definitive "stuck, not slow" signature. | | `sync_addnode_total` (`outcome` = `good` \| `duplicate` \| `invalid`) | counter | `InboundLedger.cpp` — `InboundLedger::recordBatchOutcome` | Add-Node Outcomes | SHAMap nodes received during acquire, split by result. Emitted once per received packet from the aggregated batch tally the trace log already printed — never inside the per-node `receiveNode()` loop. Separates real progress (`good`) from wasted bandwidth (`duplicate`) and a misbehaving peer (`invalid`), all three of which look like healthy throughput in traffic metrics. | | `sync_acquire_source_total` (`source` = `local` \| `network`) | counter | `InboundLedger.cpp` — `InboundLedger::init` | Acquire Source (local vs network) | Whether an acquire was satisfied entirely from the local node store or needed peers, emitted once per new acquire after the first local lookup. Sustained `network` on a node that should already hold the range means sync is disk-bound rather than peer-bound. | -| `jobq_backlog` (`metric` = `waiting` \| `running` \| `deferred`; `job_type` = the `JobTypes::name()` string) | observable gauge | `MetricsRegistry.cpp` — `registerJobQueueBacklogGauge` | Deferred Jobs by Type (starvation); Job Queue Occupancy by Type (waiting/running) | Instantaneous per-job-type queue occupancy, from `JobQueue::getJobTypeCounts()` (one mutex acquire per ~10 s tick). **`deferred` is the signal this adds:** jobs the queue accepted but withheld because the type is already at its concurrency limit, which is counted in neither `waiting` nor `running` and had no exposure anywhere before. The sync-critical types are capped at 3 (`JtLedgerReq`, `JtLedgerData` in `JobTypes.h`), so they starve first. Distinct from the existing `job_queued_total` / `job_started_total` / `job_finished_total` counters and `job_queued_us` / `job_running_us` histograms, which are event-driven from PerfLogImp and describe jobs that already moved, and from the StatsD `jobq_job_count`, which is queue-wide with no per-type split. Cardinality is bounded by the JobType enum (~46 values); every type is observed every tick, so an idle type reports 0 rather than dropping its series. | -| `jobq_saturation` (`metric` = `running_tasks` \| `worker_threads` \| `total_waiting`) | observable gauge | `MetricsRegistry.cpp` — `registerJobQueueSaturationGauge` | Worker Pool Saturation; Worker Pool Capacity & Total Backlog | Global worker-pool saturation from `JobQueue::getWorkerSaturation()`: tasks in flight, threads the pool is configured to run, and jobs queued across all types, all from one reading so the ratio and the backlog describe the same instant. `worker_threads` is exported rather than hardcoded in the dashboard because it is derived at startup from `[workers]`, node size and hardware concurrency. Exists separately from `jobq_backlog` because a pool-wide slowdown otherwise appears as an independent fault in every subsystem queued behind it; a `running_tasks / worker_threads` ratio at 1.0 **with** a non-zero `total_waiting` attributes it to pool exhaustion once. | +| `jobq_saturation` (`metric` = `running_tasks` \| `worker_threads` \| `total_waiting`) | observable gauge | `MetricsRegistry.cpp` — `registerJobQueueSaturationGauge` | Worker Pool Saturation; Worker Pool Capacity & Total Backlog | Global worker-pool saturation from `JobQueue::getWorkerSaturation()`: tasks in flight, threads the pool is configured to run, and jobs queued across all types, all from one reading so the ratio and the backlog describe the same instant. `worker_threads` is exported rather than hardcoded in the dashboard because it is derived at startup from `[workers]`, node size and hardware concurrency. Exists separately from the per-job-type gauges `JobQueue::collect()` publishes (`jobq__waiting` / `_running` / `_deferred`) because a pool-wide slowdown otherwise appears as an independent fault in every subsystem queued behind it; a `running_tasks / worker_threads` ratio at 1.0 **with** a non-zero `total_waiting` attributes it to pool exhaustion once. | | `peer_ledger_supply` (`metric` = `peers_reporting` \| `peers_serving_validated` \| `peers_serving_next` \| `supply_min_seq` \| `supply_max_seq`) | observable gauge | `MetricsRegistry.cpp` — `registerPeerLedgerSupplyGauge` (aggregating `OverlayImpl::getPeerLedgerSupply`) | Peers Able to Serve Needed Sequence; Peer Ledger Supply Window | How much of the sequence range this node needs its connected peer set can actually serve, from one pass over the active peers reading the range each already advertised in `mtSTATUS_CHANGE`. **`peers_serving_next` is the signal this exists for:** zero there with a non-zero `peers_reporting` means no connected peer holds validated + 1, so the peer set must change and waiting cannot finish the sync. `peers_reporting` is the denominator that makes the rest readable — peers advertising `[0, 0]` have not reported yet and are excluded from every field, so they cannot make a healthy peer set appear to serve from genesis; when nothing has reported, both window fields read 0 meaning **unknown**, not genesis. `supply_min_seq` / `supply_max_seq` separate "asking for history nobody kept" from "asking for a tip nobody reached". Distinct from `server_info{metric="peers"}`, a bare connection count with no notion of what those peers hold; from `sync_state{metric="ledgers_behind"}`, which uses the same per-peer maxima but collapses them to a single distance-to-tip number that cannot say how many peers can serve that distance or whether the range has a hole; and from `peer_quality{metric="peers_insane_count"}`, which counts peers on a different chain and is therefore a correctness signal, not an availability one. | | `peer_disconnect_total` (`reason` = `graceful` \| `shutdown` \| `stopping` \| `read_error` \| `write_error` \| `timer_error` \| `ping_timeout` \| `not_useful` \| `large_sendq` \| `charge_resources` \| `malformed_handshake` \| `shared_value` \| `unknown`; `direction` = `inbound` \| `outbound`) | counter | `PeerImp.cpp` — `PeerImp::close` | Peer Disconnects by Reason | Peer teardowns split by cause and by which side opened the connection. Emitted once per teardown at `close()`, the single funnel every disconnect path passes through, and `close()` already self-guards on the socket being open, so a repeated close cannot double-count and the total matches the existing unlabelled tally. `reason` is set by whichever site decided to disconnect, first writer wins, so a later generic reason never masks the real one; the value is always one of a fixed set of literals in `PeerImp.cpp`, never peer-supplied data, so cardinality is bounded by the code. The split is the whole point: it separates our-fault backpressure (`large_sendq`, `charge_resources`) from topology and network faults (`not_useful`, `ping_timeout`, `read_error`), and normal churn (`graceful`) from either. Distinct from the existing `server_info{metric="peer_disconnects_resources"}`, which counts only the resource-charge subset and carries no labels, and from the StatsD `overlay_peer_disconnects`, which is the unlabelled grand total in which every reason above collapses into one number. | | `peer_accept_total` (`outcome` = `accepted` \| `local_endpoint_fail` \| `resource_limit` \| `no_slot` \| `not_peer_request` \| `protocol_mismatch` \| `bad_cookie` \| `slot_refused` \| `handshake_error`) | counter | `OverlayImpl.cpp` — `OverlayImpl::onHandoff` via `reportAcceptOutcome` | Inbound Peer Accept Outcomes | Terminal outcome of every inbound connection this node is offered, one emit per handoff. `accepted` is reported only after `run()`, so anything that threw on the way lands on `handshake_error` instead; the two early returns that are not peer attempts at all (a handled HTTP request, and a request that never asked to upgrade) are deliberately not counted. The `outcome` names the stage that refused: no local endpoint, the resource manager, PeerFinder having no slot or seeing a duplicate, a non-peer upgrade request, protocol version disagreement, a bad security cookie, or activation being refused. This is the **inbound twin** of the existing `overlay_connect_total{outcome}`, which covers outbound dials only; without it a node refusing every inbound connection is indistinguishable from one nobody dials, and reading the two together gives the full in/out split. | diff --git a/docker/telemetry/grafana/dashboards/job-queue.json b/docker/telemetry/grafana/dashboards/job-queue.json index d53697d871..4807959e31 100644 --- a/docker/telemetry/grafana/dashboards/job-queue.json +++ b/docker/telemetry/grafana/dashboards/job-queue.json @@ -1,38 +1,8 @@ { "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - }, - { - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", - "target": { - "limit": 100, - "matchAny": false, - "tags": ["perf-iac"], - "type": "tags" - }, - "type": "tags" - } - ] + "list": [] }, - "description": "What this shows: Per-job-type throughput, queue wait times, and execution times for the node's internal worker job queue. \u2014 Use it to: Find job types that are backing up or running slowly and causing internal processing delays.", + "description": "What this shows: Per-job-type throughput, queue wait times, and execution times for the node's internal worker job queue.\nUse it to: Find job types that are backing up or running slowly and causing internal processing delays.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -62,23 +32,21 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (service_instance_id, le, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[5m]))), \"series\", \"p99 Wait\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(histogram_quantile(0.99, sum by (le, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Wait\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (service_instance_id, le, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[5m]))), \"series\", \"p99 Exec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(histogram_quantile(0.99, sum by (le, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Exec\", \"\", \"\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "\u00b5s", + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "µs", "min": 0, "thresholds": { "mode": "absolute", @@ -122,29 +90,26 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[$__rate_interval])), \"series\", \"Queued/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[$__rate_interval])), \"series\", \"Queued/s\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_started_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[$__rate_interval])), \"series\", \"Started/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_started_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[$__rate_interval])), \"series\", \"Started/s\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_finished_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[$__rate_interval])), \"series\", \"Finished/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_finished_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[$__rate_interval])), \"series\", \"Finished/s\", \"\", \"\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", "unit": "ops", "custom": { "drawStyle": "line", @@ -170,7 +135,7 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 12, + "w": 24, "x": 0, "y": 16 }, @@ -189,15 +154,14 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(topk(10, rate(job_queued_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[$__rate_interval])), \"series\", \"$1\", \"job_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(topk(10, sum by (job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[$__rate_interval]))), \"series\", \"$1\", \"job_type\", \"(.*)\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", "unit": "ops", "custom": { "drawStyle": "line", @@ -223,9 +187,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 16 + "w": 24, + "x": 0, + "y": 24 }, "options": { "tooltip": { @@ -242,15 +206,14 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(topk(10, rate(job_finished_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[$__rate_interval])), \"series\", \"$1\", \"job_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(topk(10, sum by (job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_finished_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[$__rate_interval]))), \"series\", \"$1\", \"job_type\", \"(.*)\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", "unit": "ops", "custom": { "drawStyle": "line", @@ -272,13 +235,13 @@ }, { "title": "Job Queue Wait Time", - "description": "###### What this is:\n*Distribution of how long jobs sit in the queue before a worker picks them up (p75 typical, p99 tail).*\n\n###### How it's computed:\n*Percentiles derived from the job wait-time histogram over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; a widening gap between p75 and p99 signals occasional stalls.*\n\n###### Healthy range:\n*Sub-millisecond to low-millisecond waits on an unloaded node.*\n\n###### Watch for:\n*Rising p99 wait, an early sign of worker-thread saturation.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobStarted`", + "description": "###### What this is:\n*How long jobs sit in the queue before a worker picks them up, split by job type (p75 typical, p99 tail).*\n\n###### How it's computed:\n*Percentiles derived from the job wait-time histogram over a 5-minute window, kept per job type. Limited to the ten types with the highest wait so the legend stays readable.*\n\n###### Reading it:\n*Lower is better. The legend names the job type, so a single slow queue is identifiable rather than hidden in an all-types average. A widening p75-to-p99 gap on one type signals occasional stalls there.*\n\n###### Healthy range:\n*Sub-millisecond to low-millisecond waits on an unloaded node.*\n\n###### Watch for:\n*Rising p99 wait on a capped type -- ledgerRequest, ledgerData and makeFetchPack have small concurrency limits, so they queue first. Cross-check the deferred gauge for that type.*\n\n###### Keywords:\n- **Job queue** *(per node)* — the worker pool that runs xrpld's background work; each unit of work is a job with a type.\n- **Dequeue wait** *(per node)* — time a job sits enqueued before a worker starts it, as distinct from how long it then runs.\n- **Concurrency limit** *(per node)* — the maximum number of jobs of one type allowed to run at once; work beyond it is deferred, not rejected.\n\n###### Computation boundary:\n*Result: Per node and job type — each series is one server's own value for one job type.*\n*Recorded in code as an OTel SDK histogram, then aggregated to percentiles by the Grafana query.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobStarted`\n\n###### References:\n[Job queue](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type) · [Concurrency limit](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#concurrency-limit)\n", "type": "timeseries", "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 32 }, "options": { "tooltip": { @@ -290,28 +253,26 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.75, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[5m]))), \"series\", \"p75 Wait\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(topk(10, histogram_quantile(0.75, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m])))), \"series\", \"p75 Wait $1\", \"job_type\", \"(.*)\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[5m]))), \"series\", \"p99 Wait\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(topk(10, histogram_quantile(0.99, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m])))), \"series\", \"p99 Wait $1\", \"job_type\", \"(.*)\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "\u00b5s", + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "µs", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 5, - "axisLabel": "Duration (\u03bcs)", + "axisLabel": "Duration (μs)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -327,13 +288,13 @@ }, { "title": "Job Execution Time", - "description": "###### What this is:\n*Distribution of how long jobs run once started (p75 typical, p99 tail).*\n\n###### How it's computed:\n*Percentiles derived from the job run-time histogram over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; stable p75 with a controlled p99 is healthy.*\n\n###### Healthy range:\n*Workload-dependent, but stable over time.*\n\n###### Watch for:\n*Growing execution times, which point to CPU pressure or expensive individual jobs.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobFinished`", + "description": "###### What this is:\n*How long jobs run once started, split by job type (p75 typical, p99 tail).*\n\n###### How it's computed:\n*Percentiles derived from the job run-time histogram over a 5-minute window, kept per job type. Limited to the ten slowest types so the legend stays readable.*\n\n###### Reading it:\n*Lower is better. The legend names the job type, so an expensive job type is identifiable rather than averaged away. Stable p75 with a controlled p99 is healthy.*\n\n###### Healthy range:\n*Workload-dependent, but stable over time.*\n\n###### Watch for:\n*Growing execution times, which point to CPU pressure or expensive individual jobs.*\n\n###### Keywords:\n- **Job queue** *(per node)* — the worker pool that runs xrpld's background work; each unit of work is a job with a type.\n- **Execution time** *(per node)* — time a job spends running after a worker picks it up, excluding its queue wait.\n\n###### Computation boundary:\n*Result: Per node and job type — each series is one server's own value for one job type.*\n*Recorded in code as an OTel SDK histogram, then aggregated to percentiles by the Grafana query.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobFinished`\n\n###### References:\n[Job queue](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type)\n", "type": "timeseries", "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 32 }, "options": { "tooltip": { @@ -345,28 +306,26 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.75, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[5m]))), \"series\", \"p75 Exec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(topk(10, histogram_quantile(0.75, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m])))), \"series\", \"p75 Exec $1\", \"job_type\", \"(.*)\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[5m]))), \"series\", \"p99 Exec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(topk(10, histogram_quantile(0.99, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m])))), \"series\", \"p99 Exec $1\", \"job_type\", \"(.*)\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "\u00b5s", + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "µs", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 5, - "axisLabel": "Duration (\u03bcs)", + "axisLabel": "Duration (μs)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -388,7 +347,7 @@ "h": 8, "w": 24, "x": 0, - "y": 32 + "y": 40 }, "options": { "tooltip": { @@ -405,21 +364,20 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(topk(10, histogram_quantile(0.99, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[5m])))), \"series\", \"$1\", \"job_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(topk(10, histogram_quantile(0.99, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m])))), \"series\", \"$1\", \"job_type\", \"(.*)\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "\u00b5s", + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "µs", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 5, - "axisLabel": "Duration (\u03bcs)", + "axisLabel": "Duration (μs)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -435,13 +393,13 @@ }, { "title": "Transaction Overflow Rate", - "description": "###### What this is:\n*Rate at which transaction jobs are shed when the queue's transaction limit is exceeded.*\n\n###### How it's computed:\n*Per-second rate of the overflow counter over the dashboard's rate interval, scaled to per minute. The counter is observed from the overlay's cumulative overflow tally.*\n\n###### Reading it:\n*Near zero is healthy; a rising rate means the job queue is shedding transaction work under load.*\n\n###### Healthy range:\n*0 overflows per minute.*\n\n###### Watch for:\n*Any sustained non-zero rate \u2014 the node is dropping transaction jobs because the queue is saturated.*\n\n###### Keywords:\n- **Overlay** *(per node)* \u2014 the peer-to-peer network layer over which nodes exchange transactions, proposals, and validations.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerParityCounters (observed from Overlay::getJqTransOverflow)`\n\n###### References:\n[Overlay](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay)", + "description": "###### What this is:\n*Rate at which transaction jobs are shed when the queue's transaction limit is exceeded.*\n\n###### How it's computed:\n*Per-second rate of the overflow counter over the dashboard's rate interval, scaled to per minute. The counter is observed from the overlay's cumulative overflow tally.*\n\n###### Reading it:\n*Near zero is healthy; a rising rate means the job queue is shedding transaction work under load.*\n\n###### Healthy range:\n*0 overflows per minute.*\n\n###### Watch for:\n*Any sustained non-zero rate — the node is dropping transaction jobs because the queue is saturated.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerParityCounters (observed from Overlay::getJqTransOverflow)`", "type": "timeseries", "gridPos": { "h": 8, "w": 24, "x": 0, - "y": 40 + "y": 48 }, "options": { "tooltip": { @@ -453,15 +411,14 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(jq_trans_overflow_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) * 60, \"series\", \"Overflows/min\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(jq_trans_overflow_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) * 60, \"series\", \"Overflows/min\", \"\", \"\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", "unit": "cpm", "thresholds": { "steps": [ @@ -502,19 +459,6 @@ "tags": ["node", "jobqueue"], "templating": { "list": [ - { - "name": "DS_PROMETHEUS", - "type": "datasource", - "label": "Prometheus", - "query": "prometheus", - "regex": "", - "current": {}, - "hide": 0, - "refresh": 1, - "includeAll": false, - "multi": false, - "options": [] - }, { "name": "service_name", "label": "Service Name", @@ -523,7 +467,7 @@ "query": "label_values(service_name)", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "prometheus" }, "includeAll": true, "allValue": ".*", @@ -543,7 +487,7 @@ "query": "label_values(deployment_environment)", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "prometheus" }, "includeAll": true, "allValue": ".*", @@ -563,7 +507,7 @@ "query": "label_values(xrpl_network_type)", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "prometheus" }, "includeAll": true, "allValue": ".*", @@ -643,7 +587,7 @@ "query": "label_values(target_info, service_instance_id)", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "prometheus" }, "includeAll": true, "allValue": ".*", @@ -663,7 +607,27 @@ "query": "label_values(job_queued_total, job_type)", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "prometheus" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "handler", + "label": "Handler", + "description": "Filter by enqueueing handler (sanitised addJob name, e.g. RcvGetLedger)", + "type": "query", + "query": "label_values(job_queued_total, handler)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" }, "includeAll": true, "allValue": ".*", @@ -686,5 +650,5 @@ "title": "Job Queue Analysis", "uid": "job-queue", "version": 1, - "refresh": "30s" + "refresh": "5s" } diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index d95161dfb6..c12a4378bb 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -1,38 +1,8 @@ { "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - }, - { - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", - "target": { - "limit": 100, - "matchAny": false, - "tags": ["perf-iac"], - "type": "tags" - }, - "type": "tags" - } - ] + "list": [] }, - "description": "What this shows: Ledger data exchange and object-fetch traffic between this node and its peers: ledger sync, tree-node retrieval, and transaction-set exchange. \u2014 Use it to: See how much ledger data the node is pulling or serving and spot catch-up activity.", + "description": "What this shows: Ledger data exchange and object-fetch traffic between this node and its peers: ledger sync, tree-node retrieval, and transaction-set exchange.\nUse it to: See how much ledger data the node is pulling or serving and spot catch-up activity.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -40,8 +10,8 @@ "links": [], "panels": [ { - "title": "Ledger Data \u2014 Ledger", - "description": "###### What this is:\n*Inbound bytes for ledger-data message categories, split into aggregate get/share plus the transaction-set, transaction-node, and account-state-node sub-types the node receives from peers.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Normally low and flat once synced. Account-state-node traffic dominates during state sync; transaction-set-candidate traffic dominates during consensus catch-up.*\n\n###### Healthy range:\n*workload-dependent; low and steady on a synced node.*\n\n###### Watch for:\n*Sustained high account-state or tx-node inbound bytes on a node that should be caught up (repeated re-sync, missing history), or a single peer driving all traffic.*\n\n###### Keywords:\n- **Ledger tree nodes** *(network-wide)* \u2014 the internal SHAMap nodes that make up a ledger's transaction tree and account-state tree.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n- **Back-fill / catch-up** *(per node)* \u2014 fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-tree-nodes)", + "title": "Ledger Data — Ledger", + "description": "###### What this is:\n*Inbound bytes for ledger-data message categories, split into aggregate get/share plus the transaction-set, transaction-node, and account-state-node sub-types the node receives from peers.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Normally low and flat once synced. Account-state-node traffic dominates during state sync; transaction-set-candidate traffic dominates during consensus catch-up.*\n\n###### Healthy range:\n*workload-dependent; low and steady on a synced node.*\n\n###### Watch for:\n*Sustained high account-state or tx-node inbound bytes on a node that should be caught up (repeated re-sync, missing history), or a single peer driving all traffic.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", "type": "timeseries", "gridPos": { "h": 8, @@ -59,22 +29,20 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(ledger_data_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Data Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(ledger_data_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Data Get\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(ledger_data_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Data Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(ledger_data_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Data Share\", \"\", \"\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", "unit": "Bps", "custom": { "axisLabel": "Bytes In", @@ -85,11 +53,12 @@ } }, "overrides": [] - } + }, + "id": 1 }, { - "title": "Ledger Data \u2014 Transaction", - "description": "###### What this is:\n*Inbound bytes for ledger-data message categories, split into aggregate get/share plus the transaction-set, transaction-node, and account-state-node sub-types the node receives from peers.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Normally low and flat once synced. Account-state-node traffic dominates during state sync; transaction-set-candidate traffic dominates during consensus catch-up.*\n\n###### Healthy range:\n*workload-dependent; low and steady on a synced node.*\n\n###### Watch for:\n*Sustained high account-state or tx-node inbound bytes on a node that should be caught up (repeated re-sync, missing history), or a single peer driving all traffic.*\n\n###### Keywords:\n- **Ledger tree nodes** *(network-wide)* \u2014 the internal SHAMap nodes that make up a ledger's transaction tree and account-state tree.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n- **Back-fill / catch-up** *(per node)* \u2014 fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-tree-nodes)", + "title": "Ledger Data — Transaction", + "description": "###### What this is:\n*Inbound bytes for ledger-data message categories, split into aggregate get/share plus the transaction-set, transaction-node, and account-state-node sub-types the node receives from peers.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Normally low and flat once synced. Account-state-node traffic dominates during state sync; transaction-set-candidate traffic dominates during consensus catch-up.*\n\n###### Healthy range:\n*workload-dependent; low and steady on a synced node.*\n\n###### Watch for:\n*Sustained high account-state or tx-node inbound bytes on a node that should be caught up (repeated re-sync, missing history), or a single peer driving all traffic.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", "type": "timeseries", "gridPos": { "h": 8, @@ -107,36 +76,32 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(ledger_data_transaction_set_candidate_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Set Candidate Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(ledger_data_transaction_set_candidate_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Set Candidate Get\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(ledger_data_transaction_set_candidate_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Set Candidate Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(ledger_data_transaction_set_candidate_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Set Candidate Share\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(ledger_data_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(ledger_data_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Get\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(ledger_data_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(ledger_data_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Share\", \"\", \"\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", "unit": "Bps", "custom": { "axisLabel": "Bytes In", @@ -147,11 +112,12 @@ } }, "overrides": [] - } + }, + "id": 2 }, { - "title": "Ledger Data \u2014 Account State", - "description": "###### What this is:\n*Inbound bytes for ledger-data message categories, split into aggregate get/share plus the transaction-set, transaction-node, and account-state-node sub-types the node receives from peers.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Normally low and flat once synced. Account-state-node traffic dominates during state sync; transaction-set-candidate traffic dominates during consensus catch-up.*\n\n###### Healthy range:\n*workload-dependent; low and steady on a synced node.*\n\n###### Watch for:\n*Sustained high account-state or tx-node inbound bytes on a node that should be caught up (repeated re-sync, missing history), or a single peer driving all traffic.*\n\n###### Keywords:\n- **Ledger tree nodes** *(network-wide)* \u2014 the internal SHAMap nodes that make up a ledger's transaction tree and account-state tree.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n- **Back-fill / catch-up** *(per node)* \u2014 fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-tree-nodes)", + "title": "Ledger Data — Account State", + "description": "###### What this is:\n*Inbound bytes for ledger-data message categories, split into aggregate get/share plus the transaction-set, transaction-node, and account-state-node sub-types the node receives from peers.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Normally low and flat once synced. Account-state-node traffic dominates during state sync; transaction-set-candidate traffic dominates during consensus catch-up.*\n\n###### Healthy range:\n*workload-dependent; low and steady on a synced node.*\n\n###### Watch for:\n*Sustained high account-state or tx-node inbound bytes on a node that should be caught up (repeated re-sync, missing history), or a single peer driving all traffic.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", "type": "timeseries", "gridPos": { "h": 8, @@ -169,22 +135,20 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(ledger_data_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Node Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(ledger_data_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Node Get\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(ledger_data_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Node Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(ledger_data_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Node Share\", \"\", \"\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", "unit": "Bps", "custom": { "axisLabel": "Bytes In", @@ -195,11 +159,12 @@ } }, "overrides": [] - } + }, + "id": 3 }, { - "title": "Ledger Traffic \u2014 Ledger", - "description": "###### What this is:\n*Inbound bytes for the older ledger share/get message categories and their tx-set, tx-node, and account-state sub-types.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Usually small; these legacy categories carry ledger-fetch traffic for peers using the older protocol.*\n\n###### Healthy range:\n*workload-dependent; low on a synced node.*\n\n###### Watch for:\n*Large sustained volumes indicating heavy fetch load or a peer repeatedly requesting the same data.*\n\n###### Keywords:\n- **Ledger tree nodes** *(network-wide)* \u2014 the internal SHAMap nodes that make up a ledger's transaction tree and account-state tree.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-tree-nodes)", + "title": "Ledger Traffic — Ledger", + "description": "###### What this is:\n*Inbound bytes for the older ledger share/get message categories and their tx-set, tx-node, and account-state sub-types.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Usually small; these legacy categories carry ledger-fetch traffic for peers using the older protocol.*\n\n###### Healthy range:\n*workload-dependent; low on a synced node.*\n\n###### Watch for:\n*Large sustained volumes indicating heavy fetch load or a peer repeatedly requesting the same data.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", "type": "timeseries", "gridPos": { "h": 8, @@ -217,22 +182,20 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(ledger_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Get In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(ledger_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Get In\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(ledger_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Share In\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(ledger_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Share In\", \"\", \"\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", "unit": "Bps", "custom": { "axisLabel": "Bytes In", @@ -243,11 +206,12 @@ } }, "overrides": [] - } + }, + "id": 4 }, { - "title": "Ledger Traffic \u2014 Transaction", - "description": "###### What this is:\n*Inbound bytes for the older ledger share/get message categories and their tx-set, tx-node, and account-state sub-types.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Usually small; these legacy categories carry ledger-fetch traffic for peers using the older protocol.*\n\n###### Healthy range:\n*workload-dependent; low on a synced node.*\n\n###### Watch for:\n*Large sustained volumes indicating heavy fetch load or a peer repeatedly requesting the same data.*\n\n###### Keywords:\n- **Ledger tree nodes** *(network-wide)* \u2014 the internal SHAMap nodes that make up a ledger's transaction tree and account-state tree.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-tree-nodes)", + "title": "Ledger Traffic — Transaction", + "description": "###### What this is:\n*Inbound bytes for the older ledger share/get message categories and their tx-set, tx-node, and account-state sub-types.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Usually small; these legacy categories carry ledger-fetch traffic for peers using the older protocol.*\n\n###### Healthy range:\n*workload-dependent; low on a synced node.*\n\n###### Watch for:\n*Large sustained volumes indicating heavy fetch load or a peer repeatedly requesting the same data.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", "type": "timeseries", "gridPos": { "h": 8, @@ -265,36 +229,32 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(ledger_transaction_set_candidate_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Set Candidate Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(ledger_transaction_set_candidate_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Set Candidate Get\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(ledger_transaction_set_candidate_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Set Candidate Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(ledger_transaction_set_candidate_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Set Candidate Share\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(ledger_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(ledger_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Get\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(ledger_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(ledger_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Share\", \"\", \"\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", "unit": "Bps", "custom": { "axisLabel": "Bytes In", @@ -305,11 +265,12 @@ } }, "overrides": [] - } + }, + "id": 5 }, { - "title": "Ledger Traffic \u2014 Account State", - "description": "###### What this is:\n*Inbound bytes for the older ledger share/get message categories and their tx-set, tx-node, and account-state sub-types.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Usually small; these legacy categories carry ledger-fetch traffic for peers using the older protocol.*\n\n###### Healthy range:\n*workload-dependent; low on a synced node.*\n\n###### Watch for:\n*Large sustained volumes indicating heavy fetch load or a peer repeatedly requesting the same data.*\n\n###### Keywords:\n- **Ledger tree nodes** *(network-wide)* \u2014 the internal SHAMap nodes that make up a ledger's transaction tree and account-state tree.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-tree-nodes)", + "title": "Ledger Traffic — Account State", + "description": "###### What this is:\n*Inbound bytes for the older ledger share/get message categories and their tx-set, tx-node, and account-state sub-types.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Usually small; these legacy categories carry ledger-fetch traffic for peers using the older protocol.*\n\n###### Healthy range:\n*workload-dependent; low on a synced node.*\n\n###### Watch for:\n*Large sustained volumes indicating heavy fetch load or a peer repeatedly requesting the same data.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", "type": "timeseries", "gridPos": { "h": 8, @@ -327,22 +288,20 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(ledger_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(ledger_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Get\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(ledger_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(ledger_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Share\", \"\", \"\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", "unit": "Bps", "custom": { "axisLabel": "Bytes In", @@ -353,11 +312,12 @@ } }, "overrides": [] - } + }, + "id": 6 }, { - "title": "GetObject \u2014 Ledger", - "description": "###### What this is:\n*Inbound bytes for object-fetch traffic broken down by object type: ledger headers, individual transactions, transaction-tree nodes, and state-tree nodes.*\n\n###### How it's computed:\n*Per-type inbound byte rate per node.*\n\n###### Reading it:\n*Small during steady state; grows when the node fetches missing tree nodes.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*A large share on state/tx nodes for long periods (persistent gap-filling), meaning the node keeps catching up.*\n\n###### Keywords:\n- **GetObject / object fetch** *(per node)* \u2014 peer requests for specific ledger objects \u2014 headers, transactions, or tree nodes \u2014 by type.\n- **Ledger tree nodes** *(network-wide)* \u2014 the internal SHAMap nodes that make up a ledger's transaction tree and account-state tree.\n- **Back-fill / catch-up** *(per node)* \u2014 fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#getobject-object-fetch)", + "title": "GetObject — Ledger", + "description": "###### What this is:\n*Inbound bytes for object-fetch traffic broken down by object type: ledger headers, individual transactions, transaction-tree nodes, and state-tree nodes.*\n\n###### How it's computed:\n*Per-type inbound byte rate per node.*\n\n###### Reading it:\n*Small during steady state; grows when the node fetches missing tree nodes.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*A large share on state/tx nodes for long periods (persistent gap-filling), meaning the node keeps catching up.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", "type": "timeseries", "gridPos": { "h": 8, @@ -375,22 +335,20 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(getobject_ledger_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(getobject_ledger_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Get\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(getobject_ledger_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(getobject_ledger_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Share\", \"\", \"\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", "unit": "Bps", "custom": { "axisLabel": "Bytes In", @@ -401,11 +359,12 @@ } }, "overrides": [] - } + }, + "id": 7 }, { - "title": "GetObject \u2014 Transaction", - "description": "###### What this is:\n*Inbound bytes for object-fetch traffic broken down by object type: ledger headers, individual transactions, transaction-tree nodes, and state-tree nodes.*\n\n###### How it's computed:\n*Per-type inbound byte rate per node.*\n\n###### Reading it:\n*Small during steady state; grows when the node fetches missing tree nodes.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*A large share on state/tx nodes for long periods (persistent gap-filling), meaning the node keeps catching up.*\n\n###### Keywords:\n- **GetObject / object fetch** *(per node)* \u2014 peer requests for specific ledger objects \u2014 headers, transactions, or tree nodes \u2014 by type.\n- **Ledger tree nodes** *(network-wide)* \u2014 the internal SHAMap nodes that make up a ledger's transaction tree and account-state tree.\n- **Back-fill / catch-up** *(per node)* \u2014 fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#getobject-object-fetch)", + "title": "GetObject — Transaction", + "description": "###### What this is:\n*Inbound bytes for object-fetch traffic broken down by object type: ledger headers, individual transactions, transaction-tree nodes, and state-tree nodes.*\n\n###### How it's computed:\n*Per-type inbound byte rate per node.*\n\n###### Reading it:\n*Small during steady state; grows when the node fetches missing tree nodes.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*A large share on state/tx nodes for long periods (persistent gap-filling), meaning the node keeps catching up.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", "type": "timeseries", "gridPos": { "h": 8, @@ -423,36 +382,32 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(getobject_transaction_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Transaction Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(getobject_transaction_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Transaction Get\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(getobject_transaction_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Transaction Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(getobject_transaction_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Transaction Share\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(getobject_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(getobject_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Get\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(getobject_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(getobject_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Share\", \"\", \"\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", "unit": "Bps", "custom": { "axisLabel": "Bytes In", @@ -463,11 +418,12 @@ } }, "overrides": [] - } + }, + "id": 8 }, { - "title": "GetObject \u2014 Account State", - "description": "###### What this is:\n*Inbound bytes for object-fetch traffic broken down by object type: ledger headers, individual transactions, transaction-tree nodes, and state-tree nodes.*\n\n###### How it's computed:\n*Per-type inbound byte rate per node.*\n\n###### Reading it:\n*Small during steady state; grows when the node fetches missing tree nodes.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*A large share on state/tx nodes for long periods (persistent gap-filling), meaning the node keeps catching up.*\n\n###### Keywords:\n- **GetObject / object fetch** *(per node)* \u2014 peer requests for specific ledger objects \u2014 headers, transactions, or tree nodes \u2014 by type.\n- **Ledger tree nodes** *(network-wide)* \u2014 the internal SHAMap nodes that make up a ledger's transaction tree and account-state tree.\n- **Back-fill / catch-up** *(per node)* \u2014 fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#getobject-object-fetch)", + "title": "GetObject — Account State", + "description": "###### What this is:\n*Inbound bytes for object-fetch traffic broken down by object type: ledger headers, individual transactions, transaction-tree nodes, and state-tree nodes.*\n\n###### How it's computed:\n*Per-type inbound byte rate per node.*\n\n###### Reading it:\n*Small during steady state; grows when the node fetches missing tree nodes.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*A large share on state/tx nodes for long periods (persistent gap-filling), meaning the node keeps catching up.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", "type": "timeseries", "gridPos": { "h": 8, @@ -485,22 +441,20 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(getobject_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(getobject_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Get\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(getobject_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(getobject_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Share\", \"\", \"\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", "unit": "Bps", "custom": { "axisLabel": "Bytes In", @@ -511,11 +465,12 @@ } }, "overrides": [] - } + }, + "id": 9 }, { - "title": "GetObject Messages \u2014 Ledger", - "description": "###### What this is:\n*Count of individual object-fetch request/response messages per object type.*\n\n###### How it's computed:\n*Per-type inbound message rate per node.*\n\n###### Reading it:\n*Many messages with few bytes means small piecemeal fetches; few messages with many bytes means large batch transfers.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High message counts with tiny payloads sustained over time (inefficient per-node fetching).*\n\n###### Keywords:\n- **GetObject / object fetch** *(per node)* \u2014 peer requests for specific ledger objects \u2014 headers, transactions, or tree nodes \u2014 by type.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#getobject-object-fetch)", + "title": "GetObject Messages — Ledger", + "description": "###### What this is:\n*Count of individual object-fetch request/response messages per object type.*\n\n###### How it's computed:\n*Per-type inbound message rate per node.*\n\n###### Reading it:\n*Many messages with few bytes means small piecemeal fetches; few messages with many bytes means large batch transfers.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High message counts with tiny payloads sustained over time (inefficient per-node fetching).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", "type": "timeseries", "gridPos": { "h": 8, @@ -533,15 +488,14 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(getobject_ledger_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(getobject_ledger_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Ledger Get\", \"\", \"\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", "unit": "cps", "custom": { "axisLabel": "Messages In", @@ -552,11 +506,12 @@ } }, "overrides": [] - } + }, + "id": 10 }, { - "title": "GetObject Messages \u2014 Transaction", - "description": "###### What this is:\n*Count of individual object-fetch request/response messages per object type.*\n\n###### How it's computed:\n*Per-type inbound message rate per node.*\n\n###### Reading it:\n*Many messages with few bytes means small piecemeal fetches; few messages with many bytes means large batch transfers.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High message counts with tiny payloads sustained over time (inefficient per-node fetching).*\n\n###### Keywords:\n- **GetObject / object fetch** *(per node)* \u2014 peer requests for specific ledger objects \u2014 headers, transactions, or tree nodes \u2014 by type.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#getobject-object-fetch)", + "title": "GetObject Messages — Transaction", + "description": "###### What this is:\n*Count of individual object-fetch request/response messages per object type.*\n\n###### How it's computed:\n*Per-type inbound message rate per node.*\n\n###### Reading it:\n*Many messages with few bytes means small piecemeal fetches; few messages with many bytes means large batch transfers.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High message counts with tiny payloads sustained over time (inefficient per-node fetching).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", "type": "timeseries", "gridPos": { "h": 8, @@ -574,22 +529,20 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(getobject_transaction_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Transaction Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(getobject_transaction_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Transaction Get\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(getobject_transaction_node_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(getobject_transaction_node_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"TX Node Get\", \"\", \"\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", "unit": "cps", "custom": { "axisLabel": "Messages In", @@ -600,11 +553,12 @@ } }, "overrides": [] - } + }, + "id": 11 }, { - "title": "GetObject Messages \u2014 Account State", - "description": "###### What this is:\n*Count of individual object-fetch request/response messages per object type.*\n\n###### How it's computed:\n*Per-type inbound message rate per node.*\n\n###### Reading it:\n*Many messages with few bytes means small piecemeal fetches; few messages with many bytes means large batch transfers.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High message counts with tiny payloads sustained over time (inefficient per-node fetching).*\n\n###### Keywords:\n- **GetObject / object fetch** *(per node)* \u2014 peer requests for specific ledger objects \u2014 headers, transactions, or tree nodes \u2014 by type.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#getobject-object-fetch)", + "title": "GetObject Messages — Account State", + "description": "###### What this is:\n*Count of individual object-fetch request/response messages per object type.*\n\n###### How it's computed:\n*Per-type inbound message rate per node.*\n\n###### Reading it:\n*Many messages with few bytes means small piecemeal fetches; few messages with many bytes means large batch transfers.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High message counts with tiny payloads sustained over time (inefficient per-node fetching).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", "type": "timeseries", "gridPos": { "h": 8, @@ -622,15 +576,14 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(getobject_account_state_node_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(getobject_account_state_node_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Account State Get\", \"\", \"\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", "unit": "cps", "custom": { "axisLabel": "Messages In", @@ -641,11 +594,12 @@ } }, "overrides": [] - } + }, + "id": 12 }, { - "title": "GetObject Messages \u2014 Specials", - "description": "###### What this is:\n*Count of individual object-fetch request/response messages per object type.*\n\n###### How it's computed:\n*Per-type inbound message rate per node.*\n\n###### Reading it:\n*Many messages with few bytes means small piecemeal fetches; few messages with many bytes means large batch transfers.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High message counts with tiny payloads sustained over time (inefficient per-node fetching).*\n\n###### Keywords:\n- **GetObject / object fetch** *(per node)* \u2014 peer requests for specific ledger objects \u2014 headers, transactions, or tree nodes \u2014 by type.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#getobject-object-fetch)", + "title": "GetObject Messages — Specials", + "description": "###### What this is:\n*Count of individual object-fetch request/response messages per object type.*\n\n###### How it's computed:\n*Per-type inbound message rate per node.*\n\n###### Reading it:\n*Many messages with few bytes means small piecemeal fetches; few messages with many bytes means large batch transfers.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High message counts with tiny payloads sustained over time (inefficient per-node fetching).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", "type": "timeseries", "gridPos": { "h": 8, @@ -663,29 +617,26 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(getobject_cas_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"CAS Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(getobject_cas_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"CAS Get\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(getobject_fetch_pack_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Fetch Pack Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(getobject_fetch_pack_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Fetch Pack Get\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(getobject_transactions_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Transactions Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(getobject_transactions_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Transactions Get\", \"\", \"\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", "unit": "cps", "custom": { "axisLabel": "Messages In", @@ -696,11 +647,12 @@ } }, "overrides": [] - } + }, + "id": 13 }, { - "title": "GetObject \u2014 Specials", - "description": "###### What this is:\n*Aggregate object-fetch inbound bytes plus special buckets: content-addressed storage fetches, bulk fetch-pack downloads used during catch-up, and bulk transaction fetches.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Fetch-pack rises sharply while catching up a range of ledgers; near zero when fully synced.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*Continuous fetch-pack traffic (node never fully catches up) or unexpectedly high content-store volume.*\n\n###### Keywords:\n- **GetObject / object fetch** *(per node)* \u2014 peer requests for specific ledger objects \u2014 headers, transactions, or tree nodes \u2014 by type.\n- **Fetch-pack** *(per node)* \u2014 a bulk bundle of ledger data peers send to speed up catch-up over many ledgers.\n- **Back-fill / catch-up** *(per node)* \u2014 fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#getobject-object-fetch)", + "title": "GetObject — Specials", + "description": "###### What this is:\n*Aggregate object-fetch inbound bytes plus special buckets: content-addressed storage fetches, bulk fetch-pack downloads used during catch-up, and bulk transaction fetches.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Fetch-pack rises sharply while catching up a range of ledgers; near zero when fully synced.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*Continuous fetch-pack traffic (node never fully catches up) or unexpectedly high content-store volume.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", "type": "timeseries", "gridPos": { "h": 8, @@ -718,57 +670,50 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(getobject_cas_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"CAS Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(getobject_cas_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"CAS Get\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(getobject_cas_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"CAS Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(getobject_cas_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"CAS Share\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(getobject_fetch_pack_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Fetch Pack Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(getobject_fetch_pack_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Fetch Pack Share\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(getobject_fetch_pack_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Fetch Pack Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(getobject_fetch_pack_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Fetch Pack Get\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(getobject_transactions_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Transactions Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(getobject_transactions_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Transactions Get\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(getobject_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Aggregate Get\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(getobject_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Aggregate Get\", \"\", \"\")" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(rate(getobject_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Aggregate Share\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(rate(getobject_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Aggregate Share\", \"\", \"\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", "unit": "Bps", "custom": { "axisLabel": "Bytes In", @@ -779,11 +724,12 @@ } }, "overrides": [] - } + }, + "id": 14 }, { "title": "Overlay Traffic Heatmap (All Categories, Bytes In)", - "description": "###### What this is:\n*All overlay traffic categories ranked by inbound bytes, giving an at-a-glance view of which message types consume the most receive bandwidth.*\n\n###### How it's computed:\n*Top categories by latest inbound byte value across all traffic categories. Each bar is labelled with its traffic category followed by the node identity; the shared `_bytes_in` suffix is dropped from the category name because the panel already reports inbound bytes.*\n\n###### Reading it:\n*The longest bars are the biggest bandwidth consumers; on a synced node transactions, proposals, and validations usually lead.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*A single ledger-data or fetch category dominating (ongoing sync) or an unexpected category topping the list.*\n\n###### Keywords:\n- **Overlay** *(per node)* \u2014 the peer-to-peer network layer over which nodes exchange transactions, proposals, and validations.\n- **Proposal** *(network event)* \u2014 a validator's advertised set of candidate transactions for the next ledger, revised each round.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Overlay](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) \u00b7 [Proposal](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay)", + "description": "###### What this is:\n*All overlay traffic categories ranked by inbound bytes, giving an at-a-glance view of which message types consume the most receive bandwidth.*\n\n###### How it's computed:\n*Top categories by latest inbound byte value across all traffic categories. Each bar is labelled with its traffic category followed by the node identity; the shared `_bytes_in` suffix is dropped from the category name because the panel already reports inbound bytes.*\n\n###### Reading it:\n*The longest bars are the biggest bandwidth consumers; on a synced node transactions, proposals, and validations usually lead.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*A single ledger-data or fetch category dominating (ongoing sync) or an unexpected category topping the list.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", "type": "bargauge", "gridPos": { "h": 18, @@ -808,15 +754,14 @@ "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "type": "prometheus" }, - "expr": "label_replace(label_join(label_replace(topk(20, {service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", __name__=~\".*_bytes_in\", __name__!~\"total_.*\"}), \"series\", \"$1\", \"__name__\", \"(.*)_bytes_in\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(topk(20, {service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", __name__=~\".*_bytes_in\", __name__!~\"total_.*\"}), \"series\", \"$1\", \"__name__\", \"(.*)_bytes_in\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", "unit": "decbytes", "thresholds": { "mode": "absolute", @@ -837,7 +782,8 @@ } }, "overrides": [] - } + }, + "id": 15 }, { "title": "Sync Diagnostics", @@ -849,6 +795,7 @@ "x": 0, "y": 74 }, + "id": 16, "panels": [] }, { @@ -952,7 +899,8 @@ "max": 4 }, "overrides": [] - } + }, + "id": 17 }, { "title": "Validated Ledger Age", @@ -1000,7 +948,8 @@ } }, "overrides": [] - } + }, + "id": 18 }, { "title": "Ledger Close Rate", @@ -1048,7 +997,8 @@ } }, "overrides": [] - } + }, + "id": 19 }, { "title": "Job Queue Wait p95 By Type", @@ -1124,7 +1074,8 @@ } }, "overrides": [] - } + }, + "id": 20 }, { "title": "NuDB Read Latency", @@ -1155,9 +1106,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "\u00b5s", + "unit": "µs", "custom": { - "axisLabel": "Latency (\u00b5s/read)", + "axisLabel": "Latency (µs/read)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -1165,7 +1116,8 @@ } }, "overrides": [] - } + }, + "id": 21 }, { "title": "I/O Scheduler Latency p95", @@ -1206,7 +1158,8 @@ } }, "overrides": [] - } + }, + "id": 22 }, { "title": "NuDB Cache Hit Ratio", @@ -1247,7 +1200,8 @@ } }, "overrides": [] - } + }, + "id": 23 }, { "title": "NuDB Read Pressure", @@ -1302,7 +1256,8 @@ } }, "overrides": [] - } + }, + "id": 24 }, { "title": "Job Queue Depth", @@ -1343,7 +1298,8 @@ } }, "overrides": [] - } + }, + "id": 25 }, { "title": "Load Factor & Peers", @@ -1398,26 +1354,118 @@ } }, "overrides": [] - } + }, + "id": 26 + }, + { + "title": "Job Queue Saturation", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 115 + }, + "collapsed": false, + "panels": [], + "id": 27 + }, + { + "title": "Job Queue Backlog and Deferred by Type", + "description": "###### What this is:\n*Per-job-type queue depth, two series per type. Waiting is the whole backlog: every job enqueued for that type that has not started yet. Deferred is the subset of that backlog that is blocked specifically because the type is already running at its concurrency limit. Deferred is the leading indicator of backpressure, because JobQueue::addJob never rejects a job for queue pressure -- it returns success and defers instead, so a capped type under pressure produces no error and no dropped work. Without these the only evidence is latency, which appears after the harm is already done.*\n\n###### How it's computed:\n*Two targets, each the top 10 gauges by current value: jobq__waiting and jobq__deferred. JobQueue::collect snapshots both counters under the one lock that guards them, so the pair is read at the same instant and is directly comparable, then publishes them on the 1-second export cycle. Gauges exist only for non-special job types, so the 11 special types -- the ones declared with a limit of 0, which bypass the limit logic entirely and therefore never defer -- do not appear on either series.*\n\n###### Reading it:\n*Read the two together; the ratio is the diagnostic, not either value alone. Deferred is always a subset of waiting, because addRefCountedJob increments waiting for every job and deferred only for the ones that arrive while the type is at its limit. Waiting high with deferred at zero means the type has spare slots and the backlog is just arrival burstiness -- it will drain without intervention. Waiting high with deferred also high means the concurrency limit is the binding constraint, not the work. Both near zero is the normal state. These are depths, not rates: the value is how many jobs are queued right now. finishJob drains deferred one per completion, so a deferred line that stays elevated means arrivals are outpacing completions rather than one isolated burst. Only the 10 highest series per state are drawn, which on an idle node is arbitrary among the zeros and under load is exactly the types under pressure.*\n\n###### Healthy range:\n*Deferred zero on all types. Waiting near zero, with brief spikes during ledger close.*\n\n###### Watch for:\n*ledgerrequest deferred above zero: the 3-slot ledgerRequest queue is full, so TMGetLedger service to syncing peers is being delayed. Use LedgerReq Wait by Handler next to see which of its two producers is responsible. ledgerdata or fetchtxndata deferred: inbound ledger data cannot be absorbed fast enough, which is what makes validated ledger age grow. A waiting line that climbs steadily while deferred stays flat points at the worker pool or at slow jobs rather than at the limit. Note both are sampled once per export cycle, so a sub-second spike can be missed; a reading of zero is not proof that nothing was ever queued or deferred.*\n\n###### Keywords:\n- **Deferred job** *(per node)* — a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n- **Concurrency limit** *(per node)* — the cap on how many jobs of one type may run at once; a type at its cap cannot start more work.\n- **Job queue / job type** *(per node)* — xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[core/JobQueue.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)\n\n###### Function:\n`JobQueue::addRefCountedJob / JobQueue::collect`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#deferred-job)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 116 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(10, {service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", __name__=~\"jobq_.*_deferred\"}), \"series\", \"$1 Deferred\", \"__name__\", \"jobq_(.*)_deferred\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(10, {service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", __name__=~\"jobq_.*_waiting\"}), \"series\", \"$1 Waiting\", \"__name__\", \"jobq_(.*)_waiting\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Queued Jobs (Waiting / Deferred)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 28 + }, + { + "title": "LedgerReq Wait by Handler", + "description": "###### What this is:\n*Queue wait for the ledgerRequest job type, split by which handler enqueued the job. The type has a concurrency limit of 3 and two producers that compete for those slots: RcvGetLedger, which serves TMGetLedger to syncing peers, and RcvGetObjByHash, which serves TMGetObjectByHash. Both report the same job_type, so without the handler split a wait spike cannot be attributed to either.*\n\n###### How it's computed:\n*p99 of job_queued_us for job_type=\"ledgerRequest\", grouped by the handler label. JobQueue::processTask measures the wait, then PerfLog hands it to MetricsRegistry::recordJobStarted, which is where the histogram is recorded. The handler value is the addJob name passed through a sanitizer that keeps letters-only names and folds everything else to \"other\", which bounds the label domain to 43 names plus \"other\". Both producers here are letters-only, so both appear under their own names; \"other\" is a mixed bucket and never means one specific caller.*\n\n###### Reading it:\n*This is the panel that answers which producer is starving the 3-slot queue. Both lines high together means the queue is genuinely oversubscribed and both kinds of peer request are being delayed. One line high while the other is flat means that producer is arriving faster than 3 concurrent slots can absorb, and it is the one delaying the other. Wait is queue time only, so a high line here is contention, not slow work; the work itself is on the GetObject Handler Latency Breakdown panel.*\n\n###### Healthy range:\n*Single-digit to low-tens of milliseconds p99 for both handlers, matching the wider Job Queue Wait p95 By Type panel.*\n\n###### Watch for:\n*RcvGetObjByHash wait climbing: one in-bounds TMGetObjectByHash request can perform thousands of NodeStore lookups, so a few concurrent ones occupy every slot and delay TMGetLedger to peers that are themselves syncing. Cross-check Job Queue Backlog and Deferred by Type for jobq_ledgerrequest_deferred above zero to confirm the limit, not the work, is the binding constraint.*\n\n###### Keywords:\n- **Handler label** *(per node)* — the addJob call-site name attached to job metrics, so producers sharing one job type stay separable.\n- **Concurrency limit** *(per node)* — the cap on how many jobs of one type may run at once; a type at its cap cannot start more work.\n- **Job queue / job type** *(per node)* — xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`MetricsRegistry::recordJobStarted`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#handler-label)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 116 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (le, handler, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{job_type=\"ledgerRequest\", handler=~\"$handler\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"$1 q-wait p99\", \"handler\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "µs", + "custom": { + "axisLabel": "p99 Wait (μs)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 29 } ], "schemaVersion": 39, "tags": ["ledger", "sync"], "templating": { "list": [ - { - "name": "DS_PROMETHEUS", - "type": "datasource", - "label": "Prometheus", - "query": "prometheus", - "regex": "", - "current": {}, - "hide": 0, - "refresh": 1, - "includeAll": false, - "multi": false, - "options": [] - }, { "name": "service_name", "label": "Service Name", @@ -1426,7 +1474,7 @@ "query": "label_values(service_name)", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "prometheus" }, "includeAll": true, "allValue": ".*", @@ -1446,7 +1494,7 @@ "query": "label_values(deployment_environment)", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "prometheus" }, "includeAll": true, "allValue": ".*", @@ -1466,7 +1514,7 @@ "query": "label_values(xrpl_network_type)", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "prometheus" }, "includeAll": true, "allValue": ".*", @@ -1546,7 +1594,27 @@ "query": "label_values(target_info, service_instance_id)", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "prometheus" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "handler", + "label": "Handler", + "description": "Filter by enqueueing handler (sanitised addJob name, e.g. RcvGetLedger)", + "type": "query", + "query": "label_values(job_queued_total, handler)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" }, "includeAll": true, "allValue": ".*", @@ -1565,6 +1633,5 @@ "to": "now" }, "title": "Ledger Data & Sync", - "uid": "ledger-data-sync", - "refresh": "30s" + "uid": "ledger-data-sync" } diff --git a/docker/telemetry/grafana/dashboards/ledger-sync-health.json b/docker/telemetry/grafana/dashboards/ledger-sync-health.json index aaea1f051e..3041d83f40 100644 --- a/docker/telemetry/grafana/dashboards/ledger-sync-health.json +++ b/docker/telemetry/grafana/dashboards/ledger-sync-health.json @@ -2859,215 +2859,13 @@ "title": "Worker Pool Capacity & Total Backlog", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Jobs held back because their job type is already running at its concurrency limit. A held-back job is not \"waiting\" and not \"running\" \u2014 it exists and is being denied a worker thread, which is starvation rather than idleness.*\n\n###### How it's computed:\n*jobq_backlog series deferred, per job_type, read from the JobQueue's own per-type deferred counter on each collection tick.*\n\n###### Reading it:\n*0 everywhere is healthy. Any sustained non-zero value names the job type whose concurrency limit is the bottleneck.*\n\n###### Healthy range:\n*0.*\n\n###### Watch for:\n*ledgerData or ledgerRequest deferred above zero during a fresh sync. Both run at a limit of 3, so they are the first types to starve, and this is the only place that state is visible \u2014 the job counters and queue-wait histograms cannot show it.*\n\n###### Keywords:\n- **Deferred job** *(per node)* \u2014 a job the queue accepted but withheld from a worker because its type is at its concurrency limit.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerJobQueueBacklogGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#deferred-job)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Jobs", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "short" - } - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 185 - }, - "id": 24, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(jobq_backlog{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"deferred\", job_type=~\"$job_type\"}, \"series\", \"$1 deferred\", \"job_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - } - ], - "title": "Deferred Jobs by Type (starvation)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Instantaneous per-job-type queue occupancy: how many jobs of each type are queued (waiting) and how many are executing (running) right now.*\n\n###### How it's computed:\n*jobq_backlog series waiting and running, per job_type, sampled from the JobQueue under one lock acquire so the two agree on the same instant.*\n\n###### Reading it:\n*running at a type's concurrency limit with waiting above zero means that type is the constraint. Use the Job Type variable to isolate one type.*\n\n###### Healthy range:\n*waiting near 0; running low single digits per type.*\n\n###### Watch for:\n*A waiting count that climbs while running sits flat at the limit \u2014 pair with the Deferred Jobs panel, which shows how much of that backlog the limit is actively withholding. Distinct from Job Queue Wait p95: that measures how long jobs already waited, this measures how many are waiting now.*\n\n###### Keywords:\n- **Job queue occupancy** *(per node)* \u2014 the number of jobs of a type queued or executing at the moment of sampling.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerJobQueueBacklogGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-occupancy)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Jobs", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - } - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 185 - }, - "id": 25, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(jobq_backlog{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=~\"waiting|running\", job_type=~\"$job_type\"}, \"series\", \"$1\", \"job_type\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - } - ], - "title": "Job Queue Occupancy by Type (waiting/running)", - "type": "timeseries" - }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 197 + "y": 185 }, "id": 60, "panels": [], @@ -3136,7 +2934,7 @@ "h": 12, "w": 12, "x": 0, - "y": 198 + "y": 186 }, "id": 42, "options": { @@ -3247,7 +3045,7 @@ "h": 12, "w": 12, "x": 12, - "y": 198 + "y": 186 }, "id": 44, "options": { @@ -3315,7 +3113,7 @@ "h": 12, "w": 12, "x": 0, - "y": 210 + "y": 198 }, "id": 45, "options": { @@ -3352,7 +3150,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*How many ledgers this node has fully validated but not yet published to its clients and subscribers.*\n\n###### How it's computed:\n*ledger_quorum_publish series publish_lag: the validated ledger sequence minus the published ledger sequence, floored at zero. The published sequence was never exported before, so this gap was not derivable from any other series.*\n\n###### Reading it:\n*Publishing trails validation by design, so a small lag that drains each round is normal. A lag that stays positive, or grows, means validation is healthy and the publish pipeline is not \u2014 a different fault from anything the quorum or acquire panels can show.*\n\n###### Healthy range:\n*0 to 1 ledger.*\n\n###### Watch for:\n*A monotonic climb: the publish loop is falling behind a chain tip the node already holds, so clients and subscriptions see stale data while the node itself is current. Read it with Worker Pool Saturation and Deferred Jobs by Type (starvation) \u2014 a starved job queue is the usual cause. A flat 0 is only healthy on a node that is validating: on one that never has, the 0 means nothing has been validated to publish, so read Trusted Validations vs Quorum Target first.*\n\n###### Keywords:\n- **Publish lag** *(per node)* \u2014 validated ledgers not yet published to clients and subscribers, i.e. the gap between the validated and the published sequence.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerQuorumPublishGauge`\n\n###### References:\n[Ledger close and publication on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#publish-lag)", + "description": "###### What this is:\n*How many ledgers this node has fully validated but not yet published to its clients and subscribers.*\n\n###### How it's computed:\n*ledger_quorum_publish series publish_lag: the validated ledger sequence minus the published ledger sequence, floored at zero. The published sequence was never exported before, so this gap was not derivable from any other series.*\n\n###### Reading it:\n*Publishing trails validation by design, so a small lag that drains each round is normal. A lag that stays positive, or grows, means validation is healthy and the publish pipeline is not \u2014 a different fault from anything the quorum or acquire panels can show.*\n\n###### Healthy range:\n*0 to 1 ledger.*\n\n###### Watch for:\n*A monotonic climb: the publish loop is falling behind a chain tip the node already holds, so clients and subscriptions see stale data while the node itself is current. Read it with Worker Pool Saturation and the per-job-type jobq__deferred gauges — a starved job queue is the usual cause. A flat 0 is only healthy on a node that is validating: on one that never has, the 0 means nothing has been validated to publish, so read Trusted Validations vs Quorum Target first.*\n\n###### Keywords:\n- **Publish lag** *(per node)* \u2014 validated ledgers not yet published to clients and subscribers, i.e. the gap between the validated and the published sequence.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerQuorumPublishGauge`\n\n###### References:\n[Ledger close and publication on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#publish-lag)", "fieldConfig": { "defaults": { "color": { @@ -3417,7 +3215,7 @@ "h": 12, "w": 12, "x": 12, - "y": 210 + "y": 198 }, "id": 43, "options": { @@ -3477,7 +3275,7 @@ "h": 12, "w": 12, "x": 0, - "y": 222 + "y": 210 }, "id": 52, "options": { @@ -3580,7 +3378,7 @@ "h": 12, "w": 12, "x": 12, - "y": 222 + "y": 210 }, "id": 53, "options": { @@ -3631,7 +3429,7 @@ "h": 1, "w": 24, "x": 0, - "y": 234 + "y": 222 }, "id": 61, "panels": [], @@ -3677,7 +3475,7 @@ "h": 12, "w": 12, "x": 0, - "y": 235 + "y": 223 }, "id": 36, "options": { @@ -3775,7 +3573,7 @@ "h": 12, "w": 12, "x": 12, - "y": 235 + "y": 223 }, "id": 37, "options": { @@ -3874,7 +3672,7 @@ "h": 12, "w": 12, "x": 0, - "y": 247 + "y": 235 }, "id": 34, "options": { @@ -3977,7 +3775,7 @@ "h": 12, "w": 12, "x": 12, - "y": 247 + "y": 235 }, "id": 35, "options": { @@ -4020,7 +3818,7 @@ "h": 1, "w": 24, "x": 0, - "y": 259 + "y": 247 }, "id": 62, "panels": [ @@ -4086,7 +3884,7 @@ "h": 12, "w": 12, "x": 0, - "y": 260 + "y": 248 }, "id": 38, "options": { @@ -4193,7 +3991,7 @@ "h": 12, "w": 12, "x": 12, - "y": 260 + "y": 248 }, "id": 39, "options": { @@ -4300,7 +4098,7 @@ "h": 12, "w": 12, "x": 0, - "y": 272 + "y": 260 }, "id": 40, "options": { @@ -4399,7 +4197,7 @@ "h": 12, "w": 12, "x": 12, - "y": 272 + "y": 260 }, "id": 41, "options": { @@ -4446,7 +4244,7 @@ "h": 1, "w": 24, "x": 0, - "y": 260 + "y": 248 }, "id": 63, "panels": [ @@ -4512,7 +4310,7 @@ "h": 12, "w": 12, "x": 0, - "y": 261 + "y": 249 }, "id": 48, "options": { @@ -4627,7 +4425,7 @@ "h": 12, "w": 12, "x": 12, - "y": 261 + "y": 249 }, "id": 49, "options": { @@ -4726,7 +4524,7 @@ "h": 12, "w": 12, "x": 0, - "y": 273 + "y": 261 }, "id": 46, "options": { @@ -4825,7 +4623,7 @@ "h": 12, "w": 12, "x": 12, - "y": 273 + "y": 261 }, "id": 47, "options": { @@ -4924,7 +4722,7 @@ "h": 12, "w": 12, "x": 0, - "y": 285 + "y": 273 }, "id": 50, "options": { @@ -5023,7 +4821,7 @@ "h": 12, "w": 12, "x": 12, - "y": 285 + "y": 273 }, "id": 51, "options": { @@ -5123,7 +4921,7 @@ "h": 12, "w": 24, "x": 0, - "y": 297 + "y": 285 }, "id": 54, "options": { @@ -5502,26 +5300,6 @@ "refresh": 2, "sort": 1 }, - { - "name": "job_type", - "label": "Job Type", - "description": "Filter the job-queue backlog by JobType name [ledgerData / ledgerRequest / ...]", - "type": "query", - "query": "label_values(jobq_backlog, job_type)", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "includeAll": true, - "allValue": ".*", - "current": { - "text": "All", - "value": "$__all" - }, - "multi": true, - "refresh": 2, - "sort": 1 - }, { "name": "saturation_metric", "label": "Saturation Metric", diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index f62f00c16e..80a7c48bc7 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -32,7 +32,7 @@ } ] }, - "description": "What this shows: Operational health of an XRPL node: sync state, ledger progress, caches, storage, job queue, and network economy. \u2014 Use it to: Get an at-a-glance read on whether the node is healthy, synced, and keeping up with the network.", + "description": "What this shows: Operational health of an XRPL node: sync state, ledger progress, caches, storage, job queue, and network economy. — Use it to: Get an at-a-glance read on whether the node is healthy, synced, and keeping up with the network.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -43,7 +43,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*A single at-a-glance verdict: is this node healthy and doing its job (fully synced and keeping up with the network)?*\n\n###### How it's computed:\n*1 (Healthy) only when server_state == Full AND the validated ledger age is under 30s; otherwise 0 (Not Healthy). Combines server_info{metric=\"server_state\"} and ledgermaster_validated_ledger_age.*\n\n###### Reading it:\n*Green \"Healthy\" = full sync and current. Red \"Not Healthy\" = not full, or lagging the network (catching up, flapping, or stalled).*\n\n###### Healthy range:\n*Healthy (green) in steady state.*\n\n###### Watch for:\n*Any sustained Not Healthy \u2014 drill into the Operating Mode and Validated Ledger Age panels below to see whether it is a state or a lag problem.*\n\n###### Keywords:\n- **Full** *(per node)* \u2014 the node has the current validated ledger and complete recent history.\n- **Validated ledger age** *(per node)* \u2014 seconds since the last freshly validated ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 a boolean AND of two native metrics.*\n*Recorded in xrpld code as native metrics (beast::insight); the collector only forwards them; the Grafana query combines them.*\n\n###### Source:\n[app/misc/NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::setMode / getValidatedLedgerAge`\n\n###### References:\n[Server states](https://xrpl.org/docs/concepts/networks-and-servers/rippled-server-states) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md)", + "description": "###### What this is:\n*A single at-a-glance verdict: is this node healthy and doing its job (fully synced and keeping up with the network)?*\n\n###### How it's computed:\n*1 (Healthy) only when server_state == Full AND the validated ledger age is under 30s; otherwise 0 (Not Healthy). Combines server_info{metric=\"server_state\"} and ledgermaster_validated_ledger_age.*\n\n###### Reading it:\n*Green \"Healthy\" = full sync and current. Red \"Not Healthy\" = not full, or lagging the network (catching up, flapping, or stalled).*\n\n###### Healthy range:\n*Healthy (green) in steady state.*\n\n###### Watch for:\n*Any sustained Not Healthy — drill into the Operating Mode and Validated Ledger Age panels below to see whether it is a state or a lag problem.*\n\n###### Keywords:\n- **Full** *(per node)* — the node has the current validated ledger and complete recent history.\n- **Validated ledger age** *(per node)* — seconds since the last freshly validated ledger.\n\n###### Computation boundary:\n*Result: Per node — a boolean AND of two native metrics.*\n*Recorded in xrpld code as native metrics (beast::insight); the collector only forwards them; the Grafana query combines them.*\n\n###### Source:\n[app/misc/NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::setMode / getValidatedLedgerAge`\n\n###### References:\n[Server states](https://xrpl.org/docs/concepts/networks-and-servers/rippled-server-states) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md)", "fieldConfig": { "defaults": { "color": { @@ -122,7 +122,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*How old the most recently validated ledger is, in seconds.*\n\n###### How it's computed:\n*Current value of the validated-ledger-age gauge, refreshed each collection interval.*\n\n###### Reading it:\n*Lower is better; it should stay near the network close interval.*\n\n###### Healthy range:\n*Under about 7 seconds on a healthy node.*\n\n###### Watch for:\n*Values above 20 seconds mean the node is falling behind the network.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* \u2014 a ledger confirmed final by the trusted validator quorum; its contents never change.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LedgerMaster.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/LedgerMaster.h)\n\n###### Function:\n`LedgerMaster::Stats::collectMetrics`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)", + "description": "###### What this is:\n*How old the most recently validated ledger is, in seconds.*\n\n###### How it's computed:\n*Current value of the validated-ledger-age gauge, refreshed each collection interval.*\n\n###### Reading it:\n*Lower is better; it should stay near the network close interval.*\n\n###### Healthy range:\n*Under about 7 seconds on a healthy node.*\n\n###### Watch for:\n*Values above 20 seconds mean the node is falling behind the network.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* — a ledger confirmed final by the trusted validator quorum; its contents never change.\n- **Ledger close interval** *(network-wide)* — the network's steady ledger rhythm — roughly one closed ledger every 3-5 seconds on Mainnet.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LedgerMaster.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/LedgerMaster.h)\n\n###### Function:\n`LedgerMaster::Stats::collectMetrics`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) · [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)", "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", @@ -192,7 +192,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*How old the most recently published ledger is, in seconds.*\n\n###### How it's computed:\n*Current value of the published-ledger-age gauge, refreshed each collection interval.*\n\n###### Reading it:\n*Lower is better; should track close to the validated ledger age.*\n\n###### Healthy range:\n*Under about 7 seconds on a healthy node.*\n\n###### Watch for:\n*A growing gap above the validated age indicates the publish pipeline is backing up.*\n\n###### Keywords:\n- **Published ledger** *(per node)* \u2014 the most recent validated ledger the node has finished publishing to its subscribers.\n- **Validated ledger** *(network-wide)* \u2014 a ledger confirmed final by the trusted validator quorum; its contents never change.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LedgerMaster.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/LedgerMaster.h)\n\n###### Function:\n`LedgerMaster::Stats::collectMetrics`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#published-ledger)", + "description": "###### What this is:\n*How old the most recently published ledger is, in seconds.*\n\n###### How it's computed:\n*Current value of the published-ledger-age gauge, refreshed each collection interval.*\n\n###### Reading it:\n*Lower is better; should track close to the validated ledger age.*\n\n###### Healthy range:\n*Under about 7 seconds on a healthy node.*\n\n###### Watch for:\n*A growing gap above the validated age indicates the publish pipeline is backing up.*\n\n###### Keywords:\n- **Published ledger** *(per node)* — the most recent validated ledger the node has finished publishing to its subscribers.\n- **Validated ledger** *(network-wide)* — a ledger confirmed final by the trusted validator quorum; its contents never change.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LedgerMaster.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/LedgerMaster.h)\n\n###### Function:\n`LedgerMaster::Stats::collectMetrics`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#published-ledger)", "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", @@ -262,7 +262,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*How often the node requests a ledger from its peers.*\n\n###### How it's computed:\n*Per-second rate of ledger-fetch requests over a 5-minute window.*\n\n###### Reading it:\n*Near zero in steady state; elevated while catching up.*\n\n###### Healthy range:\n*Close to zero once fully synced.*\n\n###### Watch for:\n*A sustained high rate means the node is repeatedly missing ledgers and back-filling from peers.*\n\n###### Keywords:\n- **Ledger acquire (inbound fetch)** *(per node)* \u2014 fetching a specific missing ledger from peers; tracked by duration and outcome (complete/failed).\n- **Back-fill / catch-up** *(per node)* \u2014 fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[InboundLedgers.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedgers.cpp)\n\n###### Function:\n`InboundLedgersImp`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-acquire-inbound-fetch)", + "description": "###### What this is:\n*How often the node requests a ledger from its peers.*\n\n###### How it's computed:\n*Per-second rate of ledger-fetch requests over a 5-minute window.*\n\n###### Reading it:\n*Near zero in steady state; elevated while catching up.*\n\n###### Healthy range:\n*Close to zero once fully synced.*\n\n###### Watch for:\n*A sustained high rate means the node is repeatedly missing ledgers and back-filling from peers.*\n\n###### Keywords:\n- **Ledger acquire (inbound fetch)** *(per node)* — fetching a specific missing ledger from peers; tracked by duration and outcome (complete/failed).\n- **Back-fill / catch-up** *(per node)* — fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[InboundLedgers.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedgers.cpp)\n\n###### Function:\n`InboundLedgersImp`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-acquire-inbound-fetch)", "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", @@ -328,7 +328,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate at which a locally built ledger hash fails to match the network-validated hash.*\n\n###### How it's computed:\n*Per-second rate of history-mismatch events over a 5-minute window.*\n\n###### Reading it:\n*Flat at zero is the only healthy reading.*\n\n###### Healthy range:\n*Zero.*\n\n###### Watch for:\n*Any nonzero value indicates consensus divergence or database corruption and warrants immediate investigation.*\n\n###### Keywords:\n- **Ledger history mismatch** *(per node)* \u2014 when a locally built ledger's hash does not match the network-validated hash.\n- **Validated ledger** *(network-wide)* \u2014 a ledger confirmed final by the trusted validator quorum; its contents never change.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementLedgerHistoryMismatch`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-history-mismatch)", + "description": "###### What this is:\n*Rate at which a locally built ledger hash fails to match the network-validated hash.*\n\n###### How it's computed:\n*Per-second rate of history-mismatch events over a 5-minute window.*\n\n###### Reading it:\n*Flat at zero is the only healthy reading.*\n\n###### Healthy range:\n*Zero.*\n\n###### Watch for:\n*Any nonzero value indicates consensus divergence or database corruption and warrants immediate investigation.*\n\n###### Keywords:\n- **Ledger history mismatch** *(per node)* — when a locally built ledger's hash does not match the network-validated hash.\n- **Validated ledger** *(network-wide)* — a ledger confirmed final by the trusted validator quorum; its contents never change.\n- **Consensus** *(network event)* — the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementLedgerHistoryMismatch`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) · [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-history-mismatch)", "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", @@ -394,7 +394,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*The lag between published and validated ledger ages, in seconds.*\n\n###### How it's computed:\n*Published ledger age minus validated ledger age, as a single derived value.*\n\n###### Reading it:\n*Near zero is healthy; a positive value is how far publishing trails validation.*\n\n###### Healthy range:\n*Close to zero.*\n\n###### Watch for:\n*A growing gap means the publish pipeline is falling behind and subscribers may see stale data.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* \u2014 a ledger confirmed final by the trusted validator quorum; its contents never change.\n- **Published ledger** *(per node)* \u2014 the most recent validated ledger the node has finished publishing to its subscribers.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LedgerMaster.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/LedgerMaster.h)\n\n###### Function:\n`LedgerMaster::Stats::collectMetrics`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)", + "description": "###### What this is:\n*The lag between published and validated ledger ages, in seconds.*\n\n###### How it's computed:\n*Published ledger age minus validated ledger age, as a single derived value.*\n\n###### Reading it:\n*Near zero is healthy; a positive value is how far publishing trails validation.*\n\n###### Healthy range:\n*Close to zero.*\n\n###### Watch for:\n*A growing gap means the publish pipeline is falling behind and subscribers may see stale data.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* — a ledger confirmed final by the trusted validator quorum; its contents never change.\n- **Published ledger** *(per node)* — the most recent validated ledger the node has finished publishing to its subscribers.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LedgerMaster.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/LedgerMaster.h)\n\n###### Function:\n`LedgerMaster::Stats::collectMetrics`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)", "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", @@ -579,7 +579,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Fraction of recent wall-clock time the node spent in each operating mode (Disconnected, Connected, Syncing, Tracking, Full).*\n\n###### How it's computed:\n*Per-second rate of each per-mode duration counter divided by the sum of all five mode rates, giving each mode's time share.*\n\n###### Reading it:\n*The Full share should sit at or near 1.0 and dominate; other shares should be near 0.*\n\n###### Healthy range:\n*Full share close to 1.0.*\n\n###### Watch for:\n*Share accumulating in Syncing, Connected, or Disconnected means the node is not staying fully synced.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", + "description": "###### What this is:\n*Fraction of recent wall-clock time the node spent in each operating mode (Disconnected, Connected, Syncing, Tracking, Full).*\n\n###### How it's computed:\n*Per-second rate of each per-mode duration counter divided by the sum of all five mode rates, giving each mode's time share.*\n\n###### Reading it:\n*The Full share should sit at or near 1.0 and dominate; other shares should be near 0.*\n\n###### Healthy range:\n*Full share close to 1.0.*\n\n###### Watch for:\n*Share accumulating in Syncing, Connected, or Disconnected means the node is not staying fully synced.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* — the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", "fieldConfig": { "defaults": { "color": { @@ -716,7 +716,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Cumulative count of transitions into each operating mode.*\n\n###### How it's computed:\n*Current value of each per-mode transition counter, plotted as lines.*\n\n###### Reading it:\n*Flat lines are healthy; steps up mean the node changed mode.*\n\n###### Healthy range:\n*Few transitions once the node is stable in Full mode.*\n\n###### Watch for:\n*Frequent transitions out of Full, or into Disconnected or Syncing, indicate instability.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", + "description": "###### What this is:\n*Cumulative count of transitions into each operating mode.*\n\n###### How it's computed:\n*Current value of each per-mode transition counter, plotted as lines.*\n\n###### Reading it:\n*Flat lines are healthy; steps up mean the node changed mode.*\n\n###### Healthy range:\n*Few transitions once the node is stable in Full mode.*\n\n###### Watch for:\n*Frequent transitions out of Full, or into Disconnected or Syncing, indicate instability.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* — the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", "fieldConfig": { "defaults": { "color": { @@ -851,7 +851,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Which operating mode the node is accumulating time in right now, one line per state, normalized to seconds per second.*\n\n###### How it's computed:\n*Per-second rate of each state's duration counter, scaled from microseconds to seconds. The five lines sum to about 1.0 because the node is always in exactly one state.*\n\n###### Reading it:\n*The line sitting near 1.0 is the state the node is currently in; the others sit at 0. A handover between two lines marks a state change, and its width is how long that state lasted.*\n\n###### Healthy range:\n*Full near 1.0 with every other line at 0.*\n\n###### Watch for:\n*Time accumulating in Connected or Syncing means the node is catching up rather than serving; repeated handovers mean it is flapping.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Server states](https://xrpl.org/docs/concepts/networks-and-servers/rippled-server-states)", + "description": "###### What this is:\n*Which operating mode the node is accumulating time in right now, one line per state, normalized to seconds per second.*\n\n###### How it's computed:\n*Per-second rate of each state's duration counter, scaled from microseconds to seconds. The five lines sum to about 1.0 because the node is always in exactly one state.*\n\n###### Reading it:\n*The line sitting near 1.0 is the state the node is currently in; the others sit at 0. A handover between two lines marks a state change, and its width is how long that state lasted.*\n\n###### Healthy range:\n*Full near 1.0 with every other line at 0.*\n\n###### Watch for:\n*Time accumulating in Connected or Syncing means the node is catching up rather than serving; repeated handovers mean it is flapping.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* — the node's sync level: Disconnected, Connected, Syncing, Tracking, Full.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Server states](https://xrpl.org/docs/concepts/networks-and-servers/rippled-server-states)", "fieldConfig": { "defaults": { "color": { @@ -986,7 +986,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*The share of recent wall-clock time the node spent in Full mode.*\n\n###### How it's computed:\n*Per-second rate of the Full-mode duration counter divided by the sum of the per-second rates of all five mode duration counters.*\n\n###### Reading it:\n*Higher is better; 1.0 means the node was fully synced for the entire window.*\n\n###### Healthy range:\n*At or above 0.99.*\n\n###### Watch for:\n*Values dropping below 0.9, meaning the node spent meaningful time outside Full mode.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", + "description": "###### What this is:\n*The share of recent wall-clock time the node spent in Full mode.*\n\n###### How it's computed:\n*Per-second rate of the Full-mode duration counter divided by the sum of the per-second rates of all five mode duration counters.*\n\n###### Reading it:\n*Higher is better; 1.0 means the node was fully synced for the entire window.*\n\n###### Healthy range:\n*At or above 0.99.*\n\n###### Watch for:\n*Values dropping below 0.9, meaning the node spent meaningful time outside Full mode.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* — the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", @@ -1058,7 +1058,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*The node's operating mode over time as a colored timeline (Disconnected, Connected, Syncing, Tracking, Full).*\n\n###### How it's computed:\n*Current value of the server-state gauge mapped to a mode name; equal consecutive samples are merged into a single band.*\n\n###### Reading it:\n*A solid green Full band across the window is the goal; other colors mark periods the node was not fully synced.*\n\n###### Healthy range:\n*Continuously Full (green).*\n\n###### Watch for:\n*Bands of Syncing, Connected, or Disconnected, which pinpoint exactly when the node dropped out of Full.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", + "description": "###### What this is:\n*The node's operating mode over time as a colored timeline (Disconnected, Connected, Syncing, Tracking, Full).*\n\n###### How it's computed:\n*Current value of the server-state gauge mapped to a mode name; equal consecutive samples are merged into a single band.*\n\n###### Reading it:\n*A solid green Full band across the window is the goal; other colors mark periods the node was not fully synced.*\n\n###### Healthy range:\n*Continuously Full (green).*\n\n###### Watch for:\n*Bands of Syncing, Connected, or Disconnected, which pinpoint exactly when the node dropped out of Full.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* — the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", "fieldConfig": { "defaults": { "color": { @@ -1199,7 +1199,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Cumulative object-store read, cache-hit, and write operation counts.*\n\n###### How it's computed:\n*Current values of the read, cache-hit, and write counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope reflects store activity.*\n\n###### Healthy range:\n*Reads and writes rising smoothly, with cache-hits tracking a good fraction of reads.*\n\n###### Watch for:\n*A sudden surge in reads or writes signals heavy back-end I/O, from sync, replay, or query load.*\n\n###### Keywords:\n- **NodeStore** *(per node)* \u2014 the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#nodestore)", + "description": "###### What this is:\n*Cumulative object-store read, cache-hit, and write operation counts.*\n\n###### How it's computed:\n*Current values of the read, cache-hit, and write counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope reflects store activity.*\n\n###### Healthy range:\n*Reads and writes rising smoothly, with cache-hits tracking a good fraction of reads.*\n\n###### Watch for:\n*A sudden surge in reads or writes signals heavy back-end I/O, from sync, replay, or query load.*\n\n###### Keywords:\n- **NodeStore** *(per node)* — the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#nodestore)", "fieldConfig": { "defaults": { "color": { @@ -1318,7 +1318,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Cumulative object-store read, cache-hit, and write operation counts.*\n\n###### How it's computed:\n*Current values of the read, cache-hit, and write counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope reflects store activity.*\n\n###### Healthy range:\n*Reads and writes rising smoothly, with cache-hits tracking a good fraction of reads.*\n\n###### Watch for:\n*A sudden surge in reads or writes signals heavy back-end I/O, from sync, replay, or query load.*\n\n###### Keywords:\n- **NodeStore** *(per node)* \u2014 the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#nodestore)", + "description": "###### What this is:\n*Cumulative object-store read, cache-hit, and write operation counts.*\n\n###### How it's computed:\n*Current values of the read, cache-hit, and write counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope reflects store activity.*\n\n###### Healthy range:\n*Reads and writes rising smoothly, with cache-hits tracking a good fraction of reads.*\n\n###### Watch for:\n*A sudden surge in reads or writes signals heavy back-end I/O, from sync, replay, or query load.*\n\n###### Keywords:\n- **NodeStore** *(per node)* — the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#nodestore)", "fieldConfig": { "defaults": { "color": { @@ -1421,7 +1421,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Instantaneous write-load score and read-queue depth of the object store.*\n\n###### How it's computed:\n*Current values of the write-load and read-queue gauges, plotted over time.*\n\n###### Reading it:\n*Lower is better for both; short, flat lines are healthy.*\n\n###### Healthy range:\n*Write load near zero and read queue in low double digits or less.*\n\n###### Watch for:\n*High write load means back-end pressure; a high read queue means the prefetch threads are saturated.*\n\n###### Keywords:\n- **Transaction queue (TxQ)** *(per node)* \u2014 holds transactions that meet local cost but not the open-ledger cost, to include in a later ledger.\n- **NodeStore** *(per node)* \u2014 the key-value object store holding ledger data (tree nodes), backed by NuDB.\n- **Read threads / read queue / write load** *(per node)* \u2014 NodeStore back-end I/O internals \u2014 worker threads reading, their queue depth, and write pressure.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Transaction queue (TxQ)](https://xrpl.org/docs/concepts/transactions/transaction-queue) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-queue-txq)", + "description": "###### What this is:\n*Instantaneous write-load score and read-queue depth of the object store.*\n\n###### How it's computed:\n*Current values of the write-load and read-queue gauges, plotted over time.*\n\n###### Reading it:\n*Lower is better for both; short, flat lines are healthy.*\n\n###### Healthy range:\n*Write load near zero and read queue in low double digits or less.*\n\n###### Watch for:\n*High write load means back-end pressure; a high read queue means the prefetch threads are saturated.*\n\n###### Keywords:\n- **Transaction queue (TxQ)** *(per node)* — holds transactions that meet local cost but not the open-ledger cost, to include in a later ledger.\n- **NodeStore** *(per node)* — the key-value object store holding ledger data (tree nodes), backed by NuDB.\n- **Read threads / read queue / write load** *(per node)* — NodeStore back-end I/O internals — worker threads reading, their queue depth, and write pressure.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Transaction queue (TxQ)](https://xrpl.org/docs/concepts/transactions/transaction-queue) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-queue-txq)", "fieldConfig": { "defaults": { "color": { @@ -1536,7 +1536,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Cumulative bytes read from and written to the object-store back end.*\n\n###### How it's computed:\n*Current values of the bytes-read and bytes-written counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope shows throughput.*\n\n###### Healthy range:\n*Smooth growth consistent with ledger and query activity.*\n\n###### Watch for:\n*A sharp acceleration indicates a heavy I/O phase such as sync, replay, or large queries.*\n\n###### Keywords:\n- **NodeStore** *(per node)* \u2014 the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#nodestore)", + "description": "###### What this is:\n*Cumulative bytes read from and written to the object-store back end.*\n\n###### How it's computed:\n*Current values of the bytes-read and bytes-written counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope shows throughput.*\n\n###### Healthy range:\n*Smooth growth consistent with ledger and query activity.*\n\n###### Watch for:\n*A sharp acceleration indicates a heavy I/O phase such as sync, replay, or large queries.*\n\n###### Keywords:\n- **NodeStore** *(per node)* — the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#nodestore)", "fieldConfig": { "defaults": { "color": { @@ -1647,7 +1647,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Read-thread utilization, bundled read count, and cumulative read time for the object store.*\n\n###### How it's computed:\n*Current values of the running/total read-thread gauges, read-bundle gauge, and cumulative read-duration counter, plotted as lines.*\n\n###### Reading it:\n*Running threads well below total means spare capacity; a rising duration line reflects time spent in read I/O.*\n\n###### Healthy range:\n*Running threads below the total count most of the time.*\n\n###### Watch for:\n*Running threads pinned at the total for long periods means read I/O is saturated.*\n\n###### Keywords:\n- **Read threads / read queue / write load** *(per node)* \u2014 NodeStore back-end I/O internals \u2014 worker threads reading, their queue depth, and write pressure.\n- **NodeStore** *(per node)* \u2014 the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#read-threads-read-queue-write-load)", + "description": "###### What this is:\n*Read-thread utilization, bundled read count, and cumulative read time for the object store.*\n\n###### How it's computed:\n*Current values of the running/total read-thread gauges, read-bundle gauge, and cumulative read-duration counter, plotted as lines.*\n\n###### Reading it:\n*Running threads well below total means spare capacity; a rising duration line reflects time spent in read I/O.*\n\n###### Healthy range:\n*Running threads below the total count most of the time.*\n\n###### Watch for:\n*Running threads pinned at the total for long periods means read I/O is saturated.*\n\n###### Keywords:\n- **Read threads / read queue / write load** *(per node)* — NodeStore back-end I/O internals — worker threads reading, their queue depth, and write pressure.\n- **NodeStore** *(per node)* — the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#read-threads-read-queue-write-load)", "fieldConfig": { "defaults": { "color": { @@ -1766,7 +1766,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Read-thread utilization, bundled read count, and cumulative read time for the object store.*\n\n###### How it's computed:\n*Current values of the running/total read-thread gauges, read-bundle gauge, and cumulative read-duration counter, plotted as lines.*\n\n###### Reading it:\n*Running threads well below total means spare capacity; a rising duration line reflects time spent in read I/O.*\n\n###### Healthy range:\n*Running threads below the total count most of the time.*\n\n###### Watch for:\n*Running threads pinned at the total for long periods means read I/O is saturated.*\n\n###### Keywords:\n- **Read threads / read queue / write load** *(per node)* \u2014 NodeStore back-end I/O internals \u2014 worker threads reading, their queue depth, and write pressure.\n- **NodeStore** *(per node)* \u2014 the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#read-threads-read-queue-write-load)", + "description": "###### What this is:\n*Read-thread utilization, bundled read count, and cumulative read time for the object store.*\n\n###### How it's computed:\n*Current values of the running/total read-thread gauges, read-bundle gauge, and cumulative read-duration counter, plotted as lines.*\n\n###### Reading it:\n*Running threads well below total means spare capacity; a rising duration line reflects time spent in read I/O.*\n\n###### Healthy range:\n*Running threads below the total count most of the time.*\n\n###### Watch for:\n*Running threads pinned at the total for long periods means read I/O is saturated.*\n\n###### Keywords:\n- **Read threads / read queue / write load** *(per node)* — NodeStore back-end I/O internals — worker threads reading, their queue depth, and write pressure.\n- **NodeStore** *(per node)* — the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#read-threads-read-queue-write-load)", "fieldConfig": { "defaults": { "color": { @@ -2566,7 +2566,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Hit rates for the SLE, Ledger, and AcceptedLedger caches, from 0 to 1.*\n\n###### How it's computed:\n*Current values of the per-cache hit-rate gauges, plotted as lines.*\n\n###### Reading it:\n*Higher is better; each line is the fraction of lookups served from cache.*\n\n###### Healthy range:\n*Above roughly 0.8 in steady state.*\n\n###### Watch for:\n*Low or falling hit rates indicate cache thrashing and extra back-end reads.*\n\n###### Keywords:\n- **Caches (SLE / Ledger / TreeNode / FullBelow / AcceptedLedger)** *(per node)* \u2014 in-memory caches that avoid re-reading or re-computing ledger data; higher hit rates mean less back-end work.\n- **NodeStore** *(per node)* \u2014 the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerCacheHitRateGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#caches)", + "description": "###### What this is:\n*Hit rates for the SLE, Ledger, and AcceptedLedger caches, from 0 to 1.*\n\n###### How it's computed:\n*Current values of the per-cache hit-rate gauges, plotted as lines.*\n\n###### Reading it:\n*Higher is better; each line is the fraction of lookups served from cache.*\n\n###### Healthy range:\n*Above roughly 0.8 in steady state.*\n\n###### Watch for:\n*Low or falling hit rates indicate cache thrashing and extra back-end reads.*\n\n###### Keywords:\n- **Caches (SLE / Ledger / TreeNode / FullBelow / AcceptedLedger)** *(per node)* — in-memory caches that avoid re-reading or re-computing ledger data; higher hit rates mean less back-end work.\n- **NodeStore** *(per node)* — the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerCacheHitRateGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#caches)", "fieldConfig": { "defaults": { "color": { @@ -2687,7 +2687,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Entry counts for the TreeNode cache and track set, the FullBelow cache, and the AcceptedLedger cache.*\n\n###### How it's computed:\n*Current values of the per-cache size gauges, plotted as lines.*\n\n###### Reading it:\n*Stable lines are normal; sizes grow with working set and shrink after sweeps.*\n\n###### Healthy range:\n*Stable within configured limits.*\n\n###### Watch for:\n*Unbounded growth suggests memory pressure or a cache not being swept.*\n\n###### Keywords:\n- **Caches (SLE / Ledger / TreeNode / FullBelow / AcceptedLedger)** *(per node)* \u2014 in-memory caches that avoid re-reading or re-computing ledger data; higher hit rates mean less back-end work.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerCacheHitRateGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#caches)", + "description": "###### What this is:\n*Entry counts for the TreeNode cache and track set, the FullBelow cache, and the AcceptedLedger cache.*\n\n###### How it's computed:\n*Current values of the per-cache size gauges, plotted as lines.*\n\n###### Reading it:\n*Stable lines are normal; sizes grow with working set and shrink after sweeps.*\n\n###### Healthy range:\n*Stable within configured limits.*\n\n###### Watch for:\n*Unbounded growth suggests memory pressure or a cache not being swept.*\n\n###### Keywords:\n- **Caches (SLE / Ledger / TreeNode / FullBelow / AcceptedLedger)** *(per node)* — in-memory caches that avoid re-reading or re-computing ledger data; higher hit rates mean less back-end work.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerCacheHitRateGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#caches)", "fieldConfig": { "defaults": { "color": { @@ -2814,7 +2814,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Number of entries in the FullBelowCache, which tracks subtrees known to be fully present locally.*\n\n###### How it's computed:\n*Current value of the cache size gauge, plotted over time.*\n\n###### Reading it:\n*A stable size is normal; it grows during acquisition and is trimmed by sweeps.*\n\n###### Healthy range:\n*Stable within its configured bound.*\n\n###### Watch for:\n*Unbounded growth suggests the cache is not being swept.*\n\n###### Keywords:\n- **Caches (SLE / Ledger / TreeNode / FullBelow / AcceptedLedger)** *(per node)* \u2014 in-memory caches that avoid re-reading or re-computing ledger data; higher hit rates mean less back-end work.\n- **Ledger acquire (inbound fetch)** *(per node)* \u2014 fetching a specific missing ledger from peers; tracked by duration and outcome (complete/failed).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[TaggedCache.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/basics/TaggedCache.h)\n\n###### Function:\n`TaggedCache::Stats`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#caches)", + "description": "###### What this is:\n*Number of entries in the FullBelowCache, which tracks subtrees known to be fully present locally.*\n\n###### How it's computed:\n*Current value of the cache size gauge, plotted over time.*\n\n###### Reading it:\n*A stable size is normal; it grows during acquisition and is trimmed by sweeps.*\n\n###### Healthy range:\n*Stable within its configured bound.*\n\n###### Watch for:\n*Unbounded growth suggests the cache is not being swept.*\n\n###### Keywords:\n- **Caches (SLE / Ledger / TreeNode / FullBelow / AcceptedLedger)** *(per node)* — in-memory caches that avoid re-reading or re-computing ledger data; higher hit rates mean less back-end work.\n- **Ledger acquire (inbound fetch)** *(per node)* — fetching a specific missing ledger from peers; tracked by duration and outcome (complete/failed).\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[TaggedCache.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/basics/TaggedCache.h)\n\n###### Function:\n`TaggedCache::Stats`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#caches)", "fieldConfig": { "defaults": { "color": { @@ -2917,7 +2917,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Hit-rate percentage for the FullBelowCache.*\n\n###### How it's computed:\n*Current value of the cache hit-rate gauge.*\n\n###### Reading it:\n*Higher is better; it shows how often cached subtree knowledge is reused.*\n\n###### Healthy range:\n*Above roughly 50 percent in steady state.*\n\n###### Watch for:\n*A low hit rate during steady state means redundant subtree work and warrants investigation.*\n\n###### Keywords:\n- **Caches (SLE / Ledger / TreeNode / FullBelow / AcceptedLedger)** *(per node)* \u2014 in-memory caches that avoid re-reading or re-computing ledger data; higher hit rates mean less back-end work.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[TaggedCache.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/basics/TaggedCache.h)\n\n###### Function:\n`TaggedCache::Stats`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#caches)", + "description": "###### What this is:\n*Hit-rate percentage for the FullBelowCache.*\n\n###### How it's computed:\n*Current value of the cache hit-rate gauge.*\n\n###### Reading it:\n*Higher is better; it shows how often cached subtree knowledge is reused.*\n\n###### Healthy range:\n*Above roughly 50 percent in steady state.*\n\n###### Watch for:\n*A low hit rate during steady state means redundant subtree work and warrants investigation.*\n\n###### Keywords:\n- **Caches (SLE / Ledger / TreeNode / FullBelow / AcceptedLedger)** *(per node)* — in-memory caches that avoid re-reading or re-computing ledger data; higher hit rates mean less back-end work.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[TaggedCache.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/basics/TaggedCache.h)\n\n###### Function:\n`TaggedCache::Stats`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#caches)", "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", @@ -3016,7 +3016,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Current state of each XRPL node. Green = FULL (healthy); orange/yellow = syncing in progress; red = disconnected.\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", + "description": "Current state of each XRPL node. Green = FULL (healthy); orange/yellow = syncing in progress; red = disconnected.\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* — the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", "fieldConfig": { "defaults": { "color": { @@ -3318,7 +3318,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Sequence number of the current open ledger.*\n\n###### How it's computed:\n*Current value of the open-ledger index gauge.*\n\n###### Reading it:\n*Should climb steadily; the gap above the validated sequence is the ledgers in flight.*\n\n###### Healthy range:\n*One or two ahead of the validated sequence.*\n\n###### Watch for:\n*A large or growing gap above the validated sequence means validation is lagging behind ledger creation.*\n\n###### Keywords:\n- **Ledger index** *(network-wide)* \u2014 the sequence number identifying a ledger version; increases by one each close.\n- **Open ledger** *(per node)* \u2014 the temporary workspace ledger where incoming transactions are provisionally applied before a close.\n- **Validated ledger** *(network-wide)* \u2014 a ledger confirmed final by the trusted validator quorum; its contents never change.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Ledger index](https://xrpl.org/docs/references/protocol/data-types/basic-data-types#ledger-index) \u00b7 [Open ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-index)", + "description": "###### What this is:\n*Sequence number of the current open ledger.*\n\n###### How it's computed:\n*Current value of the open-ledger index gauge.*\n\n###### Reading it:\n*Should climb steadily; the gap above the validated sequence is the ledgers in flight.*\n\n###### Healthy range:\n*One or two ahead of the validated sequence.*\n\n###### Watch for:\n*A large or growing gap above the validated sequence means validation is lagging behind ledger creation.*\n\n###### Keywords:\n- **Ledger index** *(network-wide)* — the sequence number identifying a ledger version; increases by one each close.\n- **Open ledger** *(per node)* — the temporary workspace ledger where incoming transactions are provisionally applied before a close.\n- **Validated ledger** *(network-wide)* — a ledger confirmed final by the trusted validator quorum; its contents never change.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Ledger index](https://xrpl.org/docs/references/protocol/data-types/basic-data-types#ledger-index) · [Open ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-index)", "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", @@ -3380,7 +3380,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*The spread in validated ledger sequence across all selected nodes.*\n\n###### How it's computed:\n*Highest validated ledger sequence minus the lowest, across the selected nodes.*\n\n###### Reading it:\n*0 means every node agrees on the same validated ledger; larger means they diverge.*\n\n###### Healthy range:\n*0 to 1 ledger in steady state.*\n\n###### Watch for:\n*A sustained spread above a few ledgers means some nodes are lagging or the fleet is diverging.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* \u2014 a ledger confirmed final by the trusted validator quorum; its contents never change.\n\n###### Computation boundary:\n*Result: Across the selected nodes \u2014 the query aggregates instances into one series.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)", + "description": "###### What this is:\n*The spread in validated ledger sequence across all selected nodes.*\n\n###### How it's computed:\n*Highest validated ledger sequence minus the lowest, across the selected nodes.*\n\n###### Reading it:\n*0 means every node agrees on the same validated ledger; larger means they diverge.*\n\n###### Healthy range:\n*0 to 1 ledger in steady state.*\n\n###### Watch for:\n*A sustained spread above a few ledgers means some nodes are lagging or the fleet is diverging.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* — a ledger confirmed final by the trusted validator quorum; its contents never change.\n\n###### Computation boundary:\n*Result: Across the selected nodes — the query aggregates instances into one series.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)", "fieldConfig": { "defaults": { "color": { @@ -3470,7 +3470,7 @@ "refId": "A" } ], - "title": "Validated Ledger Seq \u2014 Convergence (Max \u2212 Min)", + "title": "Validated Ledger Seq — Convergence (Max − Min)", "type": "stat" }, { @@ -3478,7 +3478,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*How far each node's validated ledger lags behind the network tip, in ledgers.*\n\n###### How it's computed:\n*Highest validated ledger sequence across the selected nodes, minus each node's own sequence.*\n\n###### Reading it:\n*0 means the node is at the tip; larger values mean it trails further behind.*\n\n###### Healthy range:\n*0 to 1 ledger on a synced node.*\n\n###### Watch for:\n*A node stuck at a growing value is falling behind and not keeping up with consensus.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* \u2014 a ledger confirmed final by the trusted validator quorum; its contents never change.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Across the selected nodes \u2014 the query aggregates instances into one series.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)", + "description": "###### What this is:\n*How far each node's validated ledger lags behind the network tip, in ledgers.*\n\n###### How it's computed:\n*Highest validated ledger sequence across the selected nodes, minus each node's own sequence.*\n\n###### Reading it:\n*0 means the node is at the tip; larger values mean it trails further behind.*\n\n###### Healthy range:\n*0 to 1 ledger on a synced node.*\n\n###### Watch for:\n*A node stuck at a growing value is falling behind and not keeping up with consensus.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* — a ledger confirmed final by the trusted validator quorum; its contents never change.\n- **Consensus** *(network event)* — the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Across the selected nodes — the query aggregates instances into one series.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) · [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)", "fieldConfig": { "defaults": { "color": { @@ -3535,7 +3535,7 @@ "refId": "A" } ], - "title": "Validated Ledger Seq \u2014 Current (Stat)", + "title": "Validated Ledger Seq — Current (Stat)", "type": "stat" }, { @@ -3603,7 +3603,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Proposer count and convergence time from the last closed consensus round.*\n\n###### How it's computed:\n*Current values of the last-close proposer-count and convergence-time gauges, plotted as lines.*\n\n###### Reading it:\n*A healthy proposer count with a low convergence time is good.*\n\n###### Healthy range:\n*Convergence time of a few seconds with the expected number of proposers.*\n\n###### Watch for:\n*A falling proposer count or rising convergence time signals degrading consensus conditions.*\n\n###### Keywords:\n- **Proposers** *(network event)* \u2014 the count of validators whose proposals this node heard in the last closed round.\n- **Convergence time** *(network event)* \u2014 the wall-clock time the network took to agree a ledger in a round.\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Proposers](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#proposers)", + "description": "###### What this is:\n*Proposer count and convergence time from the last closed consensus round.*\n\n###### How it's computed:\n*Current values of the last-close proposer-count and convergence-time gauges, plotted as lines.*\n\n###### Reading it:\n*A healthy proposer count with a low convergence time is good.*\n\n###### Healthy range:\n*Convergence time of a few seconds with the expected number of proposers.*\n\n###### Watch for:\n*A falling proposer count or rising convergence time signals degrading consensus conditions.*\n\n###### Keywords:\n- **Proposers** *(network event)* — the count of validators whose proposals this node heard in the last closed round.\n- **Convergence time** *(network event)* — the wall-clock time the network took to agree a ledger in a round.\n- **Consensus round** *(network event)* — one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Consensus** *(network event)* — the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Proposers](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#proposers)", "fieldConfig": { "defaults": { "color": { @@ -3694,7 +3694,7 @@ "refId": "A" } ], - "title": "Last Close \u2014 Converge Time", + "title": "Last Close — Converge Time", "type": "timeseries" }, { @@ -3702,7 +3702,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Proposer count and convergence time from the last closed consensus round.*\n\n###### How it's computed:\n*Current values of the last-close proposer-count and convergence-time gauges, plotted as lines.*\n\n###### Reading it:\n*A healthy proposer count with a low convergence time is good.*\n\n###### Healthy range:\n*Convergence time of a few seconds with the expected number of proposers.*\n\n###### Watch for:\n*A falling proposer count or rising convergence time signals degrading consensus conditions.*\n\n###### Keywords:\n- **Proposers** *(network event)* \u2014 the count of validators whose proposals this node heard in the last closed round.\n- **Convergence time** *(network event)* \u2014 the wall-clock time the network took to agree a ledger in a round.\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Proposers](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#proposers)", + "description": "###### What this is:\n*Proposer count and convergence time from the last closed consensus round.*\n\n###### How it's computed:\n*Current values of the last-close proposer-count and convergence-time gauges, plotted as lines.*\n\n###### Reading it:\n*A healthy proposer count with a low convergence time is good.*\n\n###### Healthy range:\n*Convergence time of a few seconds with the expected number of proposers.*\n\n###### Watch for:\n*A falling proposer count or rising convergence time signals degrading consensus conditions.*\n\n###### Keywords:\n- **Proposers** *(network event)* — the count of validators whose proposals this node heard in the last closed round.\n- **Convergence time** *(network event)* — the wall-clock time the network took to agree a ledger in a round.\n- **Consensus round** *(network event)* — one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Consensus** *(network event)* — the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Proposers](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#proposers)", "fieldConfig": { "defaults": { "color": { @@ -3793,7 +3793,7 @@ "refId": "A" } ], - "title": "Last Close \u2014 Proposers", + "title": "Last Close — Proposers", "type": "timeseries" }, { @@ -3801,7 +3801,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*The wall-clock time between consecutive ledger closes \u2014 the network close cadence.*\n\n###### How it's computed:\n*Close Interval: 1 / rate(ledgers_closed_total), the average seconds between closes from the monotonic close counter (scrape-independent, unlike a gauge delta which would alias to the scrape period). Last-Close Age: time() minus the last-close network time (server_info last_close_time gauge + Ripple-epoch offset), i.e. seconds since the last ledger closed.*\n\n###### Reading it:\n*A steady line near the network's target close interval.*\n\n###### Healthy range:\n*About 3-5s on mainnet; workload-dependent on test networks.*\n\n###### Watch for:\n*A rising interval (consensus slowing or the node lagging) or a flat line at zero (ledgers no longer closing).*\n\n###### Keywords:\n- **Ledger close** *(network event)* \u2014 the current open ledger is closed and a new closed ledger is built from the agreed transaction set.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp) \u00b7 [RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`registerServerInfoGauge (last_close_time) ; ledgers_closed_total`\n\n###### References:\n[Ledger close](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-close)", + "description": "###### What this is:\n*The wall-clock time between consecutive ledger closes — the network close cadence.*\n\n###### How it's computed:\n*Close Interval: 1 / rate(ledgers_closed_total), the average seconds between closes from the monotonic close counter (scrape-independent, unlike a gauge delta which would alias to the scrape period). Last-Close Age: time() minus the last-close network time (server_info last_close_time gauge + Ripple-epoch offset), i.e. seconds since the last ledger closed.*\n\n###### Reading it:\n*A steady line near the network's target close interval.*\n\n###### Healthy range:\n*About 3-5s on mainnet; workload-dependent on test networks.*\n\n###### Watch for:\n*A rising interval (consensus slowing or the node lagging) or a flat line at zero (ledgers no longer closing).*\n\n###### Keywords:\n- **Ledger close** *(network event)* — the current open ledger is closed and a new closed ledger is built from the agreed transaction set.\n- **Ledger close interval** *(network-wide)* — the network's steady ledger rhythm — roughly one closed ledger every 3-5 seconds on Mainnet.\n- **Consensus** *(network event)* — the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp) · [RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`registerServerInfoGauge (last_close_time) ; ledgers_closed_total`\n\n###### References:\n[Ledger close](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) · [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) · [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-close)", "fieldConfig": { "defaults": { "color": { @@ -3908,7 +3908,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*On-disk size of the NuDB object-store back end, in bytes.*\n\n###### How it's computed:\n*Current value of the NuDB storage-size gauge, plotted over time.*\n\n###### Reading it:\n*A smoothly growing line is normal; the slope is the data growth rate.*\n\n###### Healthy range:\n*Gradual growth consistent with retained ledger history.*\n\n###### Watch for:\n*A sudden jump in growth rate can indicate runaway storage or an unexpected back-fill.*\n\n###### Keywords:\n- **NuDB** *(per node)* \u2014 the append-only key-value database used as the default NodeStore backend.\n- **NodeStore** *(per node)* \u2014 the key-value object store holding ledger data (tree nodes), backed by NuDB.\n- **Back-fill / catch-up** *(per node)* \u2014 fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerStorageDetailGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#nudb)", + "description": "###### What this is:\n*On-disk size of the NuDB object-store back end, in bytes.*\n\n###### How it's computed:\n*Current value of the NuDB storage-size gauge, plotted over time.*\n\n###### Reading it:\n*A smoothly growing line is normal; the slope is the data growth rate.*\n\n###### Healthy range:\n*Gradual growth consistent with retained ledger history.*\n\n###### Watch for:\n*A sudden jump in growth rate can indicate runaway storage or an unexpected back-fill.*\n\n###### Keywords:\n- **NuDB** *(per node)* — the append-only key-value database used as the default NodeStore backend.\n- **NodeStore** *(per node)* — the key-value object store holding ledger data (tree nodes), backed by NuDB.\n- **Back-fill / catch-up** *(per node)* — fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerStorageDetailGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#nudb)", "fieldConfig": { "defaults": { "color": { @@ -4122,7 +4122,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*The contiguous ranges of ledgers the node holds locally.*\n\n###### How it's computed:\n*Current start and end bounds of each complete range, listed as table rows.*\n\n###### Reading it:\n*Fewer ranges is better; one continuous range means an unbroken history.*\n\n###### Healthy range:\n*A single range covering the configured retention window.*\n\n###### Watch for:\n*Many fragmented ranges indicate gaps in stored history from missed or failed fetches.*\n\n###### Keywords:\n- **Complete ledger ranges** *(per node)* \u2014 the contiguous spans of ledgers the node holds locally; one unbroken range is ideal.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerCompleteLedgersGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#complete-ledger-ranges)", + "description": "###### What this is:\n*The contiguous ranges of ledgers the node holds locally.*\n\n###### How it's computed:\n*Current start and end bounds of each complete range, listed as table rows.*\n\n###### Reading it:\n*Fewer ranges is better; one continuous range means an unbroken history.*\n\n###### Healthy range:\n*A single range covering the configured retention window.*\n\n###### Watch for:\n*Many fragmented ranges indicate gaps in stored history from missed or failed fetches.*\n\n###### Keywords:\n- **Complete ledger ranges** *(per node)* — the contiguous spans of ledgers the node holds locally; one unbroken range is ideal.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerCompleteLedgersGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#complete-ledger-ranges)", "fieldConfig": { "defaults": { "custom": { @@ -4303,7 +4303,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*How many historical ledgers the node is back-filling per minute.*\n\n###### How it's computed:\n*Current value of the historical-fetch-per-minute gauge.*\n\n###### Reading it:\n*Near zero once history is complete; elevated while back-filling.*\n\n###### Healthy range:\n*Close to zero in steady state.*\n\n###### Watch for:\n*A sustained high rate means the node is still filling gaps in its stored history.*\n\n###### Keywords:\n- **Historical fetch rate** *(per node)* \u2014 how many historical ledgers the node is back-filling per minute.\n- **Back-fill / catch-up** *(per node)* \u2014 fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n- **Complete ledger ranges** *(per node)* \u2014 the contiguous spans of ledgers the node holds locally; one unbroken range is ideal.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerDbMetricsGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#historical-fetch-rate)", + "description": "###### What this is:\n*How many historical ledgers the node is back-filling per minute.*\n\n###### How it's computed:\n*Current value of the historical-fetch-per-minute gauge.*\n\n###### Reading it:\n*Near zero once history is complete; elevated while back-filling.*\n\n###### Healthy range:\n*Close to zero in steady state.*\n\n###### Watch for:\n*A sustained high rate means the node is still filling gaps in its stored history.*\n\n###### Keywords:\n- **Historical fetch rate** *(per node)* — how many historical ledgers the node is back-filling per minute.\n- **Back-fill / catch-up** *(per node)* — fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n- **Complete ledger ranges** *(per node)* — the contiguous spans of ledgers the node holds locally; one unbroken range is ideal.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerDbMetricsGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#historical-fetch-rate)", "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", @@ -4369,7 +4369,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Cumulative count of peers disconnected for exceeding resource limits.*\n\n###### How it's computed:\n*Current value of the resource-disconnect gauge, plotted over time.*\n\n###### Reading it:\n*A flat line is healthy; steps up mean peers were dropped for overuse.*\n\n###### Healthy range:\n*Flat or very slowly rising.*\n\n###### Watch for:\n*A rising line indicates peers are being throttled off, consistent with abusive or misbehaving peers.*\n\n###### Keywords:\n- **Resource disconnect** *(per node)* \u2014 a peer dropped for exceeding resource/load limits \u2014 the node shedding abusive or overactive peers.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#resource-disconnect)", + "description": "###### What this is:\n*Cumulative count of peers disconnected for exceeding resource limits.*\n\n###### How it's computed:\n*Current value of the resource-disconnect gauge, plotted over time.*\n\n###### Reading it:\n*A flat line is healthy; steps up mean peers were dropped for overuse.*\n\n###### Healthy range:\n*Flat or very slowly rising.*\n\n###### Watch for:\n*A rising line indicates peers are being throttled off, consistent with abusive or misbehaving peers.*\n\n###### Keywords:\n- **Resource disconnect** *(per node)* — a peer dropped for exceeding resource/load limits — the node shedding abusive or overactive peers.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#resource-disconnect)", "fieldConfig": { "defaults": { "color": { @@ -4484,7 +4484,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*The node's local load-based fee factor that scales its minimum transaction cost; baseline 256 at idle.*\n\n###### How it's computed:\n*Current value of the local load-fee economy gauge.*\n\n###### Reading it:\n*Steady at the baseline (256) is normal; higher values mean the node is raising its fee in response to load.*\n\n###### Healthy range:\n*Around 256 (the normal baseline) when idle.*\n\n###### Watch for:\n*A climbing factor, which indicates the node is under transaction load pressure.*\n\n###### Keywords:\n- **Load factor** *(per node)* \u2014 a unitless multiplier (1.0 = no load) that scales the base transaction cost as the node comes under load.\n- **Base fee** *(network-wide)* \u2014 the baseline transaction cost for a reference transaction under minimum load, in drops.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerEconomyGauge`\n\n###### References:\n[Load factor](https://xrpl.org/docs/concepts/transactions/transaction-cost#local-load-cost) \u00b7 [Base fee](https://xrpl.org/docs/concepts/transactions/transaction-cost) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#load-factor)", + "description": "###### What this is:\n*The node's local load-based fee factor that scales its minimum transaction cost; baseline 256 at idle.*\n\n###### How it's computed:\n*Current value of the local load-fee economy gauge.*\n\n###### Reading it:\n*Steady at the baseline (256) is normal; higher values mean the node is raising its fee in response to load.*\n\n###### Healthy range:\n*Around 256 (the normal baseline) when idle.*\n\n###### Watch for:\n*A climbing factor, which indicates the node is under transaction load pressure.*\n\n###### Keywords:\n- **Load factor** *(per node)* — a unitless multiplier (1.0 = no load) that scales the base transaction cost as the node comes under load.\n- **Base fee** *(network-wide)* — the baseline transaction cost for a reference transaction under minimum load, in drops.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerEconomyGauge`\n\n###### References:\n[Load factor](https://xrpl.org/docs/concepts/transactions/transaction-cost#local-load-cost) · [Base fee](https://xrpl.org/docs/concepts/transactions/transaction-cost) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#load-factor)", "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", @@ -4550,7 +4550,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*The minimum XRP balance required to keep an account on the ledger, in drops.*\n\n###### How it's computed:\n*Current value of the reserve-base economy gauge.*\n\n###### Reading it:\n*A stable value is expected; it changes only via network amendment or vote.*\n\n###### Healthy range:\n*The configured network reserve base.*\n\n###### Watch for:\n*An unexpected change outside a known amendment or fee vote.*\n\n###### Keywords:\n- **drops** *(network-wide)* \u2014 the smallest XRP unit \u2014 one drop is 0.000001 XRP (one millionth).\n- **Reserve (base & owner)** *(network-wide)* \u2014 the minimum XRP an account must hold \u2014 a base reserve plus an increment per owned ledger object.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerEconomyGauge`\n\n###### References:\n[drops](https://xrpl.org/docs/references/protocol/data-types/basic-data-types#specifying-currency-amounts) \u00b7 [Reserve (base & owner)](https://xrpl.org/docs/concepts/accounts/reserves) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#drops)", + "description": "###### What this is:\n*The minimum XRP balance required to keep an account on the ledger, in drops.*\n\n###### How it's computed:\n*Current value of the reserve-base economy gauge.*\n\n###### Reading it:\n*A stable value is expected; it changes only via network amendment or vote.*\n\n###### Healthy range:\n*The configured network reserve base.*\n\n###### Watch for:\n*An unexpected change outside a known amendment or fee vote.*\n\n###### Keywords:\n- **drops** *(network-wide)* — the smallest XRP unit — one drop is 0.000001 XRP (one millionth).\n- **Reserve (base & owner)** *(network-wide)* — the minimum XRP an account must hold — a base reserve plus an increment per owned ledger object.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerEconomyGauge`\n\n###### References:\n[drops](https://xrpl.org/docs/references/protocol/data-types/basic-data-types#specifying-currency-amounts) · [Reserve (base & owner)](https://xrpl.org/docs/concepts/accounts/reserves) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#drops)", "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", @@ -4616,7 +4616,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*The additional XRP reserve required per owned ledger object, in drops.*\n\n###### How it's computed:\n*Current value of the reserve-increment economy gauge.*\n\n###### Reading it:\n*A stable value is expected; it changes only via network amendment or vote.*\n\n###### Healthy range:\n*The configured network owner reserve increment.*\n\n###### Watch for:\n*An unexpected change outside a known amendment or fee vote.*\n\n###### Keywords:\n- **Reserve (base & owner)** *(network-wide)* \u2014 the minimum XRP an account must hold \u2014 a base reserve plus an increment per owned ledger object.\n- **drops** *(network-wide)* \u2014 the smallest XRP unit \u2014 one drop is 0.000001 XRP (one millionth).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerEconomyGauge`\n\n###### References:\n[Reserve (base & owner)](https://xrpl.org/docs/concepts/accounts/reserves) \u00b7 [drops](https://xrpl.org/docs/references/protocol/data-types/basic-data-types#specifying-currency-amounts) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#reserve-base-owner)", + "description": "###### What this is:\n*The additional XRP reserve required per owned ledger object, in drops.*\n\n###### How it's computed:\n*Current value of the reserve-increment economy gauge.*\n\n###### Reading it:\n*A stable value is expected; it changes only via network amendment or vote.*\n\n###### Healthy range:\n*The configured network owner reserve increment.*\n\n###### Watch for:\n*An unexpected change outside a known amendment or fee vote.*\n\n###### Keywords:\n- **Reserve (base & owner)** *(network-wide)* — the minimum XRP an account must hold — a base reserve plus an increment per owned ledger object.\n- **drops** *(network-wide)* — the smallest XRP unit — one drop is 0.000001 XRP (one millionth).\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerEconomyGauge`\n\n###### References:\n[Reserve (base & owner)](https://xrpl.org/docs/concepts/accounts/reserves) · [drops](https://xrpl.org/docs/references/protocol/data-types/basic-data-types#specifying-currency-amounts) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#reserve-base-owner)", "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", @@ -4682,7 +4682,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Seconds since the last validated ledger closed, plotted over time.*\n\n###### How it's computed:\n*Current value of the ledger-age economy gauge, sampled each interval.*\n\n###### Reading it:\n*Lower is better; it should stay near the ledger close interval. Mirrors the Validated Ledger Age panel.*\n\n###### Healthy range:\n*Under about 10 seconds.*\n\n###### Watch for:\n*Growth beyond the expected close interval, meaning the node is not keeping up with validated ledgers.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* \u2014 a ledger confirmed final by the trusted validator quorum; its contents never change.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerEconomyGauge`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)", + "description": "###### What this is:\n*Seconds since the last validated ledger closed, plotted over time.*\n\n###### How it's computed:\n*Current value of the ledger-age economy gauge, sampled each interval.*\n\n###### Reading it:\n*Lower is better; it should stay near the ledger close interval. Mirrors the Validated Ledger Age panel.*\n\n###### Healthy range:\n*Under about 10 seconds.*\n\n###### Watch for:\n*Growth beyond the expected close interval, meaning the node is not keeping up with validated ledgers.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* — a ledger confirmed final by the trusted validator quorum; its contents never change.\n- **Ledger close interval** *(network-wide)* — the network's steady ledger rhythm — roughly one closed ledger every 3-5 seconds on Mainnet.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerEconomyGauge`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) · [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)", "fieldConfig": { "defaults": { "color": { @@ -4789,7 +4789,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*The network transaction throughput reported by the ledger economy metrics.*\n\n###### How it's computed:\n*Current value of the transaction-rate economy gauge, plotted over time.*\n\n###### Reading it:\n*Reflects how many transactions are being processed; higher means busier.*\n\n###### Healthy range:\n*Workload-dependent.*\n\n###### Watch for:\n*A sudden sustained surge can indicate a transaction flood; a drop to zero can indicate the node stopped processing.*\n\n###### Keywords:\n- **Transaction type** *(network-wide)* \u2014 the kind of transaction (Payment, OfferCreate, TrustSet, AMM*, NFToken*, etc.), used as a breakdown dimension.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerEconomyGauge`\n\n###### References:\n[Transaction type](https://xrpl.org/docs/references/protocol/transactions/types) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-type)", + "description": "###### What this is:\n*The network transaction throughput reported by the ledger economy metrics.*\n\n###### How it's computed:\n*Current value of the transaction-rate economy gauge, plotted over time.*\n\n###### Reading it:\n*Reflects how many transactions are being processed; higher means busier.*\n\n###### Healthy range:\n*Workload-dependent.*\n\n###### Watch for:\n*A sudden sustained surge can indicate a transaction flood; a drop to zero can indicate the node stopped processing.*\n\n###### Keywords:\n- **Transaction type** *(network-wide)* — the kind of transaction (Payment, OfferCreate, TrustSet, AMM*, NFToken*, etc.), used as a breakdown dimension.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerEconomyGauge`\n\n###### References:\n[Transaction type](https://xrpl.org/docs/references/protocol/transactions/types) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-type)", "fieldConfig": { "defaults": { "color": { @@ -4892,7 +4892,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Time to fetch a missing ledger from peers, at the 95th percentile.*\n\n###### How it's computed:\n*Per-acquire durations aggregated to their 95th percentile per node over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; populated mainly during sync or back-fill.*\n\n###### Healthy range:\n*Low when synced; higher and more active while catching up.*\n\n###### Watch for:\n*A spike signals the node is falling behind or recovering from a fork.*\n\n###### Keywords:\n- **Ledger acquire (inbound fetch)** *(per node)* \u2014 fetching a specific missing ledger from peers; tracked by duration and outcome (complete/failed).\n- **Back-fill / catch-up** *(per node)* \u2014 fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n- **Fork** *(network-wide)* \u2014 when nodes validate divergent ledger chains instead of a single agreed history.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::init`\n\n###### References:\n[Fork](https://xrpl.org/docs/concepts/consensus-protocol) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-acquire-inbound-fetch)", + "description": "###### What this is:\n*Time to fetch a missing ledger from peers, at the 95th percentile.*\n\n###### How it's computed:\n*Per-acquire durations aggregated to their 95th percentile per node over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; populated mainly during sync or back-fill.*\n\n###### Healthy range:\n*Low when synced; higher and more active while catching up.*\n\n###### Watch for:\n*A spike signals the node is falling behind or recovering from a fork.*\n\n###### Keywords:\n- **Ledger acquire (inbound fetch)** *(per node)* — fetching a specific missing ledger from peers; tracked by duration and outcome (complete/failed).\n- **Back-fill / catch-up** *(per node)* — fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n- **Fork** *(network-wide)* — when nodes validate divergent ledger chains instead of a single agreed history.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::init`\n\n###### References:\n[Fork](https://xrpl.org/docs/concepts/consensus-protocol) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-acquire-inbound-fetch)", "fieldConfig": { "defaults": { "color": { @@ -4995,7 +4995,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate of completed ledger fetches split by outcome (complete or failed).*\n\n###### How it's computed:\n*Per-second rate of finished acquisitions grouped by outcome, per node, over a 5-minute window.*\n\n###### Reading it:\n*Complete should dominate; the failed line should stay near zero.*\n\n###### Healthy range:\n*Complete tracking fetch demand, failed near zero.*\n\n###### Watch for:\n*A rising failed rate means the node cannot fetch needed ledgers from its peers.*\n\n###### Keywords:\n- **Ledger acquire (inbound fetch)** *(per node)* \u2014 fetching a specific missing ledger from peers; tracked by duration and outcome (complete/failed).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::init`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-acquire-inbound-fetch)", + "description": "###### What this is:\n*Rate of completed ledger fetches split by outcome (complete or failed).*\n\n###### How it's computed:\n*Per-second rate of finished acquisitions grouped by outcome, per node, over a 5-minute window.*\n\n###### Reading it:\n*Complete should dominate; the failed line should stay near zero.*\n\n###### Healthy range:\n*Complete tracking fetch demand, failed near zero.*\n\n###### Watch for:\n*A rising failed rate means the node cannot fetch needed ledgers from its peers.*\n\n###### Keywords:\n- **Ledger acquire (inbound fetch)** *(per node)* — fetching a specific missing ledger from peers; tracked by duration and outcome (complete/failed).\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::init`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-acquire-inbound-fetch)", "fieldConfig": { "defaults": { "color": { @@ -5092,6 +5092,114 @@ ], "title": "Ledger Acquire Rate by Outcome", "type": "timeseries" + }, + { + "title": "Job Queue Concurrency Limits", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 342 + }, + "collapsed": false, + "panels": [] + }, + { + "title": "Job Queue Saturation (Running vs Limit)", + "description": "###### What this is:\n*How close each concurrency-capped job type is to its ceiling. JobQueue enforces a per-type limit on how many jobs of that type may run at once, and the tight ones carry ledger-sync traffic: makeFetchPack 1, ledgerRequest 3, ledgerData 3, updatePaths 1, fetchTxnData 5. A type at its ceiling cannot start more work no matter how many workers are idle, so this is a different kind of limit from CPU or disk.*\n\n###### How it's computed:\n*Each jobq__running gauge divided by that type's own limit from JobTypes.h, so every line shares one 0-to-1 axis. 1.0 means running equals the limit. Multiply a reading by the limit shown in its legend to recover the raw job count. JobQueue::collect snapshots all three per-type counters under the queue's own lock and publishes them after releasing it, on the 1-second export cycle.*\n\n###### Reading it:\n*Read the distance to 1.0, not the absolute height. Below 1.0 the type has spare slots and its queue wait is not the limit's fault. Touching 1.0 briefly is normal work. Sitting at 1.0 means the type is pinned at its ceiling and every further job of that type is being deferred rather than started, which is what turns into queue wait downstream. Because the limits differ, a raw count of 3 is saturation for ledgerRequest but only 60 percent for fetchTxnData; normalizing is what makes the lines comparable.*\n\n###### Healthy range:\n*Below 1.0, with brief touches under load.*\n\n###### Watch for:\n*A line flat at 1.0: that type is the binding constraint. ledgerRequest pinned means the 3 slots shared by RcvGetLedger and RcvGetObjByHash are full, so peer ledger and object requests are queueing behind each other; the Ledger Data and Sync dashboard splits that wait by handler and shows the matching deferred depth. ledgerData or fetchTxnData pinned means inbound ledger data cannot be absorbed and validated ledger age will grow. makeFetchPack or updatePaths pinned at their limit of 1 means a single long job is blocking the whole type. These are sampled gauges, so a line that never reaches 1.0 is not proof the type was never momentarily saturated.*\n\n###### Keywords:\n- **Job queue / job type** *(per node)* — xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n- **Concurrency limit** *(per node)* — the cap on how many jobs of one type may run at once; a type at its cap cannot start more work.\n- **Deferred job** *(per node)* — a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[core/JobQueue.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)\n\n###### Function:\n`JobQueue::getNextJob (limit enforcement) / JobQueue::collect (publication)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#concurrency-limit)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 343 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(label_join(label_replace(jobq_makefetchpack_running{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"} / 1, \"series\", \"makeFetchPack (Limit 1)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(label_join(label_replace(jobq_ledgerrequest_running{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"} / 3, \"series\", \"ledgerRequest (Limit 3)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(label_join(label_replace(jobq_ledgerdata_running{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"} / 3, \"series\", \"ledgerData (Limit 3)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(label_join(label_replace(jobq_updatepaths_running{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"} / 1, \"series\", \"updatePaths (Limit 1)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(label_join(label_replace(jobq_fetchtxndata_running{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"} / 5, \"series\", \"fetchTxnData (Limit 5)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "percentunit", + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "custom": { + "axisLabel": "Running / Limit", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3, + "thresholdsStyle": { + "mode": "line" + } + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json index dd29339e57..6f21454787 100644 --- a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json +++ b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json @@ -32,7 +32,7 @@ } ] }, - "description": "What this shows: Fine-grained breakdown of peer-to-peer overlay traffic beyond the main network view: squelch relay control, protocol overhead, validator-list distribution, transaction-set exchange, transaction availability, ledger-proof and replay traffic, and unclassified messages. \u2014 Use it to: Drill into individual overlay message categories to diagnose relay efficiency, overhead, and catch-up traffic.", + "description": "What this shows: Fine-grained breakdown of peer-to-peer overlay traffic beyond the main network view: squelch relay control, protocol overhead, validator-list distribution, transaction-set exchange, transaction availability, ledger-proof and replay traffic, and unclassified messages. — Use it to: Drill into individual overlay message categories to diagnose relay efficiency, overhead, and catch-up traffic.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -41,7 +41,7 @@ "panels": [ { "title": "Squelch Traffic (Messages)", - "description": "###### What this is:\n*Squelch relay-control messages in/out, plus messages suppressed by squelch and squelch directives that were ignored. Squelch reduces redundant message forwarding between peers.*\n\n###### How it's computed:\n*Per-second message rate for the squelch, squelch-suppressed, and squelch-ignored categories, in and out.*\n\n###### Reading it:\n*High suppressed counts mean squelch is saving bandwidth; ignored should stay low.*\n\n###### Healthy range:\n*workload-dependent; suppressed far above ignored.*\n\n###### Watch for:\n*High ignored counts (peers not honoring squelch) or squelch traffic itself dominating.*\n\n###### Keywords:\n- **Transaction suppression** *(per node)* \u2014 dropping a transaction already seen from another peer, so it is not reprocessed.\n- **Squelch** *(per node)* \u2014 control messages that tell a peer to stop forwarding a given validator's messages, cutting redundancy.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-suppression)", + "description": "###### What this is:\n*Squelch relay-control messages in/out, plus messages suppressed by squelch and squelch directives that were ignored. Squelch reduces redundant message forwarding between peers.*\n\n###### How it's computed:\n*Per-second message rate for the squelch, squelch-suppressed, and squelch-ignored categories, in and out.*\n\n###### Reading it:\n*High suppressed counts mean squelch is saving bandwidth; ignored should stay low.*\n\n###### Healthy range:\n*workload-dependent; suppressed far above ignored.*\n\n###### Watch for:\n*High ignored counts (peers not honoring squelch) or squelch traffic itself dominating.*\n\n###### Keywords:\n- **Transaction suppression** *(per node)* — dropping a transaction already seen from another peer, so it is not reprocessed.\n- **Squelch** *(per node)* — control messages that tell a peer to stop forwarding a given validator's messages, cutting redundancy.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-suppression)", "type": "timeseries", "gridPos": { "h": 8, @@ -117,7 +117,7 @@ }, { "title": "Overhead Traffic Breakdown (Bytes)", - "description": "###### What this is:\n*Overlay protocol overhead bytes split into base overhead, intra-cluster overhead, and validator-manifest distribution overhead.*\n\n###### How it's computed:\n*Per-second in/out byte rate for the overhead, overhead-cluster, and overhead-manifest categories.*\n\n###### Reading it:\n*Base overhead is routine; cluster and manifest rise around cluster syncs and manifest changes.*\n\n###### Healthy range:\n*workload-dependent; low and stable.*\n\n###### Watch for:\n*Sustained high cluster or manifest overhead (frequent cluster state churn or manifest reissue).*\n\n###### Keywords:\n- **Overlay** *(per node)* \u2014 the peer-to-peer network layer over which nodes exchange transactions, proposals, and validations.\n- **Manifest** *(network-wide)* \u2014 a signed record binding a validator's rotating signing key to its stable master key.\n- **Cluster** *(cluster-wide)* \u2014 a group of trusted co-operated nodes that share load information and skip some verification.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Overlay](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) \u00b7 [Cluster](https://xrpl.org/docs/concepts/networks-and-servers/clustering) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay)", + "description": "###### What this is:\n*Overlay protocol overhead bytes split into base overhead, intra-cluster overhead, and validator-manifest distribution overhead.*\n\n###### How it's computed:\n*Per-second in/out byte rate for the overhead, overhead-cluster, and overhead-manifest categories.*\n\n###### Reading it:\n*Base overhead is routine; cluster and manifest rise around cluster syncs and manifest changes.*\n\n###### Healthy range:\n*workload-dependent; low and stable.*\n\n###### Watch for:\n*Sustained high cluster or manifest overhead (frequent cluster state churn or manifest reissue).*\n\n###### Keywords:\n- **Overlay** *(per node)* — the peer-to-peer network layer over which nodes exchange transactions, proposals, and validations.\n- **Manifest** *(network-wide)* — a signed record binding a validator's rotating signing key to its stable master key.\n- **Cluster** *(cluster-wide)* — a group of trusted co-operated nodes that share load information and skip some verification.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Overlay](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) · [Cluster](https://xrpl.org/docs/concepts/networks-and-servers/clustering) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay)", "type": "timeseries", "gridPos": { "h": 8, @@ -193,7 +193,7 @@ }, { "title": "Validator List Traffic", - "description": "###### What this is:\n*Bytes and messages exchanged distributing validator lists (trusted-list configuration) between peers.*\n\n###### How it's computed:\n*Per-second in/out byte and message rate for the validator-lists category.*\n\n###### Reading it:\n*Bursts when lists update or new peers connect; quiet otherwise.*\n\n###### Healthy range:\n*workload-dependent; occasional bursts.*\n\n###### Watch for:\n*Continuous high volume (repeated list re-fetching or churn).*\n\n###### Keywords:\n- **Validator list** *(network-wide)* \u2014 signed lists of recommended validators (UNLs) that peers distribute to each other.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Validator list](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validator-list)", + "description": "###### What this is:\n*Bytes and messages exchanged distributing validator lists (trusted-list configuration) between peers.*\n\n###### How it's computed:\n*Per-second in/out byte and message rate for the validator-lists category.*\n\n###### Reading it:\n*Bursts when lists update or new peers connect; quiet otherwise.*\n\n###### Healthy range:\n*workload-dependent; occasional bursts.*\n\n###### Watch for:\n*Continuous high volume (repeated list re-fetching or churn).*\n\n###### Keywords:\n- **Validator list** *(network-wide)* — signed lists of recommended validators (UNLs) that peers distribute to each other.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Validator list](https://xrpl.org/docs/concepts/consensus-protocol/unl) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validator-list)", "type": "timeseries", "gridPos": { "h": 8, @@ -272,7 +272,7 @@ }, { "title": "Set Get/Share Traffic (Bytes)", - "description": "###### What this is:\n*Transaction-set fetch (get) and share bytes exchanged during ledger close.*\n\n###### How it's computed:\n*Per-second in/out byte rate for the set-get and set-share categories.*\n\n###### Reading it:\n*Some exchange each ledger is normal as peers reconcile transaction sets.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High set-get (peers frequently missing transaction sets: possible sync delays).*\n\n###### Keywords:\n- **Set get/share** *(per node)* \u2014 exchange of candidate transaction sets between peers as they reconcile during a ledger close.\n- **Ledger close** *(network event)* \u2014 the current open ledger is closed and a new closed ledger is built from the agreed transaction set.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Ledger close](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#set-get-share)", + "description": "###### What this is:\n*Transaction-set fetch (get) and share bytes exchanged during ledger close.*\n\n###### How it's computed:\n*Per-second in/out byte rate for the set-get and set-share categories.*\n\n###### Reading it:\n*Some exchange each ledger is normal as peers reconcile transaction sets.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High set-get (peers frequently missing transaction sets: possible sync delays).*\n\n###### Keywords:\n- **Set get/share** *(per node)* — exchange of candidate transaction sets between peers as they reconcile during a ledger close.\n- **Ledger close** *(network event)* — the current open ledger is closed and a new closed ledger is built from the agreed transaction set.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Ledger close](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#set-get-share)", "type": "timeseries", "gridPos": { "h": 8, @@ -334,7 +334,7 @@ }, { "title": "Have/Requested Transactions (Messages)", - "description": "###### What this is:\n*Transaction-availability messages: advertisements that a peer has certain transactions, and explicit requests for transaction data.*\n\n###### How it's computed:\n*Per-second in/out message rate for the have-transactions and requested-transactions categories.*\n\n###### Reading it:\n*Compare requested versus have to gauge how well transactions are propagating.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*Requested far exceeding have (peers behind on transaction propagation).*\n\n###### Keywords:\n- **Have / requested transactions** *(per node)* \u2014 advertisements that a peer holds certain transactions, and explicit requests for transaction data.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#have-requested-transactions)", + "description": "###### What this is:\n*Transaction-availability messages: advertisements that a peer has certain transactions, and explicit requests for transaction data.*\n\n###### How it's computed:\n*Per-second in/out message rate for the have-transactions and requested-transactions categories.*\n\n###### Reading it:\n*Compare requested versus have to gauge how well transactions are propagating.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*Requested far exceeding have (peers behind on transaction propagation).*\n\n###### Keywords:\n- **Have / requested transactions** *(per node)* — advertisements that a peer holds certain transactions, and explicit requests for transaction data.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#have-requested-transactions)", "type": "timeseries", "gridPos": { "h": 8, @@ -396,7 +396,7 @@ }, { "title": "Unknown / Unclassified Traffic", - "description": "###### What this is:\n*Overlay traffic that matches no known message category, in bytes and messages.*\n\n###### How it's computed:\n*Current in/out byte and message counts for the unknown category.*\n\n###### Reading it:\n*Should be at or near zero.*\n\n###### Healthy range:\n*zero.*\n\n###### Watch for:\n*Any sustained non-zero value (protocol version mismatch, corrupted messages, or an unclassified new message type).*\n\n###### Keywords:\n- **Overlay** *(per node)* \u2014 the peer-to-peer network layer over which nodes exchange transactions, proposals, and validations.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Overlay](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay)", + "description": "###### What this is:\n*Overlay traffic that matches no known message category, in bytes and messages.*\n\n###### How it's computed:\n*Current in/out byte and message counts for the unknown category.*\n\n###### Reading it:\n*Should be at or near zero.*\n\n###### Healthy range:\n*zero.*\n\n###### Watch for:\n*Any sustained non-zero value (protocol version mismatch, corrupted messages, or an unclassified new message type).*\n\n###### Keywords:\n- **Overlay** *(per node)* — the peer-to-peer network layer over which nodes exchange transactions, proposals, and validations.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Overlay](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay)", "type": "timeseries", "gridPos": { "h": 8, @@ -475,7 +475,7 @@ }, { "title": "Proof Path Traffic", - "description": "###### What this is:\n*Proof-path request/response bytes used to verify individual ledger entries without downloading the whole ledger.*\n\n###### How it's computed:\n*Per-second in/out byte rate for the proof-path request and response categories.*\n\n###### Reading it:\n*Rises when peers verify specific state, often during catch-up.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High sustained request volume (heavy state-verification load).*\n\n###### Keywords:\n- **Path request / discovery** *(per node)* \u2014 a client's ongoing pathfinding subscription (request) and the periodic path-refresh passes (discovery).\n- **Proof path** *(per node)* \u2014 messages that prove a single ledger entry exists without transferring the whole ledger.\n- **Back-fill / catch-up** *(per node)* \u2014 fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Path request / discovery](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods/path-and-order-book-methods/path_find) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#path-request-discovery)", + "description": "###### What this is:\n*Proof-path request/response bytes used to verify individual ledger entries without downloading the whole ledger.*\n\n###### How it's computed:\n*Per-second in/out byte rate for the proof-path request and response categories.*\n\n###### Reading it:\n*Rises when peers verify specific state, often during catch-up.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High sustained request volume (heavy state-verification load).*\n\n###### Keywords:\n- **Path request / discovery** *(per node)* — a client's ongoing pathfinding subscription (request) and the periodic path-refresh passes (discovery).\n- **Proof path** *(per node)* — messages that prove a single ledger entry exists without transferring the whole ledger.\n- **Back-fill / catch-up** *(per node)* — fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Path request / discovery](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods/path-and-order-book-methods/path_find) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#path-request-discovery)", "type": "timeseries", "gridPos": { "h": 8, @@ -537,7 +537,7 @@ }, { "title": "Replay Delta Traffic", - "description": "###### What this is:\n*Replay-delta request/response bytes used to efficiently replay ledger state changes during catch-up.*\n\n###### How it's computed:\n*Per-second in/out byte rate for the replay-delta request and response categories.*\n\n###### Reading it:\n*Active during catch-up and replay; quiet when synced.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*Continuous replay traffic (node repeatedly replaying rather than staying current).*\n\n###### Keywords:\n- **Replay delta** *(per node)* \u2014 messages carrying just the changes between ledgers, to replay state efficiently during catch-up.\n- **Back-fill / catch-up** *(per node)* \u2014 fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#replay-delta)", + "description": "###### What this is:\n*Replay-delta request/response bytes used to efficiently replay ledger state changes during catch-up.*\n\n###### How it's computed:\n*Per-second in/out byte rate for the replay-delta request and response categories.*\n\n###### Reading it:\n*Active during catch-up and replay; quiet when synced.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*Continuous replay traffic (node repeatedly replaying rather than staying current).*\n\n###### Keywords:\n- **Replay delta** *(per node)* — messages carrying just the changes between ledgers, to replay state efficiently during catch-up.\n- **Back-fill / catch-up** *(per node)* — fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#replay-delta)", "type": "timeseries", "gridPos": { "h": 8, @@ -596,6 +596,252 @@ }, "overrides": [] } + }, + { + "title": "GetObject Handler (TMGetObjectByHash)", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "collapsed": false, + "panels": [] + }, + { + "title": "GetObject Handler Latency Breakdown", + "description": "###### What this is:\n*The three additive parts of TMGetObjectByHash service time, drawn on one axis so the expensive part names itself. Queue Wait is how long the RcvGetObjByHash job sat queued before a worker took it; its job type, ledgerRequest, allows only 3 to run at once. Handler Total is the whole job body once running. NodeStore Lookup is only the fetch loop inside that body. End-to-end service time is Queue Wait plus Handler Total, and Handler Total itself splits into NodeStore Lookup plus everything else.*\n\n###### How it's computed:\n*p99 of job_queued_us and job_running_us, both filtered to handler=\"RcvGetObjByHash\", plus p99 of getobject_lookup_us. Each is histogram_quantile over the microsecond bucket series, summed by le so the quantile is computed across the whole bucket set. The handler label is the sanitized addJob name, so RcvGetObjByHash is separated from RcvGetLedger even though both are job type ledgerRequest.*\n\n###### Reading it:\n*Read it as a subtraction, not as three independent lines. The timed fetch loop covers both the NodeStore fetches and the copying of each returned object into the reply, so the vertical gap between Handler Total and NodeStore Lookup is what happens after the loop: serializing the reply message, plus computing the charge and recording the metrics. So: Queue Wait tall with Handler Total flat means queue contention and the work itself is fine. Handler Total tracking NodeStore Lookup closely means storage is the bottleneck. Handler Total well above NodeStore Lookup means the cost has moved out of the fetch loop into reply serialization.*\n\n###### Healthy range:\n*All three sub-millisecond while peers ask for the handful of objects the sync path produces; workload-dependent above that.*\n\n###### Watch for:\n*Queue Wait climbing while the other two stay flat: the ledgerRequest queue is saturated, so cross-check Job Queue Backlog and Deferred by Type and LedgerReq Wait by Handler on the Ledger Data and Sync dashboard to see which producer is starving it. A widening Handler Total minus NodeStore Lookup gap: reply serialization regressed. NodeStore Lookup rising on its own: check getobject_lookups_total misses and the NuDB panels.*\n\n###### Keywords:\n- **GetObject / object fetch** *(per node)* — peer requests that fetch individual pieces of ledger data by hash, such as tree nodes or transactions.\n- **Handler label** *(per node)* — the addJob call-site name attached to job metrics, so producers sharing one job type stay separable.\n- **NodeStore lookup (hit / miss)** *(per node)* — one object-store fetch by hash; a hit is usually served from cache, a miss does a disk seek.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::recordGetObjectMetrics (lookup) / MetricsRegistry::recordJobStarted, recordJobFinished (queue, total)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#getobject-object-fetch)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 33 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{handler=\"RcvGetObjByHash\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"Queue Wait p99\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{handler=\"RcvGetObjByHash\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"Handler Total p99\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(getobject_lookup_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"NodeStore Lookup p99\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "µs", + "custom": { + "axisLabel": "Duration (μs)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3, + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 5 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + }, + { + "title": "GetObject Request Size Distribution", + "description": "###### What this is:\n*How many objects peers ask for per TMGetObjectByHash message, as a full distribution rather than an average. This characterizes the request that caused any latency seen in the breakdown panel: large batches make the work genuinely large, which is a different problem from the same work becoming slower.*\n\n###### How it's computed:\n*Counts of requests falling in each object-count band per 5-minute window, from the getobject_request_objects bucket series, drawn as color density.*\n\n###### Reading it:\n*A tight band at the bottom is honest traffic: the inbound-ledger acquire path asks for at most 4 hashes of one object type per message. Bands above 64 and above 1024 are the medium and large pricing bands, so mass there means the size surcharge is being applied. A hot cell in the top row is the overflow bucket and means requests larger than the top bucket boundary.*\n\n###### Healthy range:\n*Nearly all mass in the lowest bands (8 objects or fewer per request).*\n\n###### Watch for:\n*Mass appearing in the high bands, especially a persistent hot row near the top: a peer is batching thousands of hashes per message, which is what the differential pricing exists to charge for. Confirm with GetObject Charge Distribution and GetObject Rejections. Buckets are explicit (1,2,4,8,16,64,256,1024,4096,12288) and reach the handler's hard cap, so the top row is real traffic at the cap, not a measurement ceiling.*\n\n###### Keywords:\n- **GetObject / object fetch** *(per node)* — peer requests that fetch individual pieces of ledger data by hash, such as tree nodes or transactions.\n- **Resource charge** *(per node)* — the load cost the resource manager bills a peer per request; crossing the warning then drop threshold sheds the peer.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::recordGetObjectMetrics`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#getobject-object-fetch)", + "type": "heatmap", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 41 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "yAxis": { + "axisLabel": "Objects Per Request", + "unit": "short" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "sum(increase(getobject_request_objects_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m])) by (le)", + "legendFormat": "{{le}}", + "format": "heatmap" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "title": "GetObject Lookups by Result", + "description": "###### What this is:\n*NodeStore lookups performed by the handler, split into hits and misses. A miss does a node-store seek while a hit is usually served from cache, so the hit/miss mix is the reason NodeStore Lookup time moves.*\n\n###### How it's computed:\n*Per-second rate of getobject_lookups_total, grouped by the result label. The counter is advanced once per request with the batch totals -- hits are the objects returned, misses are the rest of the request -- not once per object, so the rate is objects per second rather than requests per second.*\n\n###### Reading it:\n*Use this to explain the NodeStore Lookup line on the breakdown panel. A miss-heavy mix makes that line rise for a real reason: seeks, not a regression. A hit-heavy mix with rising lookup time points at the storage layer instead.*\n\n###### Healthy range:\n*Hits dominating on a warm synced node; misses low and driven by genuine catch-up requests.*\n\n###### Watch for:\n*A sustained miss rate far above the hit rate: a peer is asking for hashes this node does not hold, which is either a peer far out of sync or a client requesting objects this node never stored. Cross-check GetObject Charge Distribution, since misses are billed first and at eight times the hit cost.*\n\n###### Keywords:\n- **NodeStore lookup (hit / miss)** *(per node)* — one object-store fetch by hash; a hit is usually served from cache, a miss does a disk seek.\n- **NodeStore** *(per node)* — the key-value object store holding ledger data (tree nodes), backed by NuDB.\n- **Resource charge** *(per node)* — the load cost the resource manager bills a peer per request; crossing the warning then drop threshold sheds the peer.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::recordGetObjectMetrics`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#nodestore-lookup-hit-miss)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 41 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(label_join(label_replace(sum by (result, service_instance_id, xrpl_branch, xrpl_node_role) (rate(getobject_lookups_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", result=~\"$result\"}[$__rate_interval])), \"series\", \"$1\", \"result\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cps", + "custom": { + "axisLabel": "Lookups / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "GetObject Rejections", + "description": "###### What this is:\n*Requests refused by the message handler before any NodeStore access, split by which gate refused them: oversize means more objects than the handler accepts (its hard cap is 12288), malformed_ledgerhash means the ledger hash was not 32 bytes.*\n\n###### How it's computed:\n*Per-second rate of getobject_rejected_total, grouped by the reason label. Both gates run in onMessage on the generic query path before the job is queued, so a rejection consumes no queue slot and no NodeStore lookup. The fetch-pack and transaction sub-types return earlier and never reach either gate.*\n\n###### Reading it:\n*Any non-zero value is traffic that does not conform to the protocol: the sync path asks for a handful of hashes and always sends a full-size hash. Because the gates fire before the fetch loop, rejections explain why request volume can be high while lookups stay flat.*\n\n###### Healthy range:\n*Zero. No conforming peer produces either rejection, so a flat zero line is the expected reading and is not on its own evidence that the counter is wired -- confirm that from the other GetObject panels, which do move on a healthy node.*\n\n###### Watch for:\n*A rising oversize rate: a peer is sending requests above the accepted object count. Confirm the pricing response on GetObject Charge Distribution, and expect the peer to be shed once its resource balance crosses the drop threshold. A rising malformed rate points at a broken or non-conforming client rather than at load.*\n\n###### Keywords:\n- **GetObject / object fetch** *(per node)* — peer requests that fetch individual pieces of ledger data by hash, such as tree nodes or transactions.\n- **Resource drops / warnings** *(per node)* — the resource manager warning (then dropping/blocking) a peer or client for excessive usage.\n- **Resource charge** *(per node)* — the load cost the resource manager bills a peer per request; crossing the warning then drop threshold sheds the peer.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::onMessage (TMGetObjectByHash)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#getobject-object-fetch)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 49 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(label_join(label_replace(sum by (reason, service_instance_id, xrpl_branch, xrpl_node_role) (rate(getobject_rejected_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", reason=~\"$reason\"}[$__rate_interval])), \"series\", \"$1\", \"reason\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "cps", + "custom": { + "axisLabel": "Rejections / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } + }, + { + "title": "GetObject Charge Distribution", + "description": "###### What this is:\n*The dynamic resource charge applied per TMGetObjectByHash request, as percentiles. This is the differential-pricing component only -- a flat base charge is applied separately when the message is admitted -- so it shows whether cost actually escalates with request size and miss ratio the way the pricing model intends.*\n\n###### How it's computed:\n*p50, p90 and p99 of getobject_charge over the dashboard rate interval, from its bucket series summed by le. The value recorded is the charge that was applied, computed from billable hits, billable misses and the request-size band.*\n\n###### Reading it:\n*p50 sitting at zero is the healthy shape: requests inside the free allowance cost nothing. Movement in p99 while p50 stays at zero means a small number of expensive requests, which is exactly the traffic the model is meant to price. Compare with GetObject Request Size Distribution: charge should rise in steps as requests cross the size-band edges at 64 and 1024, not smoothly.*\n\n###### Healthy range:\n*p50 at zero, p99 low. Requests of 16 objects or fewer carry no dynamic charge by design.*\n\n###### Watch for:\n*p99 climbing steadily: sustained expensive traffic, and the peers producing it should be approaching the resource drop threshold. Buckets are explicit and bracket the resource thresholds (5000 warning, 25000 drop), so p99 crossing 25000 means senders are being shed on a single message. The axis is deliberately unscaled rather than abbreviated, so those two numbers are readable exactly rather than as 5 K and 25 K.*\n\n###### Keywords:\n- **Resource charge** *(per node)* — the load cost the resource manager bills a peer per request; crossing the warning then drop threshold sheds the peer.\n- **Resource drops / warnings** *(per node)* — the resource manager warning (then dropping/blocking) a peer or client for excessive usage.\n- **GetObject / object fetch** *(per node)* — peer requests that fetch individual pieces of ledger data by hash, such as tree nodes or transactions.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::computeGetObjectByHashFee (charge) / PeerImp::recordGetObjectMetrics (recording)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#resource-charge)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 49 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.5, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(getobject_charge_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"Charge p50\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.9, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(getobject_charge_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"Charge p90\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(getobject_charge_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"Charge p99\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "none", + "custom": { + "axisLabel": "Charge (Cost Units)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3, + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 5 + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } } ], "schemaVersion": 39, @@ -754,6 +1000,46 @@ "multi": true, "refresh": 2, "sort": 1 + }, + { + "name": "result", + "label": "Lookup Result", + "description": "Filter GetObject NodeStore lookups by result [hit / miss]", + "type": "query", + "query": "label_values(getobject_lookups_total, result)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "reason", + "label": "Rejection Reason", + "description": "Filter GetObject rejections by gate [oversize / malformed_ledgerhash]", + "type": "query", + "query": "label_values(getobject_rejected_total, reason)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 } ] }, diff --git a/docker/telemetry/workload/expected_metrics.json b/docker/telemetry/workload/expected_metrics.json index b1515692b7..5807d42881 100644 --- a/docker/telemetry/workload/expected_metrics.json +++ b/docker/telemetry/workload/expected_metrics.json @@ -150,10 +150,6 @@ "sync_acquire{metric=\"received_data_depth\"}", "sync_acquire{metric=\"in_flight\"}", "shamap_cache_hit_rate{metric=\"treenode\"}", - "jobq_backlog{metric=\"waiting\",job_type=\"ledgerData\"}", - "jobq_backlog{metric=\"running\",job_type=\"ledgerData\"}", - "jobq_backlog{metric=\"deferred\",job_type=\"ledgerData\"}", - "jobq_backlog{metric=\"deferred\",job_type=\"ledgerRequest\"}", "jobq_saturation{metric=\"running_tasks\"}", "jobq_saturation{metric=\"worker_threads\"}", "jobq_saturation{metric=\"total_waiting\"}", @@ -186,7 +182,7 @@ "consensus_round_duration_ms_count" ], "_acquire_note": "The four sync_acquire sub-series and shamap_cache_hit_rate are unconditional: both are observable gauges whose callbacks observe every series on each collection tick, so each is present even when the value is 0 (an idle node reports in_flight=0 and missing_state_nodes_max=0, and a cold cache reports a 0.0 hit rate). Absence, not a zero, is the regression. The three WP-A3 counters (sync_acquire_source_total, sync_addnode_total, sync_acquire_no_progress_total) are deliberately NOT asserted here: all three are emitted only from InboundLedger, which runs only when a node must fetch a ledger it lacks. expected_spans.json already marks the ledger.acquire span optional for exactly this reason (\"A healthy local cluster rarely back-fills history\"), and the metric validator has no per-metric optional flag, so listing them would fail the harness red on a healthy run. They are covered by exact-value unit tests in src/tests/libxrpl/telemetry/MetricMacros.cpp and by the ledger-sync-health panels; add them here only alongside a harness step that forces a real acquire (e.g. starting a node against an existing ledger history).", - "_jobq_note": "The jobq_backlog and jobq_saturation series are unconditional: both are observable gauges whose callbacks iterate EVERY registered JobType (jobData_ is populated from JobTypes at JobQueue construction) and observe all three fields on each collection tick, so a series exists even when the value is 0. That is why an idle-but-registered type like ledgerData is safe to assert by name here \u2014 a fresh harness node that never defers a single job still reports jobq_backlog{metric=\"deferred\",job_type=\"ledgerData\"} = 0, and absence, not the zero, is the regression. Two job_type values are asserted (ledgerData and ledgerRequest) because they are the sync-critical types capped at concurrency 3 in JobTypes.h, so they are the ones whose deferred series must never silently vanish. Only deferred is asserted for ledgerRequest to keep the list short: the three-field fan-out is already proven by ledgerData. worker_threads is asserted because it is the denominator of the dashboard saturation ratio, and it is always at least 1 (the JobQueue ctor gives standalone mode exactly one worker), so a zero or missing reading there means the accessor regressed rather than the node being idle.", + "_jobq_note": "The three jobq_saturation series are unconditional: it is an observable gauge whose callback observes all three fields on every collection tick, so each series exists even when the value is 0, and absence rather than a zero is the regression. worker_threads is asserted because it is the denominator of the dashboard saturation ratio, and it is always at least 1 (the JobQueue ctor gives standalone mode exactly one worker), so a zero or missing reading there means the accessor regressed rather than the node being idle. The per-job-type waiting/running/deferred counts are published separately by JobQueue::collect() as the beast::insight gauges jobq__waiting / _running / _deferred, which the collector translates; they are covered by the StatsD-derived groups, not here.", "_conditional_note": "handshake_negotiation_fail_total and unl_fetch_total are conditional under the local harness: the first only exists once a handshake is rejected, and the second needs a [validator_list_sites] entry (run-full-validation.sh generates a static [validators] file instead). The validator has no per-metric optional flag, so if either reports 0 series in a harness run, move it out of this group rather than weakening the check.", "_sync_state_note": "The four sync_state sub-series are unconditional: the gauge observes all four on every collection tick, so each is present as a series even when its value is 0 (a node that never reached FULL reports initial_full_duration_us=0, and a healthy node reports server_stall_seconds=0). The check asserts series presence, not a non-zero value, which is exactly right here \u2014 a zero is a meaningful reading for these signals, and absence is the regression. server_stall_events_total is likewise always present because the observable counter reports the tally (0 or more) every tick. state_changes_total is asserted here with a from!=\"\",to!=\"\" selector rather than bare (parity_counters already asserts the bare name): the selector is what proves the WP-A2 {from,to} label dimension actually reached Prometheus, so a regression to the old unlabelled counter fails this check instead of silently passing on the bare name. It needs at least one real mode transition, which any node reaching connected/syncing produces during startup.", "_a7_note": "WP-A7 adds three observable gauges and four counters. The 16 gauge sub-series (peer_ledger_supply, peerfinder_slot_census, amendment_block) are unconditional and asserted individually: each callback in MetricsRegistry.cpp calls observe() for every field on every collection tick with no early return between them, so the series exists whatever the value. That includes the two sentinel readings \u2014 a node whose peers have advertised nothing reports peer_ledger_supply{metric=\"supply_min_seq\"} = 0 meaning unknown, and a node with no pending amendment reports amendment_block{metric=\"seconds_to_block\"} = -1 meaning healthy. Absence, not the sentinel, is the regression. Of the four counters only peer_accept_total is asserted: run-full-validation.sh gives every node a [port_peer] on 0.0.0.0 and lists the other four nodes in [ips], so all 5 nodes dial each other and each one is also dialled, which means OverlayImpl::onHandoff runs and reports outcome=accepted (or slot_refused/no_slot on the duplicate half of each mutual dial) on every node. It is asserted bare rather than with an outcome= selector because which outcome a given node records depends on dial ordering, which the harness does not control. The other three counters are deliberately NOT asserted. peer_disconnect_total is emitted only from PeerImp::close, and a healthy 5-node localhost cluster holds its 4 fixed peers for the whole run: the timer-driven reasons need maxUnknownTime (600 s) or maxDivergedTime (300 s) to elapse (Config.h) while the full-validation profile totals well under that, and the shutdown reasons only fire during teardown, which happens in run-full-validation.sh after Step 5 has already scraped. serve_refused_total needs a peer to ask this node for a ledger, tx set or object it cannot serve \u2014 on a cluster where every node has the same complete history from genesis, getLedger()/getTxSet() succeed and the send queues never approach Tuning::kDropSendQueue. ledger_jump_total needs NetworkOPsImp::switchLastClosedLedger, reached only when consensus reports an LCL this node did not build on; a healthy 5-node cluster agrees every round, so it never jumps. The metric validator has no per-metric optional flag, so listing any of the three would fail the harness red on a healthy run \u2014 the same reasoning _acquire_note applies to the WP-A3 InboundLedger counters. All four counters are covered by exact-value unit tests in src/tests/libxrpl/telemetry/MetricMacros.cpp and rendered by the ledger-sync-health panels Peer Disconnects by Reason, Ledger/Object Serve Refusals and Byzantine Ledger Jumps. To make them assertable the harness would need a fault-injection step: kill one node mid-run and re-scrape before teardown (peer_disconnect_total, reason=read_error/graceful), request a ledger sequence outside the cluster's history or drive a node past its send-queue limit (serve_refused_total), and start a node on a divergent chain tip or partition the cluster and heal it (ledger_jump_total).", diff --git a/docker/telemetry/workload/validate_telemetry.py b/docker/telemetry/workload/validate_telemetry.py index f325bb57b1..a6e1c9524d 100644 --- a/docker/telemetry/workload/validate_telemetry.py +++ b/docker/telemetry/workload/validate_telemetry.py @@ -744,14 +744,13 @@ async def validate_metrics( "sync_acquire", "sync_addnode_total", "shamap_cache_hit_rate", - # JobQueue saturation signals. jobq_backlog carries the - # per-type waiting/running/deferred sub-series and - # jobq_saturation the pool running_tasks/worker_threads/ - # total_waiting; both are asserted. No new prefix entry - # is needed -- the "jobq_" prefix above already matches - # them (it was added for the StatsD jobq_job_count - # gauge), so listing them again would only duplicate - # the diagnostic output. + # JobQueue saturation signals. jobq_saturation carries + # the pool running_tasks/worker_threads/total_waiting + # sub-series and is asserted. No new prefix entry is + # needed -- the "jobq_" prefix above already matches it + # (it was added for the StatsD jobq_job_count gauge), + # so listing it again would only duplicate the + # diagnostic output. ) ) ] diff --git a/docs/telemetry-glossary.md b/docs/telemetry-glossary.md index 0886f93d4f..491aa3da54 100644 --- a/docs/telemetry-glossary.md +++ b/docs/telemetry-glossary.md @@ -17,6 +17,7 @@ documentation. - [Consensus](#cat-consensus) - [Transaction Pipeline](#cat-transaction-pipeline) - [Fees & Queue](#cat-fees-queue) +- [Job Queue](#cat-job-queue) - [Node State & Sync](#cat-node-state-sync) - [Peer & Overlay Networking](#cat-peer-overlay-networking) - [Storage Internals](#cat-storage-internals) @@ -489,6 +490,34 @@ The transaction queue (TxQ) holds transactions that pay enough for local relay b **See also:** [Transaction queue (TxQ) on xrpl.org](https://xrpl.org/docs/concepts/transactions/transaction-queue) + + +## Job Queue + + + +### Concurrency limit + +Each job type declares how many of its jobs may run at the same time. Sync-critical types are deliberately tight so one kind of work cannot monopolize the worker pool. A type sitting at its limit cannot start more work even when worker threads are idle, which makes the limit a distinct kind of bottleneck from CPU or disk. + +**Scope:** per node — measured on and specific to this individual server. + + + +### Handler label + +Several call sites can enqueue work under the same job type, so job type alone cannot say which one caused a latency spike. The handler label carries the name of the call site that enqueued the job. Names are kept only when they are letters-only; anything else — including names that embed a ledger sequence number — folds into a shared `other` bucket, which keeps the number of series bounded. A reading under `other` therefore mixes several callers and never identifies one. + +**Scope:** per node — measured on and specific to this individual server. + + + +### Job queue / job type + +The job queue is the worker-thread pool that runs xrpld's background work. Every unit of work is enqueued under a named job type — serving a peer's ledger request, absorbing inbound ledger data, updating payment paths, and so on — and each type is accounted separately: waiting, running, and deferred. Types that carry sync-critical traffic are the ones worth watching, because a backlog there translates directly into the node falling behind. + +**Scope:** per node — measured on and specific to this individual server. + ## Node State & Sync @@ -543,7 +572,7 @@ Turning each configured peer hostname into IP addresses, which happens before an ### Deferred job -A job the queue accepted but withheld from a worker thread because its job type is already running at that type's concurrency limit. This is a third state alongside waiting and running, and it is counted in neither: the work exists and is being actively denied a thread, which is starvation rather than idleness or overload. The distinction matters because the sync-critical types run at very small limits — ledger requests and inbound ledger data are each capped at three concurrent jobs — so during a fresh sync those types routinely have work withheld while the queue looks shallow and the queue-wait quantiles look unremarkable. A sustained non-zero deferred count names the job type whose limit is the bottleneck. +A job the queue accepted but withheld from a worker thread because its job type is already running at that type's concurrency limit. This is a third state alongside waiting and running, and it is counted in neither: the work exists and is being actively denied a thread, which is starvation rather than idleness or overload. The distinction matters because the sync-critical types run at very small limits — ledger requests and inbound ledger data are each capped at three concurrent jobs — so during a fresh sync those types routinely have work withheld while the queue looks shallow and the queue-wait quantiles look unremarkable. A sustained non-zero deferred count names the job type whose limit is the bottleneck: each completing job releases one withheld job, so a count that stays high means arrivals are outpacing completions. **Scope:** per node — measured on and specific to this individual server. @@ -1011,6 +1040,14 @@ Replay-delta request/response messages transfer only the state changes between l **Scope:** per node — measured on and specific to this individual server. + + +### Resource charge + +The resource manager bills each peer a load cost per request, so expensive requests cost the sender more than cheap ones. For object fetches the charge scales with how many objects were asked for and how many of those were misses, with a surcharge once the request crosses a size band. A running balance above the warning threshold marks the peer as overactive; above the drop threshold the node sheds it. Requests inside the free allowance carry no charge beyond the flat per-message cost. + +**Scope:** per node — measured on and specific to this individual server. + ### Resource disconnect @@ -1095,6 +1132,14 @@ The NodeStore is xrpld's content-addressed object database holding all ledger tr **Scope:** per node — measured on and specific to this individual server. + + +### NodeStore lookup (hit / miss) + +A lookup is one attempt to fetch an object from the NodeStore by its hash. A hit is usually served from an in-memory cache and is cheap; a miss goes to the back-end store and costs a disk seek, so it is far more expensive. The hit/miss mix is therefore the main reason lookup time moves: a rising miss share explains slower lookups without any regression in the storage layer, while slower lookups on a hit-heavy mix point at the storage layer itself. + +**Scope:** per node — measured on and specific to this individual server. + ### NuDB diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 8e80edd981..ef6dcffc36 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -1430,18 +1430,80 @@ The `OTelCollector` implementation exports metrics via OTLP/HTTP to the same OTe #### Gauges -| Prometheus Metric | Source | Description | -| ------------------------------------- | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `ledgermaster_validated_ledger_age` | LedgerMaster.h:373 | Age of validated ledger (seconds) | -| `ledgermaster_published_ledger_age` | LedgerMaster.h:374 | Age of published ledger (seconds) | -| `state_accounting_{mode}_duration` | NetworkOPs.cpp:774 | Time in each operating mode (Disconnected/Connected/Syncing/Tracking/Full) | -| `state_accounting_{mode}_transitions` | NetworkOPs.cpp:780 | Transition count per mode | -| `peer_finder_active_inbound_peers` | PeerfinderManager.cpp:214 | Active inbound peer connections | -| `peer_finder_active_outbound_peers` | PeerfinderManager.cpp:215 | Active outbound peer connections | -| `overlay_peer_disconnects` | OverlayImpl.h:557 | Peer disconnect count | -| `jobq_job_count` | JobQueue.cpp:26 | Current job queue depth (emitted as `jobq_job_count`: the JobQueue collector is wrapped in `group("jobq")`, so the registered `job_count` gauge gains the `jobq_` prefix) | -| `{category}_bytes_in/out` | OverlayImpl.h:535 | Overlay traffic bytes per category (57 categories) | -| `{category}_messages_in/out` | OverlayImpl.h:535 | Overlay traffic messages per category | +| Prometheus Metric | Source | Description | +| ------------------------------------- | ------------------------- | -------------------------------------------------------------------------- | +| `ledgermaster_validated_ledger_age` | LedgerMaster.h:373 | Age of validated ledger (seconds) | +| `ledgermaster_published_ledger_age` | LedgerMaster.h:374 | Age of published ledger (seconds) | +| `state_accounting_{mode}_duration` | NetworkOPs.cpp:774 | Time in each operating mode (Disconnected/Connected/Syncing/Tracking/Full) | +| `state_accounting_{mode}_transitions` | NetworkOPs.cpp:780 | Transition count per mode | +| `peer_finder_active_inbound_peers` | PeerfinderManager.cpp:214 | Active inbound peer connections | +| `peer_finder_active_outbound_peers` | PeerfinderManager.cpp:215 | Active outbound peer connections | +| `overlay_peer_disconnects` | OverlayImpl.h:557 | Peer disconnect count | +| `jobq_job_count` | JobQueue.cpp:26 | Current job queue depth (all types) | +| `jobq_{jobtype}_waiting` | JobTypeData.h | Jobs of this type enqueued but not yet running | +| `jobq_{jobtype}_running` | JobTypeData.h | Jobs of this type currently executing | +| `jobq_{jobtype}_deferred` | JobTypeData.h | Jobs of this type held back because the type's concurrency limit was hit | +| `{category}_bytes_in/out` | OverlayImpl.h:535 | Overlay traffic bytes per category (57 categories) | +| `{category}_messages_in/out` | OverlayImpl.h:535 | Overlay traffic messages per category | + +Note that `job_count` is exported as `jobq_job_count`: the JobQueue is +constructed with `collectorManager_->group("jobq")` (Application.cpp:386), +`GroupImp::makeName()` joins prefix and name with a `.` (Groups.cpp:42), and +`OTelCollectorImp::formatName()` then turns the `.` into `_` and lowercases the +whole string (OTelCollector.cpp:860). The same mechanism produces the +`jobq_{jobtype}_*` names above and the pre-existing +`jobq_{jobtype}_milliseconds` timing family. + +#### Per-Job-Type Queue Saturation + +The three `jobq_{jobtype}_{waiting,running,deferred}` families expose the +per-type counters that `JobTypeData` already maintained but never exported. +`{jobtype}` is the lowercased `JobTypes` name, so `JtLedgerReq` ("ledgerRequest") +becomes `jobq_ledgerrequest_waiting` / `_running` / `_deferred`. + +They are emitted for every **non-special** job type — 35 of the 46 declared +types. A "special" type is one whose concurrency limit is 0 +(`JobTypeInfo::special()`, JobTypeInfo.h:71-74); the limit logic never applies +to it, so its `deferred` is always zero. The gauge members are declared at +JobTypeData.h:78-80 and created at :100-102, next to the existing +`dequeue`/`execute` events and under the same `!info.special()` guard (:95). They +are published by `JobQueue::collect()` under the `mutex_` that already guards the +counters (JobQueue.cpp:66-93) — no new locking. + +**`deferred` is the leading indicator.** `JobQueue::addJob()` never rejects +work — when a type is at its limit, `addRefCountedJob()` increments `deferred` +and returns `true` anyway (JobQueue.cpp:131-142), and `finishJob()` drains one +deferred job per completion (JobQueue.cpp:353-362). Backpressure on a capped type +therefore shows up **only as latency**, after the harm is done. `deferred > 0` +says the cap is being hit _now_, before the duration histograms move. + +Limits that matter for ledger sync (JobTypes.h:54-77): + +| Job type | Metric prefix | Limit | Producers | +| -------------- | --------------------- | ----- | ------------------------------------------------------------------------------- | +| `JtPack` | `jobq_makefetchpack_` | 1 | `MakeFetchPack` | +| `JtLedgerReq` | `jobq_ledgerrequest_` | 3 | `RcvGetLedger`, `RcvGetObjByHash` | +| `JtLedgerData` | `jobq_ledgerdata_` | 3 | `ProcessLData`, `GotStaleData`, `GotFetchPack`, `AcqDone`, `InboundLedger` | +| `JtUpdatePf` | `jobq_updatepaths_` | 1 | `PthFindNewReq`, `PthFindOBDB`, `PthFindNewLed`, `OB` — see the note below | +| `JtTxnData` | `jobq_fetchtxndata_` | 5 | `TxAcq`, `ComplAcquire`, `RcvPeerData` | + +> **`JtUpdatePf` has four producers, three of them individually visible.** All +> four run the same `updatePaths()` work but arrive under different names. Three +> come through `LedgerMaster::newPFWork()` (LedgerMaster.cpp:1545), which passes +> its caller's name straight to `addJob`: `PthFindNewReq` (:1512), +> `PthFindOBDB` (:1533), and `PthFindNewLed` (:1984). All three are all-letters, +> so each is its own `handler` series. The fourth is +> `"OB" + std::to_string(seq)` (OrderBookDBImpl.cpp:84), which contains digits +> and therefore folds to `handler="other"` — order-book rebuild traffic is the +> only one of the four that is not directly attributable. Do not read the whole +> type as invisible: three of its four producers are named. + +> **Sampling caveat**: these are gauges read by the `JobQueue::collect()` hook, +> which the beast::insight `PeriodicMetricReader` drives every 1 s +> (Telemetry.cpp:441). A `deferred` spike shorter than the sample interval can be +> missed entirely. Treat a non-zero reading as real saturation, but do not treat +> a zero reading as proof that no saturation occurred — cross-check +> `job_queued_us` for the same type. #### OTel MetricsRegistry Gauges @@ -1491,6 +1553,126 @@ These gauges are exported via the OTel Metrics SDK `PeriodicMetricReader` (10s i | `pathfind_fast` | PathRequests.h:23 | Fast pathfinding duration (ms) | | `pathfind_full` | PathRequests.h:24 | Full pathfinding duration (ms) | +#### Job Instruments + +These five come from the `PerfLog` job hooks, not from beast::insight, so they +are exported by the `MetricsRegistry` meter. `job_queued_us` and `job_running_us` +have explicit microsecond bucket views registered +(`addMicrosecondHistogramView()`, MetricsRegistry.cpp:253-254) spanning 100 µs to +60 s; without those the SDK default buckets stop at 10 ms and every quantile +saturates. + +| Prometheus Metric | Kind | Labels | Description | +| -------------------- | --------- | --------------------- | ------------------------------------ | +| `job_queued_total` | Counter | `job_type`, `handler` | Jobs enqueued | +| `job_started_total` | Counter | `job_type`, `handler` | Jobs dequeued and started | +| `job_finished_total` | Counter | `job_type`, `handler` | Jobs run to completion | +| `job_queued_us` | Histogram | `job_type`, `handler` | Time spent waiting in the queue (µs) | +| `job_running_us` | Histogram | `job_type`, `handler` | Time spent executing (µs) | + +#### The `handler` Label + +`job_type` names the queue a job ran on, not the code that submitted it. Several +job types have more than one producer, so `job_type` alone cannot attribute a +latency spike. The clearest case: `RcvGetLedger` (PeerImp.cpp:1566) and +`RcvGetObjByHash` (PeerImp.cpp:2603) both submit to `JtLedgerReq`, so both report +as `job_type="ledgerRequest"`. `JtLedgerData` has five producers, `JtUpdatePf` has +four, and `JtAdvance` has four. + +The `handler` label carries the name string passed to `addJob()` — the specific +call site. It resolves all of those at once, not just the GetObject path. + +**The value is sanitized, not raw.** A raw job name would be unbounded, because +two production job names embed a ledger sequence number: + +- `"Pub" + std::to_string(ledger->seq())` (LedgerPersistence.cpp:84) +- `"OB" + std::to_string(ledger->seq() % 1000000000)` (OrderBookDBImpl.cpp:84) + +A raw label would mint a new Prometheus series for every ledger — unbounded +growth at ~1 series every 3-5 s, forever. +`MetricsRegistry::sanitiseHandler()` (declared inline in +`src/xrpld/telemetry/MetricsRegistry.h`) therefore applies one rule: + +- Keep the name when it is **non-empty and every character is an ASCII letter**. +- Otherwise return the constant `"other"`. An empty name, a digit, a hyphen, or + any punctuation all fall here. + +Both dynamic names always contain digits, so both collapse to `"other"`. Because +the rule is a pure function of compile-time string literals, the label domain is +fixed at build time and cannot grow at runtime. That is a stronger guarantee than +an allowlist, which would silently mislabel any job added later; this rule +degrades to `"other"` instead. + +**Five static job names also fall into `"other"`.** Sweeping every `addJob` / +`addRefCountedJob` / `postCoro` / `newPFWork` / `TimeoutCounter::jobName` call +site outside `src/test` finds 48 distinct static name literals. 43 are +all-letters and pass through; five are not, and two more are built at runtime +from a ledger sequence. The domain is therefore **44 values** (43 names plus +`"other"`): + +| Name | Why it is not all-letters | Call site | +| ------------- | ------------------------- | ---------------------- | +| `GetConsL1` | digit | RCLConsensus.cpp:169 | +| `GetConsL2` | digit | RCLValidations.cpp:135 | +| `gRPC-Client` | hyphen | GRPCServer.cpp:156 | +| `RPC-Client` | hyphen | ServerHandler.cpp:332 | +| `WS-Client` | hyphen | ServerHandler.cpp:376 | + +> The consequence worth remembering: `handler="other"` is a **mixed bucket**, not +> a residual. It holds the two per-ledger dynamic names _and_ those five static +> ones, so `GetConsL1` and `GetConsL2` — two different `JtAdvance` producers — +> are not separable, and neither are the three RPC client-session names. Do not +> read a handler breakdown as exhaustive. The GetObject path is unaffected: +> `RcvGetObjByHash` and `RcvGetLedger` are all-letters and pass through as +> distinct series. +> +> The count is "at the time of writing" — it is a property of the source, not of +> the rule. Re-derive it from the call sites after adding a job rather than +> trusting this number. + +#### GetObject Request Metrics + +Five instruments on the `TMGetObjectByHash` query path, recorded via the +`XRPL_METRIC_*` macros at their call sites in `PeerImp.cpp`. Label cardinality is +fixed and tiny: two `result` values, two `reason` values. + +| Prometheus Metric | Kind | Labels | Description | +| --------------------------- | --------- | ----------------------------------------------- | ----------------------------------------------------- | +| `getobject_lookup_us` | Histogram | none | Time inside the NodeStore fetch loop (µs) | +| `getobject_request_objects` | Histogram | none | Objects requested per request | +| `getobject_lookups_total` | Counter | `result` = `hit` \| `miss` | NodeStore hit/miss volume | +| `getobject_rejected_total` | Counter | `reason` = `oversize` \| `malformed_ledgerhash` | Requests refused before any NodeStore access | +| `getobject_charge` | Histogram | none | Dynamic component of the differential resource charge | + +Aggregation choices worth knowing when reading these: + +- `getobject_lookup_us` times the **whole fetch loop once** + (`processGetObjectByHash()`, PeerImp.cpp:2713-2742 — the iteration cap is set + at :2713 and the loop ends at :2742), not each iteration. The loop can run up + to `kHardMaxReplyNodes` = 12288 times (Tuning.h:30); timing each + `fetchNodeObject()` would cost more than the lookups. It needs an + `addMicrosecondHistogramView()` entry for the same reason the job histograms + do — a 12288-lookup loop routinely exceeds 10 ms, so without the view the + metric saturates exactly when it matters. +- `getobject_lookups_total` is incremented **once per request with the batch + totals**, not once per object. A 12288-iteration loop incrementing per object + would be a measurable hot-path cost for no extra information. +- `getobject_request_objects` records `packet.objects_size()` — the _requested_ + count, which is what the charge bands price on, not the count actually found. +- `getobject_charge` records only the **dynamic** part returned by + `computeGetObjectByHashFee()` (PeerImp.cpp:3658-3681), applied at + PeerImp.cpp:2757-2758 just after the loop. The admission-time base charge is a + constant (`kFeeModerateBurdenPeer`, PeerImp.cpp:2634) and is already implied. +- `getobject_rejected_total` counts the two early returns in + `onMessage(TMGetObjectByHash)`: the malformed-ledgerhash check (PeerImp.cpp:2569, + counter at :2573) and the oversize gate (PeerImp.cpp:2585, counter at :2591). + Both fire before the job is enqueued, so a rejected request contributes to no + other GetObject metric. + +> On a healthy local network `getobject_rejected_total` reads zero — no honest +> peer sends an oversized request. Verify its panel with a synthetic oversized +> request; do not assume it works because the query parses. + #### Adding a New Metric @@ -2041,6 +2223,114 @@ Wait for the node to sync with the network. The `getKBUsed*()` methods require SQLite databases to exist. If running with `--standalone` or before the first ledger is stored, these will be zero. +### Slow TMGetObjectByHash service + +Use this when a peer reports slow object fetches, or when `job_queued_us` / +`job_running_us` for `job_type="ledgerRequest"` rises. The goal is to name the +cause, not to confirm the slowness. + +End-to-end handler time splits into three additive parts, and each has its own +signal: + +```mermaid +flowchart LR + A["`**Request arrives** + onMessage + TMGetObjectByHash`"] --> B["`**1. Queue wait** + job_queued_us + handler=RcvGetObjByHash`"] + B --> C["`**2. NodeStore lookup** + getobject_lookup_us`"] + C --> D["`**3. Everything else** + protobuf, serialization, + reply construction`"] + D --> E["`**Reply sent**`"] + + C -.-> F["`job_running_us + handler=RcvGetObjByHash + covers steps 2 + 3`"] + D -.-> F + + style A fill:#1f4e79,color:#ffffff + style B fill:#7b3f00,color:#ffffff + style C fill:#2d5016,color:#ffffff + style D fill:#4a148c,color:#ffffff + style E fill:#1f4e79,color:#ffffff + style F fill:#37474f,color:#ffffff +``` + +**Reading the diagram** + +- Step 1 is time the job sat in `JtLedgerReq` before a worker picked it up. Only + `job_queued_us` measures it. +- Steps 2 and 3 both happen inside the worker, so `job_running_us` covers them + together. `getobject_lookup_us` isolates step 2 alone. +- Step 3 is therefore not measured directly. Derive it: + `job_running_us − getobject_lookup_us`. That subtraction is what makes the set + able to name a cause instead of just reporting a duration. + +**Procedure** — work through these in order. The first row that matches is the +answer. + +> **Scope every query to one node.** All snippets below carry +> `service_instance_id="$node"`. On a shared Grafana stack an unscoped selector +> aggregates across every node and branch reporting to it, so another node's +> saturation would be attributed to this one. Substitute the node's public key +> for `$node` when querying Prometheus directly rather than from a dashboard. + +1. Split queue wait from run time. Compare the two p99s for the handler: + + ```promql + histogram_quantile(0.99, sum by (le) (rate(job_queued_us_bucket{handler="RcvGetObjByHash", service_instance_id="$node"}[5m]))) + histogram_quantile(0.99, sum by (le) (rate(job_running_us_bucket{handler="RcvGetObjByHash", service_instance_id="$node"}[5m]))) + ``` + +2. If run time is the larger term, split it against the fetch loop: + + ```promql + histogram_quantile(0.99, sum by (le) (rate(getobject_lookup_us_bucket{service_instance_id="$node"}[5m]))) + ``` + +3. Match the outcome below. + +| Observation | Root cause and next step | +| -------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `job_queued_us{handler="RcvGetObjByHash"}` high, `job_running_us` normal | **Queue contention** — the work is cheap, the wait is not. Confirm with `jobq_ledgerrequest_deferred{service_instance_id="$node"} > 0`, then compare `job_queued_us{handler="RcvGetLedger"}`: if it is also high, both producers are starved by the limit of 3, not by each other. | +| `job_running_us` high and within ~10% of `getobject_lookup_us` | **NodeStore is the bottleneck** — nearly all run time is in the fetch loop. Check `rate(getobject_lookups_total{result="miss"}[5m])` and the existing NuDB / `nodestore_state` panels. A miss-heavy mix means real disk seeks. | +| `job_running_us` high but `getobject_lookup_us` low | **Cost is outside the fetch loop** — protobuf, serialization, or reply construction. Storage is fine. Look at reply size: a large `getobject_request_objects` with a high hit rate means big replies to build and send. | +| `getobject_request_objects` p99 large | **Peers are sending big batches** — the work is real, not a regression. Nothing is broken; the node is being asked to do more. Decide whether to accept the load or price it higher. | +| `rate(getobject_rejected_total{reason="oversize"}[5m])` rising | **Non-conforming traffic** — requests above `kHardMaxReplyNodes` are being refused before any NodeStore access. Check `getobject_charge` to confirm the pricing escalates for the requests that _are_ accepted. | +| `rate(getobject_rejected_total{reason="malformed_ledgerhash"}[5m])` rising | **Malformed requests** — a peer is sending a ledgerhash that is not 32 bytes. Refused at the gate; no queue or storage cost incurred. | +| All GetObject metrics normal, `jobq_*_deferred` high on another type | **This path is exonerated** — the slowness is elsewhere. Find the saturated type with `topk(5, {__name__=~"jobq_.*_deferred", service_instance_id="$node"} > 0)` and investigate that producer instead. | + +Row 2 says "within ~10%", not "equal", deliberately: `job_running_us` also +covers the charge computation (PeerImp.cpp:2757) and the reply `send()` that +follow the loop, so it is always the larger of the two. Treat a small residual as +normal and only a large one as a signal — that is what row 3 is for. + +The last row matters as much as the others: the set can rule this path _out_, +which a slowness-only metric cannot. + +**Caveats** + +- `handler` collapses to `"other"` for any job name that is not all ASCII + letters, so a handler breakdown is not exhaustive — see + [The `handler` Label](#the-handler-label). This does not affect + `RcvGetObjByHash` or `RcvGetLedger`; both pass through as distinct series. +- `jobq_*_deferred` is sampled at 1 s. A zero reading does not prove no + saturation occurred — see the sampling caveat under + [Per-Job-Type Queue Saturation](#per-job-type-queue-saturation). +- The two families compared in this procedure are exported on **different + cadences by separate meter providers**: the `jobq_*` gauges every 1 s + (`Telemetry.cpp` global provider) and `job_queued_us` / `job_running_us` / + `getobject_*` every 10 s (the private `MetricsRegistry` provider). A short + spike can therefore appear in one family a step before the other. When + correlating them, widen the window rather than reading a single scrape, and + do not conclude the two disagree from one interval's difference. +- A request rejected at either gate contributes to no other GetObject metric, so + a rejection spike will _not_ show up as latency. Check the rejection counters + before concluding that traffic is normal. + ### High memory usage - Reduce trace volume with collector-side tail sampling (xrpld head sampling is @@ -2175,13 +2465,13 @@ Ledger acquires are in flight, _Ledgers Behind Network_ is flat or rising, and | _Add-Node Outcomes_ | `good` dominates | `duplicate` swamps `good` | bandwidth busy, acquire standing still — peers re-sending known data | | | | `invalid` rising | a specific misbehaving peer, not a local fault | | _Received-Data Stash Depth & In-Flight Acquires_ | stash drains | stash growing | data arrives faster than it is applied — a job-queue or disk problem, the **opposite** conclusion from a stall rate, and only this panel separates them | -| _Deferred Jobs by Type (starvation)_ | flat at 0 | sustained non-zero on `ledgerData`/`ledgerRequest` | a job the queue accepted then **withheld** at its concurrency limit of 3 — it appears in neither `waiting` nor `running`, so no other signal can show it. Starved `ledgerData` is exactly why the stash grows while missing nodes stay flat | +| `jobq__deferred` | flat at 0 | sustained non-zero on `ledgerdata`/`ledgerrequest` | a job the queue accepted then **withheld** at its concurrency limit of 3 — it appears in neither `waiting` nor `running`, so no other signal can show it. Starved `ledgerdata` is exactly why the stash grows while missing nodes stay flat | | _Worker Pool Saturation_ + _Worker Pool Capacity & Total Backlog_ | under 80% | 100% with `total_waiting` climbing | the pool is **exhausted** — every stage looks slow at once. Stop here; no per-subsystem fix helps while no thread is free | | Acquire outcome `abandoned` in Tempo (`{name="ledger.acquire" && span.outcome="abandoned"}`) | absent | present | the acquire was swept or shut down before reaching a result — without this value a stuck-then-swept fetch had no `outcome` at all and vanished from every outcome rate | **Conclusion:** distinguish "nobody is serving it" (peer supply) from "it arrives and we cannot process it" (job queue / disk). The two look identical in a log and -are separated only by the stash-depth and deferred-jobs panels. Detail: +are separated only by the stash-depth and per-type deferred gauges. Detail: [Sync pipeline](#sync-pipeline--ordered-diagnosis) steps 6-12 and 17. #### Branch D — reaching `full` but slowly, or falling back out of it @@ -2500,18 +2790,22 @@ panel it reads. while no thread is free to run its jobs. Look at what is holding the threads (long-running jobs, disk waits from step 9) or at the `[workers]` setting for the node size. - - **Then per type** — _Deferred Jobs by Type (starvation)_ - (`jobq_backlog`, `metric=deferred`). This is the signal that exists - nowhere else. A deferred job is one the queue **accepted and then - withheld** because its type is already running at its concurrency limit, - so it appears in neither `waiting` nor `running`, and neither the job - counters nor the queue-wait histograms can show it. Any sustained - non-zero value names the job type whose limit is the bottleneck. During a - fresh sync watch `job_type=ledgerData` and `job_type=ledgerRequest` - first: both run at a limit of 3, so they are the types that starve - soonest, and starved `ledgerData` is exactly why the received-data stash - in step 8 grows while the missing-node count in step 6 stays flat. - _Job Queue Occupancy by Type (waiting/running)_ gives the context — + - **Then per type** — the `jobq__deferred` gauges, one series per + job type (see + [Per-Job-Type Queue Saturation](#per-job-type-queue-saturation)). This is + the signal that exists nowhere else. A deferred job is one the queue + **accepted and then withheld** because its type is already running at its + concurrency limit, so it appears in neither `waiting` nor `running`, and + neither the job counters nor the queue-wait histograms can show it. Any + sustained non-zero value names the job type whose limit is the + bottleneck; find it with + `topk(5, {__name__=~"jobq_.*_deferred", service_instance_id="$node"})`. + During a fresh sync watch `jobq_ledgerdata_deferred` and + `jobq_ledgerrequest_deferred` first: both types run at a limit of 3, so + they are the ones that starve soonest, and starved `ledgerdata` is + exactly why the received-data stash in step 8 grows while the + missing-node count in step 6 stays flat. The matching + `jobq__running` and `_waiting` gauges give the context — `running` pinned at the limit with `waiting` above zero confirms the limit, not the supply, is what is holding the type back. Note the difference from _Job Queue Wait p95 By Type_ on the Ledger Data diff --git a/include/xrpl/core/Job.h b/include/xrpl/core/Job.h index 93b39701be..53e8566a99 100644 --- a/include/xrpl/core/Job.h +++ b/include/xrpl/core/Job.h @@ -109,6 +109,20 @@ public: [[nodiscard]] JobType getType() const; + /** + * Returns the job name supplied to JobQueue::addJob. + * + * Several job types have more than one producer (for example both + * RcvGetLedger and RcvGetObjByHash run as JtLedgerReq), so the name is + * the only thing that tells them apart. It is read by the JobQueue + * PerfLog hooks and exported as the `handler` metric label. + * + * Empty for the default and index-only constructors, which carry no + * name; callers must treat an empty name as "unknown". + */ + [[nodiscard]] std::string const& + getName() const; + /** * Returns the time when the job was queued. */ diff --git a/include/xrpl/core/JobQueue.h b/include/xrpl/core/JobQueue.h index 8f35775878..0c1e622c6c 100644 --- a/include/xrpl/core/JobQueue.h +++ b/include/xrpl/core/JobQueue.h @@ -32,7 +32,6 @@ #include #include #include -#include namespace xrpl { @@ -216,64 +215,6 @@ public: int getJobCountGE(JobType t) const; - /** - * Occupancy snapshot for a single JobType. - * - * A plain value type so it can cross the libxrpl/xrpld boundary: xrpld - * telemetry observes queue occupancy without libxrpl gaining any - * dependency on the telemetry code. - * - * `deferred` is the field with no other exposure anywhere. The - * sync-critical types run at tiny concurrency limits (`JtLedgerReq` and - * `JtLedgerData` are capped at 3 in JobTypes.h), so a job of those types - * is commonly held back rather than merely queued, and a held-back job is - * invisible in `waiting` and `running` alike. - */ - struct JobTypeCount - { - /** - * The job type these counts describe. The caller turns this into a - * label via JobTypes::name(), so the name is not duplicated here. - */ - JobType type{JtInvalid}; - - /** - * Jobs enqueued and not yet dispatched to a worker thread. - */ - int waiting{0}; - - /** - * Jobs currently executing on a worker thread. - */ - int running{0}; - - /** - * Jobs held back because this type is already at its concurrency - * limit. A non-zero value means work of this type exists and is - * being denied a worker: starvation, not idleness. - */ - int deferred{0}; - }; - - /** - * Snapshot the occupancy of every registered job type. - * - * One mutex acquire copies three integers per type, which is the same - * lock and the same fields getJobCount() already reads — the counting - * logic is not duplicated, only batched, so the caller does not have to - * take the lock once per type to build a full picture. - * - * @return One JobTypeCount per registered JobType, in JobType order. - * - * @note Thread-safe; takes the internal mutex briefly. The values are a - * point-in-time reading and are mutually consistent with each other - * because they come from one acquire. - * @note Intended for a periodic observer (the telemetry reader ticks - * every ~10 s). It is not free enough to call from a hot path. - */ - [[nodiscard]] std::vector - getJobTypeCounts() const; - /** * Worker-pool saturation reading: work in flight against capacity. * @@ -439,7 +380,9 @@ private: // any. // // Invariants: - // + // The calling thread owns the JobLock. This function mutates the + // deferred and running counts, which mutex_ guards; collect() reads + // them under the same lock. void finishJob(JobType type); diff --git a/include/xrpl/core/JobTypeData.h b/include/xrpl/core/JobTypeData.h index d53440e1ca..b503cbcbbd 100644 --- a/include/xrpl/core/JobTypeData.h +++ b/include/xrpl/core/JobTypeData.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -20,6 +21,23 @@ private: beast::insight::Collector::ptr collector_; public: + /** + * Metric-name suffixes appended to `JobTypeInfo::name()`. + * + * The constructor builds the instrument names from these. Public so + * tests can assert on the exported names without repeating the + * literals. The collector is the `"jobq"` group, so + * `GroupImp::makeName()` prefixes `jobq.` and `OTelCollector` + * lowercases and turns `.` into `_`: `ledgerRequest` + + * `kSuffixDeferred` exports as `jobq_ledgerrequest_deferred`. + */ + /** @{ */ + static constexpr char kSuffixWaiting[] = "_waiting"; + static constexpr char kSuffixRunning[] = "_running"; + static constexpr char kSuffixDeferred[] = "_deferred"; + static constexpr char kSuffixQueued[] = "_q"; + /** @} */ + /* The job category which we represent */ JobTypeInfo const& info; @@ -36,6 +54,32 @@ public: beast::insight::Event dequeue; beast::insight::Event execute; + /** + * Saturation gauges, published by `JobQueue::collect()`. + * + * Each mirrors the same-named counter above so per-job-type queue + * pressure is visible in metrics. Without them the only exported queue + * signal is the process-wide `jobq_job_count`, which cannot attribute + * pressure to a job type. + * + * `waitingGauge` is the backlog not yet started, `runningGauge` is the + * in-flight count, and `deferredGauge` is the count held back by this + * type's concurrency limit. `deferredGauge` is the leading indicator: + * `JobQueue::addJob()` never rejects, so a capped type under pressure + * shows up as latency only after the fact, whereas a non-zero deferred + * reading precedes it. + * + * Created only for non-special types (see the constructor). A default + * constructed `beast::insight::Gauge` holds a null impl and every + * mutator is a no-op, so a special type's gauge is safe to assign to + * but publishes nothing. + */ + /** @{ */ + beast::insight::Gauge waitingGauge; + beast::insight::Gauge runningGauge; + beast::insight::Gauge deferredGauge; + /** @} */ + JobTypeData( JobTypeInfo const& info, beast::insight::Collector::ptr collector, @@ -45,10 +89,17 @@ public: { load_.setTargetLatency(info.getAverageLatency(), info.getPeakLatency()); + // Special types have limit_ == 0 and bypass the limit logic + // entirely, so their `deferred` is always 0. Excluding them here + // matches the existing dequeue/execute rule. if (!info.special()) { - dequeue = collector_->makeEvent(info.name() + "_q"); + dequeue = collector_->makeEvent(info.name() + kSuffixQueued); execute = collector_->makeEvent(info.name()); + + waitingGauge = collector_->makeGauge(info.name() + kSuffixWaiting); + runningGauge = collector_->makeGauge(info.name() + kSuffixRunning); + deferredGauge = collector_->makeGauge(info.name() + kSuffixDeferred); } } diff --git a/include/xrpl/core/PerfLog.h b/include/xrpl/core/PerfLog.h index f09665e291..441a6f6c76 100644 --- a/include/xrpl/core/PerfLog.h +++ b/include/xrpl/core/PerfLog.h @@ -92,30 +92,41 @@ public: * Log queued job * * @param type Job type + * @param name Job name as given to JobQueue::addJob. Distinguishes the + * several producers that share one job type. May be empty. */ virtual void - jobQueue(JobType const type) = 0; + jobQueue(JobType const type, std::string const& name) = 0; /** * Log job executing * * @param type Job type + * @param name Job name as given to JobQueue::addJob. Distinguishes the + * several producers that share one job type. May be empty. * @param dur Duration enqueued in microseconds * @param startTime Time that execution began * @param instance JobQueue worker thread instance */ virtual void - jobStart(JobType const type, microseconds dur, steady_time_point startTime, int instance) = 0; + jobStart( + JobType const type, + std::string const& name, + microseconds dur, + steady_time_point startTime, + int instance) = 0; /** * Log job finishing * * @param type Job type + * @param name Job name as given to JobQueue::addJob. Distinguishes the + * several producers that share one job type. May be empty. * @param dur Duration running in microseconds * @param instance Jobqueue worker thread instance */ virtual void - jobFinish(JobType const type, microseconds dur, int instance) = 0; + jobFinish(JobType const type, std::string const& name, microseconds dur, int instance) = 0; /** * Render performance counters in Json diff --git a/include/xrpl/telemetry/GetObjectMetricNames.h b/include/xrpl/telemetry/GetObjectMetricNames.h new file mode 100644 index 0000000000..e2951a43d0 --- /dev/null +++ b/include/xrpl/telemetry/GetObjectMetricNames.h @@ -0,0 +1,160 @@ +#pragma once + +// cspell:ignore ISTOGRAM +// The all-caps macro name XRPL_METRIC_HISTOGRAM_RECORD trips cspell's +// compound-word splitter, which emits the subword "ISTOGRAM"; ignore it here. + +/** + * Metric names, label keys, label values, and descriptions for the + * `TMGetObjectByHash` request path. + * + * These constants are shared across two modules, which is why they live in a + * header rather than in either translation unit's unnamed namespace: + * + * GetObjectMetricNames.h + * | + * +--> PeerImp.cpp (XRPL_METRIC_* call sites: records the + * | instruments) + * | + * +--> MetricsRegistry.cpp (addMicrosecondHistogramView: registers + * the explicit bucket boundaries for + * kGetObjectLookupUs) + * + * `kGetObjectLookupUs` in particular is referenced from both sites. A + * copy-pasted literal would let the two drift, and a drifted name silently + * drops the bucket override -- the histogram would fall back to the SDK + * default boundaries, which top out at 10,000 (10 ms), so every quantile + * would saturate. This mirrors the reason the existing + * `kJobQueuedDurationUs` / `kJobRunningDurationUs` / `kRpcMethodDurationUs` + * constants exist in MetricsRegistry.cpp; those three are referenced only + * within that one file, so they stay file-local. + * + * Placed under `include/xrpl/telemetry/` because the two consumers sit in + * different levelization modules: PeerImp.cpp is `xrpld.overlay` and + * MetricsRegistry.cpp is `xrpld.telemetry`. Both are allowed to depend on + * `xrpl.telemetry` (see `xrpld.overlay > xrpl.telemetry` and + * `xrpld.telemetry > xrpl.telemetry` in the levelization ordering results), so + * `include/xrpl/` is the one level both can reach. Keeping the constants in + * `src/xrpld/telemetry/` would have required overlay to include a private + * xrpld header from another module, flipping a levelization edge. Both sites + * include this file as ``. + * + * Example usage -- recording a histogram: + * @code + * XRPL_METRIC_HISTOGRAM_RECORD( + * app_, kGetObjectRequestObjects, kGetObjectRequestObjectsDesc, requested); + * @endcode + * + * Example usage -- edge case: the same instrument recorded under two + * different label values, which is why the label key and both values are + * constants rather than literals: + * @code + * XRPL_METRIC_COUNTER_ADD_LABELED( + * app_, kGetObjectLookupsTotal, kGetObjectLookupsTotalDesc, hits, + * {{kLabelResult, std::string(kResultHit)}}); + * XRPL_METRIC_COUNTER_ADD_LABELED( + * app_, kGetObjectLookupsTotal, kGetObjectLookupsTotalDesc, misses, + * {{kLabelResult, std::string(kResultMiss)}}); + * @endcode + * + * @note These are `constexpr char[]`, not `constexpr std::string_view`. The + * OTel C++ API takes `nostd::string_view`, which on this build is OTel's own + * type (the build defines only OPENTELEMETRY_ABI_VERSION_NO, not + * OPENTELEMETRY_STL_VERSION, so `nostd::string_view` is not an alias for + * `std::string_view`). It converts from `char const*` and from + * `std::string`, but has no converting constructor from `std::string_view`, + * so a `string_view` constant would not compile at the call sites. This also + * matches the existing convention in MetricsRegistry.cpp. + * + * @note Header-only constants with no runtime state, so there is nothing to + * synchronize -- safe to include from any thread context. + */ + +namespace xrpl::telemetry { + +// ===== Metric names ========================================================== + +/** + * Requests refused by `onMessage(TMGetObjectByHash)` before any NodeStore + * access, split by the `reason` label. + */ +inline constexpr char kGetObjectRejectedTotal[] = "getobject_rejected_total"; + +/** + * Distribution of the object count requested per message. + */ +inline constexpr char kGetObjectRequestObjects[] = "getobject_request_objects"; + +/** + * Wall time spent in the NodeStore fetch loop, in microseconds. + * + * Referenced twice: here at the record site, and by + * `addMicrosecondHistogramView()` in MetricsRegistry.cpp. Both must use this + * one constant. + */ +inline constexpr char kGetObjectLookupUs[] = "getobject_lookup_us"; + +/** + * NodeStore lookups performed, split by the `result` label. + */ +inline constexpr char kGetObjectLookupsTotal[] = "getobject_lookups_total"; + +/** + * Distribution of the dynamic resource charge applied per request. + */ +inline constexpr char kGetObjectCharge[] = "getobject_charge"; + +// ===== Label keys ============================================================ + +/** + * Label key distinguishing a hit from a miss on `kGetObjectLookupsTotal`. + */ +inline constexpr char kLabelResult[] = "result"; + +/** + * Label key naming which gate refused the request. + */ +inline constexpr char kLabelReason[] = "reason"; + +// ===== Label values ========================================================== + +/** + * `kLabelResult` value: the object was found in the NodeStore. + */ +inline constexpr char kResultHit[] = "hit"; + +/** + * `kLabelResult` value: the object was not found. + */ +inline constexpr char kResultMiss[] = "miss"; + +/** + * `kLabelReason` value: more objects requested than the handler accepts. + */ +inline constexpr char kReasonOversize[] = "oversize"; + +/** + * `kLabelReason` value: the ledger hash was not uint256-sized. + */ +inline constexpr char kReasonMalformedLedgerHash[] = "malformed_ledgerhash"; + +// ===== Instrument descriptions =============================================== + +/** @{ */ +inline constexpr char kGetObjectRejectedTotalDesc[] = + "TMGetObjectByHash requests refused before any NodeStore access, by reason"; + +inline constexpr char kGetObjectRequestObjectsDesc[] = + "Objects requested per TMGetObjectByHash message"; + +inline constexpr char kGetObjectLookupUsDesc[] = + "Time spent in the TMGetObjectByHash NodeStore fetch loop, in microseconds"; + +inline constexpr char kGetObjectLookupsTotalDesc[] = + "TMGetObjectByHash NodeStore lookups, by hit/miss result"; + +inline constexpr char kGetObjectChargeDesc[] = + "Dynamic resource charge applied per TMGetObjectByHash request"; +/** @} */ + +} // namespace xrpl::telemetry diff --git a/src/libxrpl/core/detail/Job.cpp b/src/libxrpl/core/detail/Job.cpp index 89fe42db05..70c11b3bf1 100644 --- a/src/libxrpl/core/detail/Job.cpp +++ b/src/libxrpl/core/detail/Job.cpp @@ -36,6 +36,12 @@ Job::getType() const return type_; } +std::string const& +Job::getName() const +{ + return name_; +} + Job::clock_type::time_point const& Job::queueTime() const { diff --git a/src/libxrpl/core/detail/JobQueue.cpp b/src/libxrpl/core/detail/JobQueue.cpp index a12b4a3531..c235e25f5f 100644 --- a/src/libxrpl/core/detail/JobQueue.cpp +++ b/src/libxrpl/core/detail/JobQueue.cpp @@ -66,8 +66,45 @@ JobQueue::~JobQueue() void JobQueue::collect() { - std::scoped_lock const lock(mutex_); - jobCount_ = jobSet_.size(); + // Gauge::value_type is unsigned. The counters are only ever expected to + // be non-negative (the surrounding asserts enforce it), but clamp anyway: + // an unclamped negative would wrap to ~1.8e19 and swamp every dashboard + // reading this family. + auto const toGauge = [](int value) { + return static_cast(std::max(0, value)); + }; + + // Snapshot under the lock, publish after releasing it. + // + // Writing gauges while holding `mutex_` would invert a lock order: a + // collector's gauge write can take the collector's own lock, while the + // collector's flush thread already holds that lock when it calls this + // hook and then needs `mutex_`. Publishing outside the lock keeps + // `mutex_` strictly innermost, so no cycle can form. It also keeps the + // queue's hot lock (every addJob and every job start/finish takes it) + // off the per-gauge write path, which on some collectors allocates. + // + // A snapshot can be one flush interval stale; these are sampled + // saturation gauges, so that is the intended accuracy. + std::vector> snapshot; + { + std::scoped_lock const lock(mutex_); + jobCount_ = jobSet_.size(); + + snapshot.reserve(jobData_.size()); + for (auto& [type, data] : jobData_) + snapshot.emplace_back(&data, data.waiting, data.running, data.deferred); + } + + // Gauges exist only for non-special types (JobTypeData's ctor). Assigning + // to a special type's default-constructed Gauge is a safe no-op -- + // Gauge::set() checks its impl pointer -- so this needs no special case. + for (auto const& [data, waiting, running, deferred] : snapshot) + { + data->waitingGauge = toGauge(waiting); + data->runningGauge = toGauge(running); + data->deferredGauge = toGauge(deferred); + } } bool @@ -99,7 +136,9 @@ JobQueue::addRefCountedJob(JobType type, std::string const& name, JobFunction co JobType const type(job.getType()); XRPL_ASSERT(type != JtInvalid, "xrpl::JobQueue::addRefCountedJob : has valid job type"); XRPL_ASSERT(jobSet_.contains(job), "xrpl::JobQueue::addRefCountedJob : job found"); - perfLog_.jobQueue(type); + // `name` is the addJob name; it becomes the `handler` metric label so + // producers sharing a job type stay individually attributable. + perfLog_.jobQueue(type, name); JobTypeData& data(getJobTypeData(type)); @@ -154,29 +193,6 @@ JobQueue::getJobCountGE(JobType t) const return ret; } -std::vector -JobQueue::getJobTypeCounts() const -{ - std::vector out; - - std::scoped_lock const lock(mutex_); - - // Reserve once so the loop itself cannot reallocate while the lock is - // held; the body is then three integer reads per type. - out.reserve(jobData_.size()); - for (auto const& [type, data] : jobData_) - { - out.push_back( - JobTypeCount{ - .type = type, - .waiting = data.waiting, - .running = data.running, - .deferred = data.deferred}); - } - - return out; -} - JobQueue::WorkerSaturation JobQueue::getWorkerSaturation() const { @@ -402,7 +418,7 @@ JobQueue::processTask(int instance) // The amount of time that the job was in the queue auto const qTime = ceil(startTime - job.queueTime()); - perfLog_.jobStart(type, qTime, startTime, instance); + perfLog_.jobStart(type, job.getName(), qTime, startTime, instance); job.doJob(); @@ -414,7 +430,9 @@ JobQueue::processTask(int instance) getJobTypeData(type).dequeue.notify(qTime); getJobTypeData(type).execute.notify(xTime); } - perfLog_.jobFinish(type, xTime, instance); + // `job` is still alive here: it is scoped to the enclosing block + // and doJob() only releases the callable, not the name. + perfLog_.jobFinish(type, job.getName(), xTime, instance); } } diff --git a/src/test/basics/PerfLog_test.cpp b/src/test/basics/PerfLog_test.cpp index 41b5f81f5d..792934f0fe 100644 --- a/src/test/basics/PerfLog_test.cpp +++ b/src/test/basics/PerfLog_test.cpp @@ -533,7 +533,7 @@ public: // the jobs data with every addition. for (int i = 0; i < jobs.size(); ++i) { - perfLog->jobQueue(jobs[i].type); + perfLog->jobQueue(jobs[i].type, jobs[i].typeName); json::Value const jqCounters{perfLog->countersJson()[jss::job_queue]}; BEAST_EXPECT(jqCounters.size() == i + 2); @@ -581,7 +581,8 @@ public: // be half as many queued as started... for (int i = 0; i < jobs.size(); ++i) { - perfLog->jobStart(jobs[i].type, microseconds{i + 1}, steady_clock::now(), i * 2); + perfLog->jobStart( + jobs[i].type, jobs[i].typeName, microseconds{i + 1}, steady_clock::now(), i * 2); std::this_thread::sleep_for(microseconds(10)); // Check each jobType counter entry. @@ -623,7 +624,8 @@ public: BEAST_EXPECT(total[jss::running_duration_us] == "0"); } - perfLog->jobStart(jobs[i].type, microseconds{0}, steady_clock::now(), (i * 2) + 1); + perfLog->jobStart( + jobs[i].type, jobs[i].typeName, microseconds{0}, steady_clock::now(), (i * 2) + 1); std::this_thread::sleep_for(microseconds{10}); // Verify that every entry in jobs appears twice in currents. @@ -651,7 +653,7 @@ public: // A number of the computations in this loop care about the // number of jobs that have finished. Make that available. int const finished = ((jobs.size() - i) * 2) - 1; - perfLog->jobFinish(jobs[i].type, microseconds(finished), (i * 2) + 1); + perfLog->jobFinish(jobs[i].type, jobs[i].typeName, microseconds(finished), (i * 2) + 1); std::this_thread::sleep_for(microseconds(10)); json::Value const jqCounters{perfLog->countersJson()[jss::job_queue]}; @@ -697,7 +699,7 @@ public: BEAST_EXPECT(jsonToUInt64(total[jss::running_duration_us]) == runningDur); } - perfLog->jobFinish(jobs[i].type, microseconds(finished + 1), (i * 2)); + perfLog->jobFinish(jobs[i].type, jobs[i].typeName, microseconds(finished + 1), (i * 2)); std::this_thread::sleep_for(microseconds(10)); // Verify that the two jobs we just finished no longer appear in @@ -891,25 +893,25 @@ public: }; // Start an ID that's too large. - perfLog->jobStart(jobType, microseconds{11}, steady_clock::now(), 2); + perfLog->jobStart(jobType, jobTypeName, microseconds{11}, steady_clock::now(), 2); std::this_thread::sleep_for(microseconds{10}); verifyCounters(perfLog->countersJson(), 1, 0, 11, 0); verifyEmptyCurrent(perfLog->currentJson()); // Start a negative ID - perfLog->jobStart(jobType, microseconds{13}, steady_clock::now(), -1); + perfLog->jobStart(jobType, jobTypeName, microseconds{13}, steady_clock::now(), -1); std::this_thread::sleep_for(microseconds{10}); verifyCounters(perfLog->countersJson(), 2, 0, 24, 0); verifyEmptyCurrent(perfLog->currentJson()); // Finish the too large ID - perfLog->jobFinish(jobType, microseconds{17}, 2); + perfLog->jobFinish(jobType, jobTypeName, microseconds{17}, 2); std::this_thread::sleep_for(microseconds{10}); verifyCounters(perfLog->countersJson(), 2, 1, 24, 17); verifyEmptyCurrent(perfLog->currentJson()); // Finish the negative ID - perfLog->jobFinish(jobType, microseconds{19}, -1); + perfLog->jobFinish(jobType, jobTypeName, microseconds{19}, -1); std::this_thread::sleep_for(microseconds{10}); verifyCounters(perfLog->countersJson(), 2, 2, 24, 36); verifyEmptyCurrent(perfLog->currentJson()); diff --git a/src/test/core/JobQueue_test.cpp b/src/test/core/JobQueue_test.cpp index 35167c2e16..4ef521c5e8 100644 --- a/src/test/core/JobQueue_test.cpp +++ b/src/test/core/JobQueue_test.cpp @@ -1,18 +1,310 @@ #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include #include #include +#include #include +#include +#include #include #include +#include +#include +#include +#include +#include +#include #include +#include #include +#include +#include #include +#include +#include namespace xrpl::test { +namespace { + +/** + * A beast::insight::Collector that remembers the last value set on every + * gauge it created, keyed by gauge name. + * + * Needed because the production collectors are write-only sinks: StatsD + * sends over UDP and NullCollector discards. `JobQueue::collect()` assigns + * to the per-job-type gauges, so the only way to assert on the published + * values in-process is to supply a Collector that keeps them. + * + * RecordingCollector (Collector) + * | makeGauge(name) + * v + * RecordingGaugeImpl (GaugeImpl) --writes--> Values (shared, mutex-guarded) + * ^ ^ + * | | + * JobQueue::collect() assigns gaugeValue(name) reads + * + * Only gauges are recorded; counters, events, meters and hooks get inert + * implementations, because no assertion here needs them. The hook handler is + * kept so the test can invoke the collection pass on demand rather than + * waiting for a collector's own timer -- that makes the reads deterministic. + * + * @note Thread-safe: the value map is guarded by its own mutex, because + * `JobQueue::collect()` may run on a different thread from the assertions. + */ +class RecordingCollector : public beast::insight::Collector +{ +public: + /** + * Shared storage so gauge handles outliving the collector stay valid. + */ + struct Values + { + std::mutex mutex; + std::map gauges; + }; + +private: + /** + * A gauge that writes each assigned value into the shared map. + */ + class RecordingGaugeImpl : public beast::insight::GaugeImpl + { + std::shared_ptr values_; + std::string name_; + + public: + RecordingGaugeImpl(std::shared_ptr values, std::string name) + : values_(std::move(values)), name_(std::move(name)) + { + } + + void + set(value_type value) override + { + std::scoped_lock const lock(values_->mutex); + values_->gauges[name_] = value; + } + + void + increment(difference_type amount) override + { + std::scoped_lock const lock(values_->mutex); + values_->gauges[name_] += static_cast(amount); + } + }; + + /** @{ */ + /** + * Inert implementations for the metric kinds no assertion reads. + */ + class InertCounterImpl : public beast::insight::CounterImpl + { + void + increment(value_type) override + { + } + }; + + class InertEventImpl : public beast::insight::EventImpl + { + void + notify(value_type const&) override + { + } + }; + + class InertMeterImpl : public beast::insight::MeterImpl + { + void + increment(value_type) override + { + } + }; + + /** + * A hook that owns its handler, so releasing the Hook releases the + * handler with it. + * + * The collector holds only a weak reference (see `hooks_`). That + * reproduces the documented beast::insight lifetime rule -- "when the + * last reference goes away, the metric is no longer collected" -- and + * matters here because `JobQueue::~JobQueue()` unhooks by assigning an + * empty Hook. A collector holding the handler strongly would keep + * calling back into a destroyed JobQueue. + */ + class InertHookImpl : public beast::insight::HookImpl + { + public: + explicit InertHookImpl(HandlerType handler) : handler_(std::move(handler)) + { + } + + void + invoke() const + { + if (handler_) + handler_(); + } + + private: + HandlerType handler_; + }; + /** @} */ + + std::shared_ptr values_{std::make_shared()}; + std::vector> hooks_; + +public: + beast::insight::Hook + makeHook(beast::insight::HookImpl::HandlerType const& handler) override + { + auto impl = std::make_shared(handler); + hooks_.push_back(impl); + return beast::insight::Hook(std::move(impl)); + } + + beast::insight::Counter + makeCounter(std::string const&) override + { + return beast::insight::Counter(std::make_shared()); + } + + beast::insight::Event + makeEvent(std::string const&) override + { + return beast::insight::Event(std::make_shared()); + } + + beast::insight::Gauge + makeGauge(std::string const& name) override + { + return beast::insight::Gauge(std::make_shared(values_, name)); + } + + beast::insight::Meter + makeMeter(std::string const&) override + { + return beast::insight::Meter(std::make_shared()); + } + + /** + * Run every still-live hook, i.e. force one collection pass. + * + * Expired hooks are skipped rather than resurrected, so calling this + * after the JobQueue has been destroyed is a no-op instead of a + * use-after-free. + */ + void + runHooks() const + { + for (auto const& weak : hooks_) + { + if (auto const hook = weak.lock()) + hook->invoke(); + } + } + + /** + * The last value published for @p name. + * + * @return The value, or std::nullopt when no gauge of that name has + * ever been written -- which distinguishes "gauge absent" from + * "gauge present and reading zero". + */ + [[nodiscard]] std::optional + gaugeValue(std::string const& name) const + { + std::scoped_lock const lock(values_->mutex); + auto const iter = values_->gauges.find(name); + if (iter == values_->gauges.end()) + return std::nullopt; + return iter->second; + } +}; + +/** + * A perf::PerfLog that ignores everything. + * + * JobQueue requires a PerfLog reference; these tests assert on gauges, not + * on the perf hooks, so every override is empty. + */ +class SilentPerfLog : public perf::PerfLog +{ + void + rpcStart(std::string const&, std::uint64_t) override + { + } + void + rpcFinish(std::string const&, std::uint64_t) override + { + } + void + rpcError(std::string const&, std::uint64_t) override + { + } + void + jobQueue(JobType, std::string const&) override + { + } + void + jobStart( + JobType, + std::string const&, + std::chrono::microseconds, + std::chrono::time_point, + int) override + { + } + void + jobFinish(JobType, std::string const&, std::chrono::microseconds, int) override + { + } + [[nodiscard]] json::Value + countersJson() const override + { + return json::Value(); + } + [[nodiscard]] json::Value + currentJson() const override + { + return json::Value(); + } + void + resizeJobs(int) override + { + } + void + rotate() override + { + } +}; + +// Gauge-name suffixes. Aliased from JobTypeData rather than re-spelled, so a +// rename there fails here instead of silently asserting on a stale name. +/** @{ */ +constexpr auto& kSuffixWaiting = JobTypeData::kSuffixWaiting; +constexpr auto& kSuffixRunning = JobTypeData::kSuffixRunning; +constexpr auto& kSuffixDeferred = JobTypeData::kSuffixDeferred; +/** @} */ + +} // namespace + //------------------------------------------------------------------------------ class JobQueue_test : public beast::unit_test::Suite @@ -134,133 +426,286 @@ class JobQueue_test : public beast::unit_test::Suite } } + //-------------------------------------------------------------------------- + // Per-job-type saturation gauges (waiting / running / deferred) + //-------------------------------------------------------------------------- + /** - * The telemetry accessors added for the job-queue saturation gauges. + * Owns a JobQueue wired to a RecordingCollector. * - * These feed `jobq_backlog{metric,job_type}` and `jobq_saturation{metric}`, - * which are polled from an xrpld observable-gauge callback. The values are - * asserted exactly, because the whole point of the signals is that a - * specific count (especially `deferred`) is correct -- a plausible-looking - * number would misreport starvation as health. + * A standalone JobQueue rather than `env.app().getJobQueue()`, for two + * reasons: the application's queue uses a write-only collector whose + * gauge values cannot be read back, and it carries background jobs whose + * timing would make exact counts unreproducible. Constructed with the + * given thread count so a concurrency limit can be exceeded on demand. + */ + struct GaugeFixture + { + Logs logs{beast::Severity::Disabled}; + SilentPerfLog perfLog; + std::shared_ptr collector{std::make_shared()}; + JobQueue queue; + + explicit GaugeFixture(int threadCount) + : queue(threadCount, collector, logs.journal("JobQueue"), logs, perfLog) + { + } + + /** + * Publish one collection pass, then read a gauge by job type. + */ + [[nodiscard]] std::optional + read(JobType type, char const* suffix) const + { + collector->runHooks(); + return collector->gaugeValue(JobTypes::name(type) + suffix); + } + }; + + /** + * A gauge exists for a limited job type and not for a special one. + * + * Creation is observed through the RecordingCollector: a name no gauge + * was created for is absent from its value map even after a collection + * pass, whereas a created one is present. That distinguishes "never + * created" from "created and reading 0", which a value check alone + * cannot. */ void - testTelemetryAccessors() + testGaugeCreation() { - testcase("telemetry occupancy accessors"); + testcase("Saturation gauge creation"); - jtx::Env env{*this}; - JobQueue& jQueue = env.app().getJobQueue(); + GaugeFixture fixture(1); - // --- Every registered type is present, and an idle one reads zero --- - // Absence and zero must be distinguishable: the gauge observes every - // type on every tick, so a missing type would be an exporter bug, not - // an idle queue. - auto const counts = jQueue.getJobTypeCounts(); + // JtLedgerReq has limit 3, so it is not special and must be gauged. + BEAST_EXPECT(!JobTypes::instance().get(JtLedgerReq).special()); + BEAST_EXPECT(JobTypes::instance().get(JtLedgerReq).limit() == 3); + BEAST_EXPECT(fixture.read(JtLedgerReq, kSuffixWaiting).has_value()); + BEAST_EXPECT(fixture.read(JtLedgerReq, kSuffixRunning).has_value()); + BEAST_EXPECT(fixture.read(JtLedgerReq, kSuffixDeferred).has_value()); - // One entry per registered JobType. JobTypes is the registry the - // JobQueue constructor populates jobData_ from, so the sizes must - // agree exactly -- a mismatch means a type is silently unreported. - BEAST_EXPECT(counts.size() == JobTypes::instance().size()); + // A quiescent queue publishes exactly zero, not "no value". + BEAST_EXPECT(fixture.read(JtLedgerReq, kSuffixWaiting) == 0u); + BEAST_EXPECT(fixture.read(JtLedgerReq, kSuffixRunning) == 0u); + BEAST_EXPECT(fixture.read(JtLedgerReq, kSuffixDeferred) == 0u); - auto findType = [&counts](JobType t) { - return std::ranges::find_if(counts, [t](auto const& c) { return c.type == t; }); + // JtPeer is special (limit_ == 0) and must have no gauges at all. + BEAST_EXPECT(JobTypes::instance().get(JtPeer).special()); + BEAST_EXPECT(JobTypes::instance().get(JtPeer).limit() == 0); + BEAST_EXPECT(!fixture.read(JtPeer, kSuffixWaiting).has_value()); + BEAST_EXPECT(!fixture.read(JtPeer, kSuffixRunning).has_value()); + BEAST_EXPECT(!fixture.read(JtPeer, kSuffixDeferred).has_value()); + } + + /** + * Driving a capped type past its limit reports exact running and + * deferred counts, and deferred returns to exactly 0 once drained. + * + * JtPack is used because its limit is 1, which makes both figures + * unambiguous: with the single slot occupied by a job that blocks until + * released, every further submission must defer. + */ + void + testDeferredGaugeExactValues() + { + testcase("Saturation gauge deferred counts"); + + int const limit = JobTypes::instance().get(JtPack).limit(); + BEAST_EXPECT(limit == 1); + + // More threads than the type's limit, so `running` is capped by the + // job type rather than by thread availability. + GaugeFixture fixture(4); + + // The first job blocks inside doJob() until released, holding the + // one slot. `entered` proves it really is running before the + // remaining jobs are submitted. + std::mutex mutex; + std::condition_variable cv; + bool release = false; + int entered = 0; + + auto blockingJob = [&]() { + std::unique_lock lock(mutex); + ++entered; + cv.notify_all(); + cv.wait(lock, [&release] { return release; }); }; - // A sync-critical type is present even though nothing has been - // enqueued for it, and reads exactly zero on all three fields. - auto const ledgerData = findType(JtLedgerData); - BEAST_EXPECT(ledgerData != counts.end()); - if (ledgerData != counts.end()) + BEAST_EXPECT(fixture.queue.addJob(JtPack, "GaugeHold", blockingJob)); { - BEAST_EXPECT(ledgerData->waiting == 0); - BEAST_EXPECT(ledgerData->running == 0); - BEAST_EXPECT(ledgerData->deferred == 0); + std::unique_lock lock(mutex); + BEAST_EXPECT( + cv.wait_for(lock, std::chrono::seconds(10), [&entered] { return entered == 1; })); } - // JtInvalid is NOT a registered type: jobData_ is built from - // JobTypes, whose map excludes it. So the snapshot must not carry it. - BEAST_EXPECT(findType(JtInvalid) == counts.end()); + // Every one of these must defer: the limit is already reached. + int const extra = 4; + for (int i = 0; i < extra; ++i) + BEAST_EXPECT(fixture.queue.addJob(JtPack, "GaugeDefer", blockingJob)); - // --- The per-type snapshot agrees with the existing accessor --- - // getJobTypeCounts() must not re-implement counting: `waiting` is the - // same field getJobCount() returns, and `waiting + running` is what - // getJobCountTotal() returns. Asserted on every type so a divergence - // anywhere fails, not just on the one type a test happens to poke. - for (auto const& count : counts) + // Exact values, not bounds: `running` equals the type's limit and + // `deferred` equals the number of submissions beyond it. Both are + // deterministic here because no JtPack job can complete until + // `release` is set. + BEAST_EXPECT(fixture.read(JtPack, kSuffixRunning) == static_cast(limit)); + BEAST_EXPECT(fixture.read(JtPack, kSuffixDeferred) == static_cast(extra)); + + // `waiting` counts everything submitted and not yet started, which + // is the deferred jobs -- the running one was decremented when + // getNextJob() picked it up. + BEAST_EXPECT(fixture.read(JtPack, kSuffixWaiting) == static_cast(extra)); + + // Cross-check against the public accessors, so a gauge that silently + // published a stale or unrelated number would be caught. + BEAST_EXPECT(fixture.queue.getJobCount(JtPack) == extra); + BEAST_EXPECT(fixture.queue.getJobCountTotal(JtPack) == extra + limit); + + // Release everything and drain. { - BEAST_EXPECT(count.waiting == jQueue.getJobCount(count.type)); - BEAST_EXPECT(count.waiting + count.running == jQueue.getJobCountTotal(count.type)); + std::scoped_lock const lock(mutex); + release = true; + } + cv.notify_all(); + fixture.queue.stop(); + BEAST_EXPECT(fixture.queue.isStopped()); + + // All five jobs ran, so the backlog is gone: deferred is exactly 0, + // and so are waiting and running. + BEAST_EXPECT(entered == extra + limit); + BEAST_EXPECT(fixture.read(JtPack, kSuffixDeferred) == 0u); + BEAST_EXPECT(fixture.read(JtPack, kSuffixWaiting) == 0u); + BEAST_EXPECT(fixture.read(JtPack, kSuffixRunning) == 0u); + } + + /** + * Negative path: an uncapped job type never reports non-zero deferred. + * + * JtClient's limit is std::numeric_limits::max(), so + * `addRefCountedJob()` can never take the `++data.deferred` branch. The + * gauge must therefore read exactly 0 both while jobs are in flight and + * after the queue drains -- otherwise a dashboard would attribute + * backpressure to a type that cannot experience it. + */ + void + testUncappedTypeNeverDefers() + { + testcase("Saturation gauge uncapped type"); + + int const limit = JobTypes::instance().get(JtClient).limit(); + BEAST_EXPECT(limit == std::numeric_limits::max()); + BEAST_EXPECT(!JobTypes::instance().get(JtClient).special()); + + // One thread, so submissions greatly outnumber the workers that can + // service them. Under a capped type this would defer; here it must + // not, which is what separates "waiting" from "deferred". + GaugeFixture fixture(1); + + std::mutex mutex; + std::condition_variable cv; + bool release = false; + int entered = 0; + + int const jobs = 6; + for (int i = 0; i < jobs; ++i) + { + BEAST_EXPECT(fixture.queue.addJob(JtClient, "GaugeUncapped", [&]() { + std::unique_lock lock(mutex); + ++entered; + cv.notify_all(); + cv.wait(lock, [&release] { return release; }); + })); } - // --- Saturation: the pool reports its own capacity --- - auto const saturation = jQueue.getWorkerSaturation(); - - // jtx::Env runs standalone, which the JobQueue ctor maps to exactly - // one worker thread. This is the dashboard ratio's denominator, so it - // must be the real configured count and never zero (a zero would make - // the ratio undefined). - BEAST_EXPECT(saturation.workerThreads == 1); - - // totalWaiting is the sum of the per-type waiting counts from the same - // fields, so the two accessors must agree. - int const summedWaiting = - std::accumulate(counts.begin(), counts.end(), 0, [](int acc, auto const& c) { - return acc + c.waiting; - }); - BEAST_EXPECT(saturation.totalWaiting == summedWaiting); - - // An in-flight task count can never exceed the configured pool size. - BEAST_EXPECT(saturation.runningTasks >= 0); - BEAST_EXPECT(saturation.runningTasks <= saturation.workerThreads); - - // --- A queued job is actually observed --- - // Block one job inside its handler so the queue provably holds work - // while it is sampled: without this the sample could race the job to - // completion and read zeros, which would pass vacuously. - std::atomic release{false}; - std::atomic started{false}; - BEAST_EXPECT(jQueue.addJob(JtClient, "OccupancyBlocker", [&release, &started]() { - started = true; - while (!release) - std::this_thread::yield(); - })); - - while (!started) - std::this_thread::yield(); - - // With the single standalone worker occupied, the type reports exactly - // one job running. - auto const busy = jQueue.getJobTypeCounts(); - auto const busyClient = - std::ranges::find_if(busy, [](auto const& c) { return c.type == JtClient; }); - BEAST_EXPECT(busyClient != busy.end()); - if (busyClient != busy.end()) - BEAST_EXPECT(busyClient->running == 1); - - // And the pool reports exactly one task in flight out of one thread: - // a fully saturated pool, which is the reading the gauge exists for. - auto const busySaturation = jQueue.getWorkerSaturation(); - BEAST_EXPECT(busySaturation.runningTasks == 1); - BEAST_EXPECT(busySaturation.workerThreads == 1); - - release = true; - jQueue.rendezvous(); - - // After draining, the same type reads zero again -- the counters are - // live readings, not a high-water mark. rendezvous() returns with the - // queue mutex having seen finishJob(), so the running count is settled - // by here. (Workers::runningTaskCount_ is decremented only after - // processTask returns, which rendezvous does not wait for, so it is - // deliberately not asserted at this point.) - auto const drained = jQueue.getJobTypeCounts(); - auto const drainedClient = - std::ranges::find_if(drained, [](auto const& c) { return c.type == JtClient; }); - BEAST_EXPECT(drainedClient != drained.end()); - if (drainedClient != drained.end()) + // At least one job is in flight and the rest are backlogged, yet + // deferred stays at exactly 0 because the type has no limit. { - BEAST_EXPECT(drainedClient->waiting == 0); - BEAST_EXPECT(drainedClient->running == 0); + std::unique_lock lock(mutex); + BEAST_EXPECT( + cv.wait_for(lock, std::chrono::seconds(10), [&entered] { return entered >= 1; })); } + BEAST_EXPECT(fixture.read(JtClient, kSuffixDeferred) == 0u); + + { + std::scoped_lock const lock(mutex); + release = true; + } + cv.notify_all(); + fixture.queue.stop(); + + BEAST_EXPECT(entered == jobs); + BEAST_EXPECT(fixture.read(JtClient, kSuffixDeferred) == 0u); + BEAST_EXPECT(fixture.read(JtClient, kSuffixWaiting) == 0u); + BEAST_EXPECT(fixture.read(JtClient, kSuffixRunning) == 0u); + } + + /** + * Exactly the non-special job types are gauged, three gauges each, and + * every published value is non-negative. + * + * The coverage half pins the cardinality the metric family adds: one + * gauge per non-special type per counter and none for special types, so + * a job type gaining or losing a limit shows up here. + * + * The non-negativity half is the observable consequence of the clamp in + * `collect()`. The clamp itself cannot be triggered from a test: it + * guards `waiting` / `running` / `deferred`, which are private to + * JobQueue and only ever incremented and decremented in matched pairs, + * so forcing one negative would need the internals hacked. What is + * assertable is the property the clamp exists to guarantee -- since + * `Gauge::value_type` is unsigned, an unclamped negative would surface + * as a value near 2^64 rather than as a small number, which is exactly + * what the upper bound below rules out. + */ + void + testGaugeCoverageAndNonNegative() + { + testcase("Saturation gauge coverage"); + + GaugeFixture fixture(1); + fixture.collector->runHooks(); + + // Sanity-check the fixture against the job-type table itself, so the + // expected counts are derived rather than hard-coded. + int nonSpecial = 0; + int special = 0; + int gauges = 0; + bool allSmall = true; + + // No job has been submitted, so every published value must be 0. + // The bound is deliberately generous: it is here to catch an + // unsigned wrap, not to re-assert the exact zero above. + auto const kWrapGuard = static_cast(1) << 32; + + for (auto const& [type, info] : JobTypes::instance()) + { + if (type == JtInvalid) + continue; + + info.special() ? ++special : ++nonSpecial; + + for (char const* suffix : {kSuffixWaiting, kSuffixRunning, kSuffixDeferred}) + { + auto const value = fixture.collector->gaugeValue(info.name() + suffix); + + // Presence must agree with speciality, in both directions. + BEAST_EXPECT(value.has_value() == !info.special()); + if (!value) + continue; + + ++gauges; + if (*value >= kWrapGuard) + allSmall = false; + BEAST_EXPECT(*value == 0u); + } + } + + BEAST_EXPECT(nonSpecial == 35); + BEAST_EXPECT(special == 11); + BEAST_EXPECT(gauges == nonSpecial * 3); + BEAST_EXPECT(gauges == 105); + BEAST_EXPECT(allSmall); } public: @@ -269,7 +714,10 @@ public: { testAddJob(); testPostCoro(); - testTelemetryAccessors(); + testGaugeCreation(); + testGaugeCoverageAndNonNegative(); + testDeferredGaugeExactValues(); + testUncappedTypeNeverDefers(); } }; diff --git a/src/test/core/Workers_test.cpp b/src/test/core/Workers_test.cpp index fe3820b84a..102f5df14d 100644 --- a/src/test/core/Workers_test.cpp +++ b/src/test/core/Workers_test.cpp @@ -36,13 +36,14 @@ class PerfLogTest : public PerfLog } void - jobQueue(JobType const type) override + jobQueue(JobType const type, std::string const& name) override { } void jobStart( JobType const type, + std::string const& name, std::chrono::microseconds dur, std::chrono::time_point startTime, int instance) override @@ -50,7 +51,11 @@ class PerfLogTest : public PerfLog } void - jobFinish(JobType const type, std::chrono::microseconds dur, int instance) override + jobFinish( + JobType const type, + std::string const& name, + std::chrono::microseconds dur, + int instance) override { } diff --git a/src/test/overlay/TMGetObjectByHash_test.cpp b/src/test/overlay/TMGetObjectByHash_test.cpp index e579989181..b0e3b70dfa 100644 --- a/src/test/overlay/TMGetObjectByHash_test.cpp +++ b/src/test/overlay/TMGetObjectByHash_test.cpp @@ -20,7 +20,9 @@ #include #include #include +#include #include +#include #include #include @@ -33,6 +35,9 @@ #include #include +#include +#include +#include #include #include @@ -100,6 +105,50 @@ class TMGetObjectByHash_test : public beast::unit_test::Suite return lastSentMessage_; } + /** + * Capture the charge the handler applies, then apply it for real. + * + * `Peer::charge()` is pure virtual and `processGetObjectByHash()` + * calls it unqualified, so this override sees the exact + * `Resource::Charge` the handler built -- the same object passed to + * `computeGetObjectByHashFee()`'s caller. That makes the handler's + * *choice of argument* observable, which reading `fee_` or calling + * the pricing helper with the test's own arguments cannot do. + * + * Recorded before forwarding so the base-class strand hop cannot + * reorder the observation; forwarding keeps the production + * disconnect/accounting behaviour intact. + */ + void + charge(Resource::Charge const& fee, std::string const& context) override + { + lastAppliedCharge_ = fee; + lastChargeContext_ = context; + PeerImp::charge(fee, context); + } + + /** + * The charge captured by the override above, or nullopt if none. + * + * `Resource::Charge` has no default constructor, so the optional + * also distinguishes "not charged at all" from "charged zero" -- + * a distinction the rejection-gate tests depend on. + */ + [[nodiscard]] std::optional const& + getLastAppliedCharge() const + { + return lastAppliedCharge_; + } + + /** + * The context string that accompanied the captured charge. + */ + [[nodiscard]] std::string const& + getLastChargeContext() const + { + return lastChargeContext_; + } + // Synchronous test access to the JobQueue-dispatched processor. // The production path runs this on JtLedgerReq; tests need a // synchronous entry point to inspect the reply via send(). @@ -111,6 +160,25 @@ class TMGetObjectByHash_test : public beast::unit_test::Suite processGetObjectByHash(m); } + // Read the accumulated per-message charge. `currentFeeCharge()` is + // protected on PeerImp; exposed here because it is the one + // deterministic, same-thread witness that a rejection gate fired -- + // `charge()` itself dispatches to the peer's strand. + [[nodiscard]] Resource::Charge + peekFeeCharge() const + { + return currentFeeCharge(); + } + + // The differential-pricing helper, so a test can compare the charge + // applied by the handler against the helper's own result for the + // same inputs. Static and protected on PeerImp. + [[nodiscard]] static Resource::Charge + peekComputeFee(int const requested, int const found) + { + return computeGetObjectByHashFee(requested, found); + } + static void resetId() { @@ -119,12 +187,55 @@ class TMGetObjectByHash_test : public beast::unit_test::Suite private: inline static Peer::id_t id = 0; + + /** + * The last message handed to send(). + * + * @note Not synchronised. When the handler runs on a JobQueue + * worker (the `onMessage()` tests), this is written on that worker + * and read on the test thread, so every such test must call + * `env.app().getJobQueue().rendezvous()` before reading it. The + * rendezvous supplies the happens-before edge: the worker's + * `--processCount_` under `mutex_` in `JobQueue::processTask()` + * releases, and the waiter's predicate acquires the same mutex. + * Tests that drive `runProcessGetObjectByHash()` directly run + * wholly on the test thread and need no rendezvous. + */ std::shared_ptr lastSentMessage_; + + /** + * @see getLastAppliedCharge(). Same threading rules as above. + */ + std::optional lastAppliedCharge_; + + /** + * @see getLastChargeContext(). Same threading rules as above. + */ + std::string lastChargeContext_; }; shared_context context_{makeSslContext("")}; ProtocolVersion protocolVersion_{1, 7}; + /** + * Seed offset for hashes that must NOT be present in the NodeStore. + * + * `createRequest()` stores `sha512Half(i)` for i in [0, numObjects), and + * numObjects can reach kHardMaxReplyNodes. Offsetting well past that + * keeps "unstored" hashes genuinely absent. + */ + static constexpr int kUnstoredHashSeed = 1'000'000; + + /** + * Build a live PeerTest registered with the overlay. + * + * @note `overlay.addActive()` stores only `std::weak_ptr`s + * (`OverlayImpl::peers_`, `ids_` and `list_` are all weak), so it does + * *not* keep the peer alive. The returned `shared_ptr` is the sole + * owner; keep it in scope for the whole test. Safety for the + * JobQueue-dispatched path comes from the job lambda locking its own + * `weak_ptr` plus the `rendezvous()` each such test performs. + */ std::shared_ptr createPeer(jtx::Env& env) { @@ -187,6 +298,27 @@ class TMGetObjectByHash_test : public beast::unit_test::Suite return request; } + /** + * Parse the reply captured by PeerTest::send(). + * + * @return The decoded message, or std::nullopt when nothing was sent. + */ + std::optional + parseReply(std::shared_ptr const& peer) + { + auto const sentMessage = peer->getLastSentMessage(); + if (!sentMessage) + return std::nullopt; + + auto const& buffer = sentMessage->getBuffer(compression::Compressed::Off); + BEAST_EXPECT(buffer.size() > 6); + + // Skip the 6-byte message header (4 size + 2 type). + protocol::TMGetObjectByHash reply; + BEAST_EXPECT(reply.ParseFromArray(buffer.data() + 6, buffer.size() - 6) == true); + return reply; + } + /** * Test that reply is limited to hardMaxReplyNodes when more objects * are requested than the limit allows. @@ -209,28 +341,675 @@ class TMGetObjectByHash_test : public beast::unit_test::Suite peer->runProcessGetObjectByHash(request); // Verify that a reply was sent - auto sentMessage = peer->getLastSentMessage(); - BEAST_EXPECT(sentMessage != nullptr); - - // Parse the reply message - auto const& buffer = sentMessage->getBuffer(compression::Compressed::Off); - - BEAST_EXPECT(buffer.size() > 6); - // Skip the message header (6 bytes: 4 for size, 2 for type) - protocol::TMGetObjectByHash reply; - BEAST_EXPECT(reply.ParseFromArray(buffer.data() + 6, buffer.size() - 6) == true); + auto reply = parseReply(peer); + BEAST_EXPECT(reply.has_value()); + if (!reply) + return; // Verify the reply is limited to expectedReplySize - BEAST_EXPECT(reply.objects_size() == expectedReplySize); + BEAST_EXPECT(reply->objects_size() == expectedReplySize); + } + + //-------------------------------------------------------------------------- + // Request-gate rejection paths + //-------------------------------------------------------------------------- + + /** + * Build a query request with @p numObjects hashes and nothing stored. + * + * Distinct from `createRequest()`, which writes every hash to the + * NodeStore. The rejection gates return before any NodeStore access, so + * storing 12289 objects to test them would cost real time and prove + * nothing. Hashes are derived from the index but need not resolve. + * + * @param numObjects Objects to place in the request. + * @param type Message type; must not be otFETCH_PACK or + * otTRANSACTIONS, both of which are intercepted by + * earlier branches of onMessage(). + */ + static std::shared_ptr + createUnstoredRequest( + int const numObjects, + protocol::TMGetObjectByHash::ObjectType const type = + protocol::TMGetObjectByHash_ObjectType_otLEDGER) + { + auto request = std::make_shared(); + request->set_type(type); + request->set_query(true); + + for (int i = 0; i < numObjects; ++i) + { + // Offset the seed so these hashes cannot collide with the ones + // createRequest() stores, keeping "unstored" unambiguous. + uint256 const hash(xrpl::sha512Half(i + kUnstoredHashSeed)); + auto* object = request->add_objects(); + object->set_hash(hash.data(), hash.size()); + } + return request; + } + + /** + * An oversized request is refused with no reply and an exact fee. + * + * This is the gate that `getobject_rejected_total{reason="oversize"}` + * counts. The counter itself is not readable in-process (see the note + * on run()), so the assertions are on the two observable effects of the + * same early return: no message was sent, and `fee_` holds exactly + * `kFeeInvalidData`. + */ + void + testOversizeRejection() + { + testcase("Oversize Rejection"); + + Env env(*this); + PeerTest::resetId(); + auto peer = createPeer(env); + + // Successful-setup assertion: a fresh peer starts at the trivial + // fee, so the post-condition below can only come from this call. + BEAST_EXPECT(peer->peekFeeCharge().cost() == Resource::kFeeTrivialPeer.cost()); + BEAST_EXPECT(peer->getLastSentMessage() == nullptr); + + int const oversize = static_cast(Tuning::kHardMaxReplyNodes) + 1; + peer->onMessage(createUnstoredRequest(oversize)); + + // State: nothing was replied to, because the gate returns before + // the job is queued. + BEAST_EXPECT(peer->getLastSentMessage() == nullptr); + + // Cause: the charge is exactly the invalid-data fee, not merely + // "some larger fee". The label pins which gate fired -- the + // malformed-ledgerhash gate charges a different constant. + BEAST_EXPECT(peer->peekFeeCharge().cost() == Resource::kFeeInvalidData.cost()); + BEAST_EXPECT(peer->peekFeeCharge().cost() == 400); + BEAST_EXPECT(peer->peekFeeCharge().label() == Resource::kFeeInvalidData.label()); + + // Negative path for the differential charge: the gate returns before + // the handler runs, so computeGetObjectByHashFee() is never reached + // and charge() is never called. Nothing was applied at all -- which + // the optional distinguishes from a zero-cost charge. + BEAST_EXPECT(!peer->getLastAppliedCharge().has_value()); + } + + /** + * Exactly at the limit the request is accepted, so the gate is a strict + * `>` and not `>=`. + * + * Negative control for testOversizeRejection: without it, a gate that + * rejected everything would pass that test. + */ + void + testAtLimitNotRejected() + { + testcase("At Limit Not Rejected"); + + Env env(*this); + PeerTest::resetId(); + auto peer = createPeer(env); + + int const atLimit = static_cast(Tuning::kHardMaxReplyNodes); + peer->onMessage(createUnstoredRequest(atLimit)); + + // Accepted: the request was queued, so the fee is the + // moderate-burden admission charge, not the invalid-data charge. + // + // `fee_.update()` for the admission charge runs on this thread, but + // the enqueued worker also writes the peer's reply. Drain the queue + // before reading anything so the observation cannot race the worker. + env.app().getJobQueue().rendezvous(); + + BEAST_EXPECT(peer->peekFeeCharge().cost() == Resource::kFeeModerateBurdenPeer.cost()); + BEAST_EXPECT(peer->peekFeeCharge().cost() != Resource::kFeeInvalidData.cost()); + BEAST_EXPECT(peer->peekFeeCharge().cost() == 250); + + // The request was in bounds, so the worker ran and replied. Nothing + // was stored, so every lookup missed and the reply is empty. + auto reply = parseReply(peer); + BEAST_EXPECT(reply.has_value()); + if (reply) + BEAST_EXPECT(reply->objects_size() == 0); + + // Positive counterpart to the oversize test: because the handler did + // run, a differential charge was applied, and it is exactly the + // all-miss price for the full request size. + auto const& applied = peer->getLastAppliedCharge(); + BEAST_EXPECT(applied.has_value()); + if (applied) + { + BEAST_EXPECT(applied->cost() == PeerTest::peekComputeFee(atLimit, 0).cost()); + BEAST_EXPECT(applied->cost() == 99176); + } + } + + /** + * A wrong-sized ledgerhash is refused with no reply and an exact fee. + * + * This is the gate `getobject_rejected_total{reason="malformed_ledgerhash"}` + * counts. `stringIsUInt256Sized` requires exactly `uint256::size()` + * bytes, so both a short and a long hash must be refused; a test using + * only one would miss an off-by-one in either direction. + * + * @param hashSize Byte length of the malformed ledgerhash. + */ + void + testMalformedLedgerHashRejection(std::size_t const hashSize) + { + testcase("Malformed LedgerHash Rejection: " + std::to_string(hashSize) + " bytes"); + + BEAST_EXPECT(hashSize != uint256::size()); + + Env env(*this); + PeerTest::resetId(); + auto peer = createPeer(env); + + BEAST_EXPECT(peer->peekFeeCharge().cost() == Resource::kFeeTrivialPeer.cost()); + + // Small in-bounds object count, so this can only be the ledgerhash + // gate: the oversize gate is checked afterwards and cannot fire. + auto request = createUnstoredRequest(1); + request->set_ledgerhash(std::string(hashSize, 'x')); + peer->onMessage(request); + + BEAST_EXPECT(peer->getLastSentMessage() == nullptr); + BEAST_EXPECT(peer->peekFeeCharge().cost() == Resource::kFeeMalformedRequest.cost()); + BEAST_EXPECT(peer->peekFeeCharge().cost() == 200); + BEAST_EXPECT(peer->peekFeeCharge().label() == Resource::kFeeMalformedRequest.label()); + + // Negative path: the gate returns before the handler, so no + // differential charge was ever applied. + BEAST_EXPECT(!peer->getLastAppliedCharge().has_value()); + } + + /** + * A correctly sized ledgerhash passes the gate. + * + * Negative control for testMalformedLedgerHashRejection. + */ + void + testWellFormedLedgerHashAccepted() + { + testcase("Well-Formed LedgerHash Accepted"); + + Env env(*this); + PeerTest::resetId(); + auto peer = createPeer(env); + + auto request = createUnstoredRequest(1); + uint256 const ledgerHash(xrpl::sha512Half(0)); + BEAST_EXPECT(ledgerHash.size() == uint256::size()); + request->set_ledgerhash(ledgerHash.data(), ledgerHash.size()); + peer->onMessage(request); + + // Drain the enqueued worker before observing, as in + // testAtLimitNotRejected. + env.app().getJobQueue().rendezvous(); + + // Not the malformed charge: the request was admitted. + BEAST_EXPECT(peer->peekFeeCharge().cost() == Resource::kFeeModerateBurdenPeer.cost()); + BEAST_EXPECT(peer->peekFeeCharge().cost() != Resource::kFeeMalformedRequest.cost()); + BEAST_EXPECT(peer->peekFeeCharge().cost() == 250); + + // A reply was produced, and it echoes the request's ledgerhash. + auto reply = parseReply(peer); + BEAST_EXPECT(reply.has_value()); + if (reply) + { + BEAST_EXPECT(reply->has_ledgerhash()); + BEAST_EXPECT(reply->ledgerhash() == request->ledgerhash()); + } + } + + //-------------------------------------------------------------------------- + // Hit / miss split and charge + //-------------------------------------------------------------------------- + + /** + * Build a request that interleaves stored and unstored hashes. + * + * One of each is taken in turn until a side runs out, then whichever + * side remains is drained. Interleaving matters: a handler that stopped + * at the first miss would return fewer objects than expected, which a + * stored-then-unstored layout could hide. + * + * Asserts on its own setup, so a miscount here is reported at this call + * rather than as a confusing reply-size failure later. + * + * @param env Environment whose NodeStore receives the writes. + * @param numStored Hashes written to the NodeStore, i.e. hits. + * @param numUnstored Hashes left absent, i.e. misses. + * @param storedHashes Out-param populated with the stored hashes. + * @return The assembled request. + */ + std::shared_ptr + buildInterleavedRequest( + Env& env, + int const numStored, + int const numUnstored, + std::set& storedHashes) + { + auto& nodeStore = env.app().getNodeStore(); + + auto request = std::make_shared(); + request->set_type(protocol::TMGetObjectByHash_ObjectType_otLEDGER); + request->set_query(true); + + int stored = 0; + int unstored = 0; + for (int i = 0; i < numStored + numUnstored; ++i) + { + // Alternate while both remain; then drain whichever is left. + bool const takeStored = + (stored < numStored) && (unstored >= numUnstored || (i % 2) == 0); + + uint256 const hash( + xrpl::sha512Half(takeStored ? stored : unstored + kUnstoredHashSeed)); + + if (takeStored) + { + Blob data(100, static_cast(stored % 256)); + nodeStore.store( + NodeObjectType::Ledger, std::move(data), hash, nodeStore.earliestLedgerSeq()); + BEAST_EXPECT(storedHashes.insert(hash).second); + ++stored; + } + else + { + ++unstored; + } + + auto* object = request->add_objects(); + object->set_hash(hash.data(), hash.size()); + } + + // Setup assertions: the mix is exactly what was asked for. + BEAST_EXPECT(stored == numStored); + BEAST_EXPECT(unstored == numUnstored); + BEAST_EXPECT(storedHashes.size() == static_cast(numStored)); + BEAST_EXPECT(request->objects_size() == numStored + numUnstored); + + return request; + } + + /** + * Every replied object is a distinct hash drawn from @p storedHashes. + * + * Without the distinctness check a handler that returned the same hit + * twice would still satisfy a reply-size assertion. + * + * @param reply The decoded reply. + * @param storedHashes The hashes that were written to the NodeStore. + * @param numStored Expected number of distinct returned hashes. + */ + void + verifyReplyObjects( + protocol::TMGetObjectByHash const& reply, + std::set const& storedHashes, + int const numStored) + { + std::set returned; + for (int i = 0; i < reply.objects_size(); ++i) + { + auto const& obj = reply.objects(i); + BEAST_EXPECT(obj.hash().size() == uint256::size()); + BEAST_EXPECT(returned.insert(uint256::fromRaw(obj.hash())).second); + BEAST_EXPECT(storedHashes.contains(uint256::fromRaw(obj.hash()))); + } + BEAST_EXPECT(returned.size() == static_cast(numStored)); + } + + /** + * A mixed request returns exactly the stored objects and nothing else. + * + * This is the split `getobject_lookups_total{result=hit|miss}` records: + * the handler derives the miss count as `requested - found`, so an + * exact reply size is exactly the hit count the metric would report. + * + * @param numStored Hashes written to the NodeStore before the call. + * @param numUnstored Hashes that will miss. + */ + void + testHitMissSplit(int const numStored, int const numUnstored) + { + testcase( + "Hit/Miss Split: " + std::to_string(numStored) + " stored, " + + std::to_string(numUnstored) + " unstored"); + + Env env(*this); + PeerTest::resetId(); + auto peer = createPeer(env); + + std::set storedHashes; + auto request = buildInterleavedRequest(env, numStored, numUnstored, storedHashes); + + int const requested = numStored + numUnstored; + peer->runProcessGetObjectByHash(request); + + auto reply = parseReply(peer); + BEAST_EXPECT(reply.has_value()); + if (!reply) + return; + + // The exact hit count. Every stored hash is returned and no + // unstored one is, so hits == numStored and the derived miss count + // is exactly numUnstored. + BEAST_EXPECT(reply->objects_size() == numStored); + BEAST_EXPECT(requested - reply->objects_size() == numUnstored); + + verifyReplyObjects(*reply, storedHashes, numStored); + + // Cause: the value recorded as getobject_charge is exactly the + // charge the handler applied, captured by PeerTest::charge(). + auto const& applied = peer->getLastAppliedCharge(); + BEAST_EXPECT(applied.has_value()); + if (!applied) + return; + BEAST_EXPECT(applied->cost() == PeerTest::peekComputeFee(requested, numStored).cost()); + BEAST_EXPECT(applied->label() == "GetObject differential"); + + // These request sizes are all within kFreeObjectsPerRequest, so the + // charge is exactly zero regardless of the split. Asserted rather + // than assumed: it is why this test does not also pin a non-trivial + // fee -- testComputeFeeExactValues covers the billable bands. + BEAST_EXPECT(requested <= static_cast(Tuning::kFreeObjectsPerRequest)); + BEAST_EXPECT(applied->cost() == 0); + + // `fee_` is untouched on this path: the handler charges through + // charge(), never through fee_.update(). + BEAST_EXPECT(peer->peekFeeCharge().cost() == Resource::kFeeTrivialPeer.cost()); + } + + /** + * One pricing case: inputs, the derived expectation, and the literal. + * + * Both expectations are kept. `derived` is written from the Tuning + * constants so a deliberate re-pricing needs one edit; `literal` is the + * number as of this branch so a re-pricing cannot pass unnoticed by + * being self-consistently wrong. + */ + struct FeeCase + { + /** + * Objects the peer asked for. + */ + int requested; + /** + * Objects that resolved in the NodeStore. + */ + int found; + /** + * Expectation computed from the Tuning constants. + */ + int derived; + /** + * The same value as a hard number. + */ + int literal; + /** + * Reported when the case fails. + */ + char const* why; + }; + + /** + * Assert computeGetObjectByHashFee() equals both expectations of a case. + * + * @param fc The case to check. + */ + void + checkFeeCase(FeeCase const& fc) + { + auto const cost = PeerTest::peekComputeFee(fc.requested, fc.found).cost(); + BEAST_EXPECTS(cost == fc.derived, fc.why); + BEAST_EXPECTS(cost == fc.literal, fc.why); + } + + /** + * The Tuning constants the fee expectations are built from. + * + * Verified against their literal values by testComputeFeeExactValues() + * so a silent re-pricing shows up as a failure there rather than as a + * self-consistent but wrong expectation in every case below. + */ + struct FeeConstants + { + int free{static_cast(Tuning::kFreeObjectsPerRequest)}; + int hit{static_cast(Tuning::kCostPerLookupHit)}; + int miss{static_cast(Tuning::kCostPerLookupMiss)}; + int bandSmall{static_cast(Tuning::kCostBandSmall)}; + int bandMedium{static_cast(Tuning::kCostBandMedium)}; + int bandLarge{static_cast(Tuning::kCostBandLarge)}; + int smallMax{static_cast(Tuning::kBandSmallMax)}; + int mediumMax{static_cast(Tuning::kBandMediumMax)}; + }; + + /** + * Every pricing case, in one table. + * + * @param k The Tuning constants to derive expectations from. + */ + static std::vector + makeFeeCases(FeeConstants const& k) + { + return { + // Wholly free: at or below the free allowance nothing is + // billable, so only the small size band applies -- which is 0. + {k.free, k.free, k.bandSmall, 0, "at the free allowance"}, + {0, 0, k.bandSmall, 0, "empty request"}, + {1, 1, k.bandSmall, 0, "one object, one hit"}, + + // All hits, one object past the allowance: one billable hit. + {k.free + 1, k.free + 1, k.hit + k.bandSmall, 1, "one billable hit"}, + + // All misses, one past the allowance: misses are billed first, + // so the single billable object is priced as a miss, not a hit. + {k.free + 1, 0, k.miss + k.bandSmall, 8, "one billable miss"}, + + // Mixed at the small-band edge: 64 requested, 32 found. + // Billable is 64-16 = 48; misses are 32 and all billable, + // leaving 16 billable hits. + {k.smallMax, + 32, + (16 * k.hit) + (32 * k.miss) + k.bandSmall, + 272, + "small-band edge, mixed"}, + + // One past the small band moves to the medium surcharge. + {k.smallMax + 1, + k.smallMax + 1, + ((k.smallMax + 1 - k.free) * k.hit) + k.bandMedium, + 149, + "first medium-band size"}, + + // The medium band's last size, then one past it, which moves to + // the large surcharge. + {k.mediumMax, + k.mediumMax, + ((k.mediumMax - k.free) * k.hit) + k.bandMedium, + 1108, + "last medium-band size"}, + {k.mediumMax + 1, + k.mediumMax + 1, + ((k.mediumMax + 1 - k.free) * k.hit) + k.bandLarge, + 2009, + "first large-band size"}, + + // Clamp: found > requested cannot make the miss count negative, + // so the fee is the same as the all-hit case (1). + {k.free + 1, k.free + 10, k.hit + k.bandSmall, 1, "found exceeds requested"}, + }; + } + + /** + * computeGetObjectByHashFee() returns exactly the documented value. + * + * The metric records the helper's result verbatim, so pinning the helper + * pins what `getobject_charge` reports. + */ + void + testComputeFeeExactValues() + { + testcase("Compute Fee Exact Values"); + + FeeConstants const k; + + // Verify the constants themselves, so a silent re-pricing shows up + // here rather than as a self-consistent but wrong expectation. + BEAST_EXPECT(k.free == 16); + BEAST_EXPECT(k.hit == 1); + BEAST_EXPECT(k.miss == 8); + BEAST_EXPECT(k.bandSmall == 0); + BEAST_EXPECT(k.bandMedium == 100); + BEAST_EXPECT(k.bandLarge == 1000); + BEAST_EXPECT(k.smallMax == 64); + BEAST_EXPECT(k.mediumMax == 1024); + + auto const cases = makeFeeCases(k); + BEAST_EXPECT(cases.size() == 10); + for (auto const& fc : cases) + checkFeeCase(fc); + + // A miss costs strictly more than a hit for the same request size. + // Relational, so it cannot be expressed as a table row. + BEAST_EXPECT( + PeerTest::peekComputeFee(k.free + 1, 0).cost() > + PeerTest::peekComputeFee(k.free + 1, k.free + 1).cost()); + + // The label is fixed, so a charge can be attributed to this helper. + BEAST_EXPECT(PeerTest::peekComputeFee(k.free + 1, 0).label() == "GetObject differential"); + } + + /** + * The charge the handler *applies* is priced on `requested`, not on the + * capped iteration count and not on `found`. + * + * Load-bearing because `getobject_charge` records the applied value: if + * `processGetObjectByHash()` priced on `iterLimit` -- i.e. + * `min(requested, kHardMaxReplyNodes)` -- the metric would under-report + * abusive batches by exactly the overshoot. + * + * The assertion is on `PeerTest::charge()`, which overrides the virtual + * the handler calls, so it observes the very `Resource::Charge` object + * the handler constructed. Two other candidate witnesses were rejected: + * - `fee_` / `peekFeeCharge()`: this path never touches `fee_`, it + * goes through `charge()`. + * - `usage_.balance()`: `Entry::add()` returns + * `localBalance.add(...) + remoteBalance` and `DecayingSample::add()` + * returns `value_ / Window` with `Window == kDecayWindowSeconds == + * 32`, so the balance is the applied cost divided by 32 with integer + * truncation. 99184 / 32 and 99176 / 32 are both 3099, so the + * balance cannot distinguish the mutation this test exists to catch. + * It also decays with wall-clock time (`BasicSecondsClock`), making + * any exact expectation racy. + */ + void + testChargeUsesRequestedCount() + { + testcase("Charge Uses Requested Count"); + + Env env(*this); + PeerTest::resetId(); + auto peer = createPeer(env); + + int const requested = static_cast(Tuning::kHardMaxReplyNodes) + 1; + int const capped = static_cast(Tuning::kHardMaxReplyNodes); + + // Successful-setup assertion: nothing has been charged yet, so the + // post-condition below can only come from the handler call. + BEAST_EXPECT(!peer->getLastAppliedCharge().has_value()); + + // No hashes are stored, so every lookup misses and `found` is 0. + // Called directly, so the handler and the charge both run on this + // thread: `charge()` dispatches to strand_, and boost's strand + // `dispatch` runs the function inline when the caller is not already + // in the strand and the strand is idle. The capture in the override + // happens before that hop regardless, so the observation is + // deterministic either way. + peer->runProcessGetObjectByHash(createUnstoredRequest(requested)); + + auto reply = parseReply(peer); + BEAST_EXPECT(reply.has_value()); + if (!reply) + return; + BEAST_EXPECT(reply->objects_size() == 0); + + // State: a charge was applied at all. + auto const& applied = peer->getLastAppliedCharge(); + BEAST_EXPECT(applied.has_value()); + if (!applied) + return; + + // Cause: it is exactly the requested-count price. This is the + // assertion the test is named for. Under either plausible + // mis-pricing it reads a different number and therefore fails: + // priced on `iterLimit` (12288) -> 99176 + // priced on `found` (0) -> 0, since billable clamps to 0 + // and the band drops to Small + BEAST_EXPECT(applied->cost() == 99184); + BEAST_EXPECT(applied->cost() == PeerTest::peekComputeFee(requested, 0).cost()); + BEAST_EXPECT(applied->cost() != PeerTest::peekComputeFee(capped, 0).cost()); + + // Attribution: the charge came from the differential helper, not + // from one of the flat admission or rejection constants. + BEAST_EXPECT(applied->label() == "GetObject differential"); + BEAST_EXPECT(peer->getLastChargeContext() == "processed get object by hash request"); + + // `fee_` is untouched on this path, which is why the override above + // exists rather than a peekFeeCharge() assertion. + BEAST_EXPECT(peer->peekFeeCharge().cost() == Resource::kFeeTrivialPeer.cost()); + + // Pricing on the requested count is strictly more expensive than + // pricing on the capped count, which is what makes the choice + // observable at all. + BEAST_EXPECT( + PeerTest::peekComputeFee(requested, 0).cost() > + PeerTest::peekComputeFee(capped, 0).cost()); + + // Exact values for both, so a change to either input is caught. + BEAST_EXPECT(PeerTest::peekComputeFee(requested, 0).cost() == 99184); + BEAST_EXPECT(PeerTest::peekComputeFee(capped, 0).cost() == 99176); } void run() override { + // NOTE ON METRIC COVERAGE. The five getobject_* instruments are + // recorded through the XRPL_METRIC_* macros, which push into the + // OpenTelemetry SDK. That API is write-only by design -- there is no + // read-back accessor and no in-memory metric reader in this build -- + // and a default jtx::Env leaves telemetry disabled, so the macros do + // not execute at all here. These tests therefore assert the + // observable behaviour of each instrumented code path, which pins + // the values the instruments are fed: + // getobject_request_objects <- the request's objects_size() + // getobject_lookups_total <- reply size (hits) and the derived + // miss count, per testHitMissSplit + // getobject_charge <- the applied Resource::Charge, + // captured by PeerTest::charge() + // getobject_rejected_total <- the two gates' exact fee_ values + // plus "no charge was applied" + // Only getobject_lookup_us has no in-process witness, being a wall + // clock reading. The counter and histogram values themselves remain + // unverified by unit test and are checked live against Prometheus + // per the design's live-validation step. int const limit = static_cast(Tuning::kHardMaxReplyNodes); testReplyLimit(limit + 1, limit); testReplyLimit(limit, limit); testReplyLimit(limit - 1, limit - 1); + + testOversizeRejection(); + testAtLimitNotRejected(); + testMalformedLedgerHashRejection(uint256::size() - 1); + testMalformedLedgerHashRejection(uint256::size() + 1); + testMalformedLedgerHashRejection(0); + testWellFormedLedgerHashAccepted(); + + testHitMissSplit(5, 3); + testHitMissSplit(0, 4); + testHitMissSplit(4, 0); + + testComputeFeeExactValues(); + testChargeUsesRequestedCount(); } }; diff --git a/src/tests/libxrpl/telemetry/MetricMacros.cpp b/src/tests/libxrpl/telemetry/MetricMacros.cpp index 9b661186d1..fae442cbf0 100644 --- a/src/tests/libxrpl/telemetry/MetricMacros.cpp +++ b/src/tests/libxrpl/telemetry/MetricMacros.cpp @@ -29,9 +29,7 @@ #include #include -#include #include -#include #include #include @@ -57,7 +55,6 @@ #include #include #include -#include using namespace xrpl; @@ -1617,135 +1614,22 @@ TEST(MetricMacros, acquire_counters_emit_nothing_when_registry_disabled) // ----------------------------------------------------------------- // JobQueue saturation diagnostics (WP-A4). // -// Asserts the EXACT values and label shapes of the two gauges: -// jobq_backlog{metric,job_type} MetricsRegistry::registerJobQueueBacklogGauge -// waiting / running / deferred, per type +// Asserts the EXACT values and label shapes of the pool-wide gauge: // jobq_saturation{metric} MetricsRegistry::registerJobQueueSaturationGauge // running_tasks / worker_threads / total_waiting // -// Both are observable instruments registered directly on the SDK meter, +// It is an observable instrument registered directly on the SDK meter, // mirroring the production callback shape, because the real MetricsRegistry's // enabled path cannot be linked into this standalone binary (see the file -// header). The snapshot types are the REAL JobQueue::JobTypeCount and -// JobQueue::WorkerSaturation, and the label values come from the real -// JobTypes::name(), so a rename or reorder on either side breaks these tests -// instead of silently drifting from production. +// header). The reading type is the REAL JobQueue::WorkerSaturation, so a +// rename or reorder on either side breaks this test instead of silently +// drifting from production. +// +// The per-job-type waiting/running/deferred counts are a separate mechanism: +// JobQueue::collect() publishes them as the beast::insight gauges +// jobq__waiting / _running / _deferred, which JobQueue_test covers. // ----------------------------------------------------------------- -// jobq_backlog must keep waiting, running and deferred on separate series per -// job type. `deferred` is the reason this gauge exists: a job held back by its -// type's concurrency limit is counted in neither of the other two fields, and -// appears in no other metric at all. The values chosen are a starved -// JtLedgerData -- limit 3, so 3 running and the rest deferred. -TEST(MetricMacros, jobq_backlog_gauge_separates_waiting_running_and_deferred) -{ - CollectingProvider const provider; - - // The real snapshot type the production callback iterates. JtLedgerData is - // at its limit of 3 with 5 more jobs held back; JtLedgerReq has one job - // merely waiting; JtSweep is registered but idle. - std::vector observed{ - JobQueue::JobTypeCount{.type = JtLedgerData, .waiting = 5, .running = 3, .deferred = 5}, - JobQueue::JobTypeCount{.type = JtLedgerReq, .waiting = 1, .running = 0, .deferred = 0}, - JobQueue::JobTypeCount{.type = JtSweep, .waiting = 0, .running = 0, .deferred = 0}}; - - // Keep the instrument alive for the whole test: destroying the handle - // deregisters the callback, which is why the real registry holds a member. - auto gauge = provider.meter()->CreateInt64ObservableGauge( - telemetry::metric::jobqBacklog, - "JobQueue occupancy per job type (waiting/running/deferred)"); - gauge->AddCallback( - [](opentelemetry::metrics::ObserverResult result, void* state) { - auto const* counts = static_cast const*>(state); - // Same two-label Observe() form the production callback uses. - auto observe = [&](char const* field, std::string const& jobType, std::int64_t value) { - opentelemetry::nostd::get>>(result) - ->Observe( - value, - {{telemetry::label::metric, field}, {telemetry::label::jobType, jobType}}); - }; - for (auto const& count : *counts) - { - // The same name helper production uses -- never a literal. - auto const& jobType = JobTypes::name(count.type); - observe("waiting", jobType, count.waiting); - observe("running", jobType, count.running); - observe("deferred", jobType, count.deferred); - } - }, - &observed); - - auto const starved = provider.collect(); - - // Three types x three fields, every one its own series: no field and no - // type collapses into another. - ASSERT_EQ(starved.at("jobq_backlog").size(), 9u); - - // The starved type, exactly as configured. The limit of 3 is visible as - // running=3, and the 5 jobs the limit is denying are the deferred series. - EXPECT_EQ( - gaugeValue(starved, "jobq_backlog", attrs("metric", "waiting", "job_type", "ledgerData")), - 5); - EXPECT_EQ( - gaugeValue(starved, "jobq_backlog", attrs("metric", "running", "job_type", "ledgerData")), - 3); - EXPECT_EQ( - gaugeValue(starved, "jobq_backlog", attrs("metric", "deferred", "job_type", "ledgerData")), - 5); - - // A type that is queued but NOT deferred reads deferred=0 while waiting=1. - // This is the distinction the gauge exists to make: "queued" and "denied a - // worker" are different states, and only the latter is starvation. - EXPECT_EQ( - gaugeValue( - starved, "jobq_backlog", attrs("metric", "waiting", "job_type", "ledgerRequest")), - 1); - EXPECT_EQ( - gaugeValue( - starved, "jobq_backlog", attrs("metric", "deferred", "job_type", "ledgerRequest")), - 0); - - // An idle registered type reports zeros rather than dropping out. Absence - // would be indistinguishable from a broken exporter, so every type is - // observed on every tick. - ASSERT_EQ( - starved.at("jobq_backlog").count(attrs("metric", "waiting", "job_type", "sweep")), 1u); - EXPECT_EQ( - gaugeValue(starved, "jobq_backlog", attrs("metric", "waiting", "job_type", "sweep")), 0); - - // Exactly two label keys, in the documented order, on every series. A - // third label would multiply the series count per job type. - for (auto const& [labels, point] : starved.at("jobq_backlog")) - { - ASSERT_EQ(labels.size(), 2u); - EXPECT_EQ(labels.count("metric"), 1u); - EXPECT_EQ(labels.count("job_type"), 1u); - } - - // NEGATIVE: a type never present in the snapshot has no series, so the - // readings above are not an artifact of a catch-all series. - EXPECT_EQ( - starved.at("jobq_backlog").count(attrs("metric", "waiting", "job_type", "transaction")), - 0u); - // NEGATIVE: the label VALUE is the JobTypes name, not the enum spelling. - EXPECT_EQ( - starved.at("jobq_backlog").count(attrs("metric", "waiting", "job_type", "JtLedgerData")), - 0u); - - // The starvation clearing is the recovery reading: deferred drains to 0 - // while running stays at the limit, so the panel shows work flowing again. - observed[0] = - JobQueue::JobTypeCount{.type = JtLedgerData, .waiting = 0, .running = 3, .deferred = 0}; - auto const draining = provider.collect(); - EXPECT_EQ( - gaugeValue(draining, "jobq_backlog", attrs("metric", "deferred", "job_type", "ledgerData")), - 0); - EXPECT_EQ( - gaugeValue(draining, "jobq_backlog", attrs("metric", "running", "job_type", "ledgerData")), - 3); -} - // jobq_saturation exports the worker-thread count alongside the in-flight // count so a dashboard can form the ratio without hardcoding a denominator // that is derived at startup. The values chosen are a fully exhausted pool. diff --git a/src/tests/libxrpl/telemetry/MetricsRegistry.cpp b/src/tests/libxrpl/telemetry/MetricsRegistry.cpp index 8453e191e4..275273e737 100644 --- a/src/tests/libxrpl/telemetry/MetricsRegistry.cpp +++ b/src/tests/libxrpl/telemetry/MetricsRegistry.cpp @@ -1,5 +1,20 @@ /** - * GTest unit tests for MetricsRegistry (no-op / telemetry-disabled path). + * GTest unit tests for MetricsRegistry. + * + * Two independent groups, split by what they can link: + * + * 1. sanitiseHandler() — the `handler` label sanitiser. Runs in **both** + * builds. sanitiseHandler() is a public static constexpr defined inline + * in the header, so it needs no part of MetricsRegistry.cpp on the link + * line. These tests therefore sit outside the guard below; putting them + * inside it would silently compile them out of the telemetry-enabled + * build, which is the build that actually exports the label. + * + * 2. The no-op / telemetry-disabled path — construction, start()/stop() + * lifecycle, and the synchronous record*() methods. Guarded, because + * when XRPL_ENABLE_TELEMETRY is defined MetricsRegistry.cpp is not + * compiled into this binary (see src/tests/libxrpl/CMakeLists.txt) and + * its out-of-line symbols are unresolvable here. * * Tests cover: * - Construction with telemetry disabled (no-op behavior). @@ -20,7 +35,7 @@ * CONSEQUENCE for the sync-diagnostics gauges (`unl_quorum`, * `clock_close_offset_seconds`, `sync_state`, * `server_stall_events_total`, `sync_acquire`, `shamap_cache_hit_rate`, - * `jobq_backlog`, `jobq_saturation`, `peer_ledger_supply`, + * `jobq_saturation`, `peer_ledger_supply`, * `peerfinder_slot_census`, `amendment_block`, `nodestore_latency`): * this file CANNOT assert an observed gauge * value, because on this build the gauges do not exist -- their registration @@ -37,12 +52,280 @@ * enabled. */ +#include + +#include + +#include +#include +#include +#include + +namespace { + +using xrpl::telemetry::MetricsRegistry; + +/** + * Every job name reaching the JobQueue in non-test code that + * sanitiseHandler() must return unchanged, i.e. every one consisting solely + * of ASCII letters. + * + * How to re-derive this set (it is read off the source, not off docs): + * + * 1. Enumerate the four surfaces that put a name into the JobQueue, + * excluding `src/test/` and `src/tests/`: + * - `JobQueue::addJob` + * - `JobQueue::postCoro` (its `name` becomes `Coro::name_`, which + * `Coro::post()` hands to `addRefCountedJob`) + * - `LedgerMaster::newPFWork`, a thin `addJob` wrapper + * - each `TimeoutCounter` subclass's `.jobName =` designated + * initialiser, consumed by `TimeoutCounter::queueJob()`'s `addJob` + * `addRefCountedJob` is private and has only those callers, so the four + * surfaces close the set. + * 2. Resolve each `name` argument to the literal(s) it can hold. Most are + * literals written in place, but four call sites pass a variable and + * must be traced back: + * - `PeerImp.cpp` passes a local `std::string const name = + * isTrusted ? "ChkTrust" : "ChkUntrust";`, so one call site + * contributes *two* names. Neither literal appears at an `addJob` + * call site, which is why a grep of `addJob` alone misses them. + * - `LedgerMaster::newPFWork` forwards its own `char const* name` + * parameter; its three callers supply the three `PthFind*` literals. + * - `TimeoutCounter` forwards `queueJobParameter_.jobName`, set by a + * `.jobName =` designated initialiser in each of its five + * subclasses. + * - `Coro::post()` forwards `name_`, set from `postCoro`'s argument. + * 3. Keep the names that satisfy sanitiseHandler()'s rule — non-empty and + * all ASCII letters. Everything else belongs in kFoldToOtherHandlers. + * + * Note what step 3 excludes. Two call sites compose the name at runtime from + * a literal prefix and a ledger sequence: `"Pub" + std::to_string(seq)` and + * `"OB" + std::to_string(seq % 1000000000)`. The bare prefixes `"Pub"` and + * `"OB"` are all letters, but they are never what reaches the JobQueue — the + * *composed* form is, and it always carries digits. So they are absent here + * and their composed forms appear in kFoldToOtherHandlers instead. + * + * `JobQueue::makeLoadEvent` is deliberately out of scope: its `name` feeds + * `LoadEvent`/`LoadMonitor`, neither of which reaches MetricsRegistry, so + * names like `"cmd:" + method` never become a `handler` label value. + * + * Asserting on the real list is the point of the test: it proves the + * cardinality bound holds for the names actually in the binary, so a job + * added later whose name breaks the rule shows up as a failure here rather + * than as an unexplained `other` bucket on a dashboard. + */ +constexpr std::array kPassThroughHandlers = { + std::string_view{"AcceptLedger"}, // RCLConsensus.cpp + std::string_view{"AcqDone"}, // InboundLedger.cpp + std::string_view{"AdvanceLedger"}, // LedgerMaster.cpp + std::string_view{"ChkTrust"}, // PeerImp.cpp (local, ternary) + std::string_view{"ChkUntrust"}, // PeerImp.cpp (local, ternary) + std::string_view{"ComplAcquire"}, // TransactionAcquire.cpp + std::string_view{"DoTxs"}, // PeerImp.cpp + std::string_view{"GotFetchPack"}, // LedgerMaster.cpp + std::string_view{"GotStaleData"}, // InboundLedgers.cpp + std::string_view{"HandleHaveTxs"}, // PeerImp.cpp + std::string_view{"HistTxStream"}, // NetworkOPs.cpp + std::string_view{"InboundLedger"}, // InboundLedger.cpp (.jobName) + std::string_view{"LedReplDelta"}, // LedgerDeltaAcquire.cpp (.jobName) + std::string_view{"LedReplTask"}, // LedgerReplayTask.cpp (.jobName) + std::string_view{"MakeFetchPack"}, // PeerImp.cpp + std::string_view{"NObjStore"}, // NodeStoreScheduler.cpp + std::string_view{"NetCluster"}, // NetworkOPs.cpp + std::string_view{"NetHeart"}, // NetworkOPs.cpp + std::string_view{"OnLedBuilt"}, // LedgerDeltaAcquire.cpp + std::string_view{"ProcessLData"}, // InboundLedgers.cpp + std::string_view{"PthFindNewLed"}, // LedgerMaster.cpp (newPFWork) + std::string_view{"PthFindNewReq"}, // LedgerMaster.cpp (newPFWork) + std::string_view{"PthFindOBDB"}, // LedgerMaster.cpp (newPFWork) + std::string_view{"PubCons"}, // NetworkOPs.cpp + std::string_view{"PubFee"}, // NetworkOPs.cpp + std::string_view{"RPCSubSendThr"}, // RPCSub.cpp + std::string_view{"RcvCheckTx"}, // PeerImp.cpp + std::string_view{"RcvGetLedger"}, // PeerImp.cpp + std::string_view{"RcvGetObjByHash"}, // PeerImp.cpp + std::string_view{"RcvManifests"}, // PeerImp.cpp + std::string_view{"RcvPeerData"}, // PeerImp.cpp + std::string_view{"RcvProofPReq"}, // PeerImp.cpp + std::string_view{"RcvReplDReq"}, // PeerImp.cpp + std::string_view{"SkipListAcq"}, // SkipListAcquire.cpp (.jobName) + std::string_view{"SubmitTxn"}, // NetworkOPs.cpp + std::string_view{"TryFill"}, // LedgerMaster.cpp + std::string_view{"TxAcq"}, // TransactionAcquire.cpp (.jobName) + std::string_view{"TxBatchAsync"}, // NetworkOPs.cpp + std::string_view{"TxBatchSync"}, // NetworkOPs.cpp + std::string_view{"TxsToTxn"}, // ConsensusTransSetSF.cpp + std::string_view{"WAL"}, // SociDB.cpp + std::string_view{"checkPropose"}, // PeerImp.cpp (lowercase start) + std::string_view{"sweep"}, // Application.cpp (all lowercase) +}; + +/** + * Names that must fold to kHandlerOther. + * + * The first two are the composed forms of the only two dynamically built job + * names in the tree: `"Pub" + std::to_string(ledger->seq())` + * (LedgerPersistence.cpp) and + * `"OB" + std::to_string(ledger->seq() % 1000000000)` (OrderBookDBImpl.cpp). + * They are the reason the sanitiser exists — used raw they would mint a + * Prometheus series per ledger — so realistic sequence values are used + * rather than short placeholders. The sequence is unbounded at the `"Pub"` + * site and masked to nine digits at the `"OB"` site, but a digit appears + * either way (even `seq == 0` gives `"Pub0"`), so no reachable input at + * either site can produce an all-letter name. + * + * The next five are static literals that already fail the rule today, so + * the fallback is exercised by real code and not only by synthetic input. + * + * The remainder are the edge cases: empty, and one entry per disallowed + * character class (space, digit, underscore, hyphen, non-ASCII byte). The + * non-ASCII entry is UTF-8 'e-acute'; on a signed-char platform its lead + * byte is negative, which the explicit ASCII range check rejects where a + * locale-sensitive std::isalpha might not. + */ +constexpr std::array kFoldToOtherHandlers = { + std::string_view{"Pub97531234"}, // dynamic: "Pub" + ledger seq + std::string_view{"OB123456789"}, // dynamic: "OB" + ledger seq % 1e9 + std::string_view{"GetConsL1"}, // static, digit (RCLConsensus.cpp) + std::string_view{"GetConsL2"}, // static, digit (RCLValidations.cpp) + std::string_view{"gRPC-Client"}, // static, hyphen (GRPCServer.cpp) + std::string_view{"RPC-Client"}, // static, hyphen (ServerHandler.cpp) + std::string_view{"WS-Client"}, // static, hyphen (ServerHandler.cpp) + std::string_view{""}, // empty + std::string_view{"Rcv Ledger"}, // space + std::string_view{"Handler7"}, // digit + std::string_view{"Rcv_Ledger"}, // underscore + std::string_view{"Rcv-Ledger"}, // hyphen + std::string_view{"caf\xC3\xA9"}, // non-ASCII byte (UTF-8 e-acute) +}; + +/** + * True when sanitiseHandler() returns each pass-through name unchanged. + * + * consteval so a regression is a compile error rather than a test failure: + * the sanitiser is constexpr precisely so this bound can be checked without + * running anything. + */ +consteval bool +allPassThroughUnchanged() +{ + for (auto const name : kPassThroughHandlers) + { + if (MetricsRegistry::sanitiseHandler(name) != name) + return false; + } + return true; +} + +/** + * True when sanitiseHandler() maps every listed name to kHandlerOther. + */ +consteval bool +allFoldToOther() +{ + for (auto const name : kFoldToOtherHandlers) + { + if (MetricsRegistry::sanitiseHandler(name) != MetricsRegistry::kHandlerOther) + return false; + } + return true; +} + +// Compile-time guarantees. Duplicated at runtime below so a failure names +// the offending input instead of only pointing at the assertion. +static_assert(allPassThroughUnchanged()); +static_assert(allFoldToOther()); + +// The verified size of the pass-through set as of this branch: 43 all-letter +// job-name literals. Pinned so that adding or removing a job name without +// revisiting the label-cardinality budget fails the build here. +static_assert(kPassThroughHandlers.size() == 43); + +/** + * Total distinct `handler` label values reachable from the inputs above: + * one per pass-through name plus the single shared kHandlerOther bucket. + * This is the number the Prometheus cardinality budget is sized against. + */ +constexpr std::size_t kExpectedHandlerDomain = kPassThroughHandlers.size() + 1; +static_assert(kExpectedHandlerDomain == 44); + +} // namespace + +TEST(MetricsRegistrySanitiseHandler, static_job_names_pass_through_unchanged) +{ + // Every all-letter job name in the tree survives sanitisation, so the + // `handler` label keeps its attribution value for real producers. + for (auto const name : kPassThroughHandlers) + { + EXPECT_EQ(MetricsRegistry::sanitiseHandler(name), name) + << "job name should pass through unchanged: " << name; + // Cause, not just state: it passed because it is not the fallback. + EXPECT_NE(MetricsRegistry::sanitiseHandler(name), MetricsRegistry::kHandlerOther) + << "job name wrongly folded to the fallback: " << name; + } +} + +TEST(MetricsRegistrySanitiseHandler, dynamic_and_non_letter_names_fold_to_other) +{ + // Negative path: everything that is not an all-letter name collapses + // into exactly one bucket, which is what bounds the label domain. + for (auto const name : kFoldToOtherHandlers) + { + EXPECT_EQ(MetricsRegistry::sanitiseHandler(name), MetricsRegistry::kHandlerOther) + << "name should fold to the fallback: " << name; + } +} + +TEST(MetricsRegistrySanitiseHandler, empty_name_folds_to_other) +{ + // Called out separately because it is the one case the all-letter scan + // cannot catch: std::ranges::all_of() is vacuously true on an empty + // range, so the sanitiser needs its own emptiness check. + EXPECT_EQ(MetricsRegistry::sanitiseHandler(std::string_view{}), MetricsRegistry::kHandlerOther); + EXPECT_EQ(MetricsRegistry::sanitiseHandler(""), MetricsRegistry::kHandlerOther); +} + +TEST(MetricsRegistrySanitiseHandler, fallback_value_is_the_shared_constant) +{ + // The fallback must be the constant the dashboards and the reference doc + // are written against, not merely some non-empty string. + EXPECT_EQ(MetricsRegistry::kHandlerOther, std::string_view{"other"}); + EXPECT_EQ(MetricsRegistry::kHandlerOther.size(), 5u); + + // "other" is itself all letters, so sanitising it is idempotent -- a + // handler genuinely named "other" is indistinguishable from the bucket. + EXPECT_EQ( + MetricsRegistry::sanitiseHandler(MetricsRegistry::kHandlerOther), + MetricsRegistry::kHandlerOther); +} + +TEST(MetricsRegistrySanitiseHandler, output_domain_is_exactly_44_values) +{ + // The cardinality bound itself: over every input above -- 43 real job + // names, 2 dynamic names, 5 non-conforming static names and 6 edge + // cases -- the sanitiser can emit only 44 distinct label values (43 + // names plus the single "other" bucket). + std::set domain; + for (auto const name : kPassThroughHandlers) + domain.insert(MetricsRegistry::sanitiseHandler(name)); + for (auto const name : kFoldToOtherHandlers) + domain.insert(MetricsRegistry::sanitiseHandler(name)); + + EXPECT_EQ(domain.size(), kExpectedHandlerDomain); + EXPECT_EQ(domain.size(), 44u); + + // State plus cause: the domain is the pass-through names and nothing + // else besides the one fallback bucket. + EXPECT_TRUE(domain.contains(MetricsRegistry::kHandlerOther)); + EXPECT_EQ(domain.size() - 1, kPassThroughHandlers.size()); + for (auto const name : kPassThroughHandlers) + EXPECT_TRUE(domain.contains(name)) << "missing from domain: " << name; +} + // When telemetry is globally enabled, MetricsRegistry.cpp requires xrpld // link dependencies we cannot satisfy in a standalone GTest binary. #ifndef XRPL_ENABLE_TELEMETRY -#include - #include #include #include @@ -50,8 +333,6 @@ #include -#include - #include #include #include @@ -364,9 +645,9 @@ TEST_F(MetricsRegistryTest, disabled_recording_methods) registry.recordRpcStarted("server_info"); registry.recordRpcFinished("server_info", 1000); registry.recordRpcErrored("ledger", 500); - registry.recordJobQueued("ledgerData"); - registry.recordJobStarted("ledgerData", 200); - registry.recordJobFinished("ledgerData", 3000); + registry.recordJobQueued("ledgerData", "ProcessLData"); + registry.recordJobStarted("ledgerData", "ProcessLData", 200); + registry.recordJobFinished("ledgerData", "ProcessLData", 3000); registry.stop(); } @@ -388,8 +669,8 @@ TEST_F(MetricsRegistryTest, destructor_calls_stop) // `clock_close_offset_seconds` reads TimeKeeper::closeOffset(); `sync_state` and // `server_stall_events_total` read NetworkOPs and LoadManager; `sync_acquire` // reads InboundLedgers::acquireProgress() and `shamap_cache_hit_rate` reads the -// node Family's tree-node cache; `jobq_backlog` and `jobq_saturation` read -// JobQueue::getJobTypeCounts() / getWorkerSaturation(); `peer_ledger_supply` and +// node Family's tree-node cache; `jobq_saturation` reads +// JobQueue::getWorkerSaturation(); `peer_ledger_supply` and // `peerfinder_slot_census` read Overlay::getPeerLedgerSupply() / // getSlotCensus() and `amendment_block` reads // AmendmentTable::firstUnsupportedExpected(). All are @@ -469,10 +750,9 @@ TEST_F(MetricsRegistryTest, disabled_lifecycle_never_consults_gauge_services) // Family's tree-node cache. Neither was consulted above. EXPECT_THROW(mockApp_.getInboundLedgers(), std::logic_error); EXPECT_THROW(mockApp_.getNodeFamily(), std::logic_error); - // The service both WP-A4 job-queue gauges read: jobq_backlog polls - // getJobTypeCounts() and jobq_saturation polls getWorkerSaturation(), - // both on the JobQueue. Neither was consulted above, so neither gauge - // took the JobQueue mutex on a telemetry-off build. + // The service the WP-A4 job-queue gauge reads: jobq_saturation polls + // getWorkerSaturation() on the JobQueue. Not consulted above, so the + // gauge never took the JobQueue mutex on a telemetry-off build. EXPECT_THROW(mockApp_.getJobQueue(), std::logic_error); // The service both WP-A7 peer gauges read: peer_ledger_supply polls // getPeerLedgerSupply(), which walks the active-peer list, and diff --git a/src/xrpld/overlay/detail/PeerImp.cpp b/src/xrpld/overlay/detail/PeerImp.cpp index 98132100be..bc7f44e2bd 100644 --- a/src/xrpld/overlay/detail/PeerImp.cpp +++ b/src/xrpld/overlay/detail/PeerImp.cpp @@ -1,3 +1,7 @@ +// cspell:ignore ISTOGRAM +// The all-caps macro name XRPL_METRIC_HISTOGRAM_RECORD trips cspell's +// compound-word splitter, which emits the subword "ISTOGRAM"; ignore it here. + #include #include @@ -72,6 +76,7 @@ #include #include #include +#include #include #include #include @@ -2638,6 +2643,12 @@ PeerImp::onMessage(std::shared_ptr const& m) { JLOG(pJournal_.debug()) << "GetObj: malformed ledgerhash from peer " << id_; fee_.update(Resource::kFeeMalformedRequest, "get object ledger hash"); + XRPL_METRIC_COUNTER_INC_LABELED( + app_, + telemetry::kGetObjectRejectedTotal, + telemetry::kGetObjectRejectedTotalDesc, + {{telemetry::kLabelReason, + std::string(telemetry::kReasonMalformedLedgerHash)}}); return; } } @@ -2650,6 +2661,11 @@ PeerImp::onMessage(std::shared_ptr const& m) << "GetObj: oversized request from peer " << id_ << " (" << packet.objects_size() << " > " << Tuning::kHardMaxReplyNodes << ")"; fee_.update(Resource::kFeeInvalidData, "oversized get object request"); + XRPL_METRIC_COUNTER_INC_LABELED( + app_, + telemetry::kGetObjectRejectedTotal, + telemetry::kGetObjectRejectedTotalDesc, + {{telemetry::kLabelReason, std::string(telemetry::kReasonOversize)}}); return; } @@ -2769,6 +2785,11 @@ PeerImp::processGetObjectByHash(std::shared_ptr con int const requested = packet.objects_size(); int const iterLimit = std::min(requested, Tuning::kHardMaxReplyNodes); + // Time the whole loop once, not each iteration: the loop can run up to + // kHardMaxReplyNodes times, so per-iteration clock reads would cost more + // than the lookups they measure. + auto const lookupStart = std::chrono::steady_clock::now(); + for (int i = 0; i < iterLimit; ++i) { auto const& obj = packet.objects(i); @@ -2793,20 +2814,74 @@ PeerImp::processGetObjectByHash(std::shared_ptr con newObj.set_ledgerseq(obj.ledgerseq()); } + auto const lookupElapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - lookupStart); + // Apply work-proportional charge. `charge()` posts the disconnect // step (if any) back to strand_, so it is safe to call from this // JobQueue worker thread. - charge( - // We pass `requested` directly here, instead of actual lookups done. Which could be - // std::min(packet.objects_size(), static_cast(Tuning::kHardMaxReplyNodes)); - // Because we want to charge as per the request size, to discourage large requests. - computeGetObjectByHashFee(requested, reply.objects_size()), - "processed get object by hash request"); + // + // We pass `requested` directly here, instead of actual lookups done. Which could be + // std::min(packet.objects_size(), static_cast(Tuning::kHardMaxReplyNodes)); + // Because we want to charge as per the request size, to discourage large requests. + // + // Computed into a local so the recorded metric is exactly the charge + // that is applied -- calling the helper twice could diverge. + Resource::Charge const fee = computeGetObjectByHashFee(requested, reply.objects_size()); + charge(fee, "processed get object by hash request"); + + recordGetObjectMetrics(requested, reply.objects_size(), lookupElapsed, fee); JLOG(pJournal_.trace()) << "GetObj: " << reply.objects_size() << " of " << requested; send(std::make_shared(reply, protocol::mtGET_OBJECTS)); } +void +PeerImp::recordGetObjectMetrics( + int const requested, + int const found, + std::chrono::microseconds const lookupElapsed, + Resource::Charge const& fee) +{ + using namespace telemetry; + + XRPL_METRIC_HISTOGRAM_RECORD( + app_, kGetObjectRequestObjects, kGetObjectRequestObjectsDesc, requested); + + XRPL_METRIC_HISTOGRAM_RECORD( + app_, kGetObjectLookupUs, kGetObjectLookupUsDesc, lookupElapsed.count()); + + XRPL_METRIC_HISTOGRAM_RECORD(app_, kGetObjectCharge, kGetObjectChargeDesc, fee.cost()); + + // Batch totals, added once per request rather than once per object: + // per-object increments on a loop bounded by kHardMaxReplyNodes would be + // a measurable cost for no extra information. + // + // `found` is the reply size, which the fetch loop only grows on a + // successful lookup within `iterLimit <= requested`, so `found <= + // requested` always holds. std::max still clamps both values, so a future + // caller passing found > requested cannot make the miss count wrap + // negative -- the counter takes an unsigned amount, where a wrap would + // read as ~1.8e19 rather than as an error. + // + // Written as two calls rather than a loop over a {hit, miss} pair: the + // macros expand to empty statements in a telemetry-off build, which would + // leave a loop's induction variable unused and fail the -Werror build. + XRPL_METRIC_COUNTER_ADD_LABELED( + app_, + kGetObjectLookupsTotal, + kGetObjectLookupsTotalDesc, + static_cast(std::max(0, found)), + {{kLabelResult, std::string(kResultHit)}}); + + XRPL_METRIC_COUNTER_ADD_LABELED( + app_, + kGetObjectLookupsTotal, + kGetObjectLookupsTotalDesc, + static_cast(std::max(0, requested - found)), + {{kLabelResult, std::string(kResultMiss)}}); +} + void PeerImp::onMessage(std::shared_ptr const& m) { diff --git a/src/xrpld/overlay/detail/PeerImp.h b/src/xrpld/overlay/detail/PeerImp.h index 67db379055..0529e1a51a 100644 --- a/src/xrpld/overlay/detail/PeerImp.h +++ b/src/xrpld/overlay/detail/PeerImp.h @@ -791,6 +791,34 @@ private: static void finishServeSpan(telemetry::SpanGuard& span, protocol::TMLedgerData const& ledgerData) noexcept; + /** + * Record the OTel metrics for one completed `TMGetObjectByHash` request. + * + * Extracted from `processGetObjectByHash()` purely to keep that method + * within the 80-line limit; it holds no logic of its own beyond deriving + * the hit/miss split from `requested` and `found`. Called once per + * request, after the fetch loop and the `charge()` call. + * + * Records `getobject_request_objects`, `getobject_lookup_us`, + * `getobject_charge`, and both label values of + * `getobject_lookups_total`. Compiles to nothing when telemetry is + * disabled, because the `XRPL_METRIC_*` macros do. + * + * @param requested Objects the peer asked for (`objects_size()`). + * @param found Objects returned, i.e. the reply's object count. + * Expected to be `<= requested`; clamped either way + * so the derived miss count cannot go negative. + * @param lookupElapsed Wall time of the whole fetch loop. + * @param fee The dynamic charge that was applied, so the + * recorded value is exactly the one charged. + */ + void + recordGetObjectMetrics( + int const requested, + int const found, + std::chrono::microseconds const lookupElapsed, + Resource::Charge const& fee); + protected: // Kept `protected` so test subclasses (see // TMGetObjectByHash_test) can drive the diff --git a/src/xrpld/perflog/detail/PerfLogImp.cpp b/src/xrpld/perflog/detail/PerfLogImp.cpp index f840ec2bf3..f43464dd1d 100644 --- a/src/xrpld/perflog/detail/PerfLogImp.cpp +++ b/src/xrpld/perflog/detail/PerfLogImp.cpp @@ -414,7 +414,7 @@ PerfLogImp::rpcEnd(std::string const& method, std::uint64_t const requestId, boo } void -PerfLogImp::jobQueue(JobType const type) +PerfLogImp::jobQueue(JobType const type, std::string const& name) { auto counter = counters_.jq.find(type); if (counter == counters_.jq.end()) @@ -429,12 +429,13 @@ PerfLogImp::jobQueue(JobType const type) // Task 9.5: Record job enqueue in OTel metrics pipeline. if (auto* mr = app_.getMetricsRegistry()) - mr->recordJobQueued(JobTypes::name(type)); + mr->recordJobQueued(JobTypes::name(type), name); } void PerfLogImp::jobStart( JobType const type, + std::string const& name, microseconds dur, steady_time_point startTime, int instance) @@ -459,11 +460,11 @@ PerfLogImp::jobStart( // Task 9.5: Record job start in OTel metrics pipeline. if (auto* mr = app_.getMetricsRegistry()) - mr->recordJobStarted(JobTypes::name(type), dur.count()); + mr->recordJobStarted(JobTypes::name(type), name, dur.count()); } void -PerfLogImp::jobFinish(JobType const type, microseconds dur, int instance) +PerfLogImp::jobFinish(JobType const type, std::string const& name, microseconds dur, int instance) { auto counter = counters_.jq.find(type); if (counter == counters_.jq.end()) @@ -485,7 +486,7 @@ PerfLogImp::jobFinish(JobType const type, microseconds dur, int instance) // Task 9.5: Record job finish in OTel metrics pipeline. if (auto* mr = app_.getMetricsRegistry()) - mr->recordJobFinished(JobTypes::name(type), dur.count()); + mr->recordJobFinished(JobTypes::name(type), name, dur.count()); } void diff --git a/src/xrpld/perflog/detail/PerfLogImp.h b/src/xrpld/perflog/detail/PerfLogImp.h index 14477512ff..9c8f38bb74 100644 --- a/src/xrpld/perflog/detail/PerfLogImp.h +++ b/src/xrpld/perflog/detail/PerfLogImp.h @@ -153,12 +153,16 @@ public: } void - jobQueue(JobType const type) override; + jobQueue(JobType const type, std::string const& name) override; void - jobStart(JobType const type, microseconds dur, steady_time_point startTime, int instance) - override; + jobStart( + JobType const type, + std::string const& name, + microseconds dur, + steady_time_point startTime, + int instance) override; void - jobFinish(JobType const type, microseconds dur, int instance) override; + jobFinish(JobType const type, std::string const& name, microseconds dur, int instance) override; json::Value countersJson() const override diff --git a/src/xrpld/telemetry/MetricMacros.h b/src/xrpld/telemetry/MetricMacros.h index 940f916de5..6bea92d8e3 100644 --- a/src/xrpld/telemetry/MetricMacros.h +++ b/src/xrpld/telemetry/MetricMacros.h @@ -46,10 +46,19 @@ * if (queueIsFull) * XRPL_METRIC_COUNTER_INC_LABELED(app, "txq_dropped_total", * "Transactions refused admission to the queue", - * ({{"reason", std::string("queue_full")}})); + * {{"reason", std::string("queue_full")}}); * } * @endcode * + * Pass the label set as a bare brace-enclosed list, as above. Do not wrap + * it in an extra pair of parentheses: the list is forwarded verbatim into + * the OTel `Add()`/`Record()` call, which takes an initializer_list, and + * the extra parentheses do not compile. + * + * Wrap each label *value* in `std::string`. `AttributeValue` is a variant + * in which a bare `const char*` selects the boolean alternative, so an + * unwrapped literal is recorded as `true`. + * * Example usage -- UpDownCounter (edge case: value that can decrease): * @code * void ServerHandler::onRpcStart() diff --git a/src/xrpld/telemetry/MetricNames.h b/src/xrpld/telemetry/MetricNames.h index ad2ecbf1d1..900ef88383 100644 --- a/src/xrpld/telemetry/MetricNames.h +++ b/src/xrpld/telemetry/MetricNames.h @@ -48,7 +48,7 @@ * The unit belongs in the name because the OTel `unit` argument is not * surfaced on the Prometheus metric name. * - A gauge that is a snapshot of current state takes no suffix - * (`jobq_backlog`, `sync_state`). + * (`jobq_saturation`, `sync_state`). * - Label VALUES are declared here only when they come from a fixed set that * the code itself writes (`namespace lval`), which is what keeps series * cardinality bounded. A value derived from runtime data -- a site URI, a @@ -199,10 +199,6 @@ inline constexpr char syncAddnodeTotal[] = "sync_addnode_total"; // ===== JobQueue: is the worker pool the bottleneck? ========================== -/** - * Instantaneous JobQueue occupancy, per job type and per state. - */ -inline constexpr char jobqBacklog[] = "jobq_backlog"; /** * Worker-pool saturation: tasks in flight, threads, and jobs queued. */ @@ -337,6 +333,16 @@ inline constexpr char metric[] = "metric"; * Job type, as produced by `JobTypes::name()`. */ inline constexpr char jobType[] = "job_type"; +/** + * Which producer submitted a job, within its job type. + * + * A job type has several producers (`RcvGetLedger` and `RcvGetObjByHash` both + * run as `JtLedgerReq`), so this is what attributes a latency spike to one of + * them. Bounded by `MetricsRegistry::sanitiseHandler()`, which folds any job + * name that is not all ASCII letters -- the ones embedding a ledger sequence + * -- down to a single `other` value. + */ +inline constexpr char handler[] = "handler"; /** * Terminal result of a bounded operation. */ @@ -534,18 +540,6 @@ inline constexpr char duplicate[] = "duplicate"; inline constexpr char invalid[] = "invalid"; } // namespace addnode -/** - * `jobq_backlog` sub-metrics -- the three occupancy states of a job type. - * - * `deferred` has no other exposure anywhere: a job held back by its type's - * concurrency limit counts as neither waiting nor running. - */ -namespace jobq_backlog { -inline constexpr char waiting[] = "waiting"; -inline constexpr char running[] = "running"; -inline constexpr char deferred[] = "deferred"; -} // namespace jobq_backlog - /** * `jobq_saturation` sub-metrics: the numerator, denominator and the backlog. */ diff --git a/src/xrpld/telemetry/MetricsRegistry.cpp b/src/xrpld/telemetry/MetricsRegistry.cpp index 27b3e35c2b..d4e662b0f1 100644 --- a/src/xrpld/telemetry/MetricsRegistry.cpp +++ b/src/xrpld/telemetry/MetricsRegistry.cpp @@ -43,7 +43,6 @@ #include #include #include -#include #include #include #include @@ -53,6 +52,7 @@ #include #include #include +#include #include #include @@ -73,6 +73,7 @@ #include #include +#include #include #include #include @@ -126,8 +127,44 @@ constexpr char kConsensusRoundDurationMs[] = "consensus_round_duration_ms"; * @param boundaries Bucket upper bounds, in the instrument's own unit, * ascending. */ + +/** + * Bucket boundaries for microsecond-valued duration instruments. + * + * 100 µs, 500 µs, 1 ms, 5 ms, 10 ms, 25 ms, 50 ms, 100 ms, 250 ms, 500 ms, + * 1 s, 2.5 s, 5 s, 10 s, 30 s, 60 s. Covers sub-millisecond jobs through + * multi-second stalls without saturating. + */ +constexpr std::array kMicrosecondBoundaries{ + 100.0, + 500.0, + 1'000.0, + 5'000.0, + 10'000.0, + 25'000.0, + 50'000.0, + 100'000.0, + 250'000.0, + 500'000.0, + 1'000'000.0, + 2'500'000.0, + 5'000'000.0, + 10'000'000.0, + 30'000'000.0, + 60'000'000.0}; + +/** + * Register an explicit-bucket histogram view. + * + * The SDK's default boundaries top out at 10,000, so any instrument whose + * values exceed that saturates and every quantile reads as the ceiling. + * + * @param views The registry to add the view to. + * @param name Instrument name to match (e.g. "job_running_us"). + * @param boundaries Bucket upper bounds, ascending. + */ void -addDurationHistogramView( +addHistogramView( metric_sdk::ViewRegistry& views, std::string const& name, std::vector boundaries) @@ -145,11 +182,12 @@ addDurationHistogramView( } /** - * Register the explicit-bucket view for a MICROSECOND-valued instrument. + * Register the microsecond-ladder view for a duration instrument. * - * Boundaries: 100µs, 500µs, 1ms, 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, - * 1s, 2.5s, 5s, 10s, 30s, 60s — sub-millisecond jobs through multi-second - * stalls, without saturating. + * Job wait/run times and RPC latencies routinely exceed the SDK default + * ceiling, so they all share `kMicrosecondBoundaries`: 100µs, 500µs, 1ms, 5ms, + * 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 2.5s, 5s, 10s, 30s, 60s — + * sub-millisecond jobs through multi-second stalls, without saturating. * * @param views The registry to add the view to. * @param name Instrument name to match (e.g. "job_running_us"). @@ -157,25 +195,7 @@ addDurationHistogramView( void addMicrosecondHistogramView(metric_sdk::ViewRegistry& views, std::string const& name) { - addDurationHistogramView( - views, - name, - {100.0, - 500.0, - 1'000.0, - 5'000.0, - 10'000.0, - 25'000.0, - 50'000.0, - 100'000.0, - 250'000.0, - 500'000.0, - 1'000'000.0, - 2'500'000.0, - 5'000'000.0, - 10'000'000.0, - 30'000'000.0, - 60'000'000.0}); + addHistogramView(views, name, {kMicrosecondBoundaries.begin(), kMicrosecondBoundaries.end()}); } /** @@ -201,7 +221,7 @@ addMicrosecondHistogramView(metric_sdk::ViewRegistry& views, std::string const& void addRoundDurationHistogramView(metric_sdk::ViewRegistry& views, std::string const& name) { - addDurationHistogramView( + addHistogramView( views, name, {500.0, @@ -304,6 +324,31 @@ MetricsRegistry::initExporterAndProvider(std::string const& endpoint, std::strin // view is declared here (see the constant's comment). addRoundDurationHistogramView(*views, kConsensusRoundDurationMs); + // Recorded at its PeerImp.cpp call site, not created here, so the name + // comes from the shared constant both sites use. + addMicrosecondHistogramView(*views, kGetObjectLookupUs); + + // The remaining two GetObject histograms are not durations, so the + // microsecond ladder above does not fit them. Both still need explicit + // boundaries: the SDK default stops at 10,000 and both ranges exceed it. + // + // Object counts run 1..kHardMaxReplyNodes (12288). The honest sync path + // asks for at most 8, so the low buckets are fine-grained and the upper + // ones follow the charge size bands (64, 1024) up to the hard cap. + addHistogramView( + *views, + kGetObjectRequestObjects, + {1.0, 2.0, 4.0, 8.0, 16.0, 64.0, 256.0, 1'024.0, 4'096.0, 12'288.0}); + + // Charge values span 0 (free tier) to ~99k for a full-size all-miss + // request. Boundaries bracket the resource thresholds that decide a + // peer's fate -- kWarningThreshold (5000) and kDropThreshold (25000) -- + // so a dashboard can show how close charges run to each. + addHistogramView( + *views, + kGetObjectCharge, + {0.0, 100.0, 500.0, 1'000.0, 5'000.0, 10'000.0, 25'000.0, 50'000.0, 100'000.0}); + // Create MeterProvider with resource, then attach the metric reader. provider_ = metric_sdk::MeterProviderFactory::Create(std::move(views), resourceAttrs); provider_->AddMetricReader(std::move(reader)); @@ -482,25 +527,35 @@ MetricsRegistry::recordRpcErrored(std::string_view method, std::int64_t duration // ----------------------------------------------------------------- void -MetricsRegistry::recordJobQueued(std::string_view jobType) +MetricsRegistry::recordJobQueued(std::string_view jobType, std::string_view jobName) { #ifdef XRPL_ENABLE_TELEMETRY if (!enabled_ || !jobQueuedCounter_) return; - jobQueuedCounter_->Add(1, {{"job_type", std::string(jobType)}}); + jobQueuedCounter_->Add( + 1, + {{label::jobType, std::string(jobType)}, + {label::handler, std::string(sanitiseHandler(jobName))}}); #else (void)jobType; + (void)jobName; (void)enabled_; #endif } void -MetricsRegistry::recordJobStarted(std::string_view jobType, std::int64_t queuedDurUs) +MetricsRegistry::recordJobStarted( + std::string_view jobType, + std::string_view jobName, + std::int64_t queuedDurUs) { #ifdef XRPL_ENABLE_TELEMETRY if (!enabled_ || !jobStartedCounter_) return; - jobStartedCounter_->Add(1, {{"job_type", std::string(jobType)}}); + // Build the attribute pair once: both the counter and the histogram + // must carry the identical label set or they cannot be joined. + std::string const handler(sanitiseHandler(jobName)); + jobStartedCounter_->Add(1, {{label::jobType, std::string(jobType)}, {label::handler, handler}}); if (jobQueuedDurationHistogram_ && queuedDurUs >= 0) { // Guard against negative queued durations: the caller derives this @@ -509,32 +564,39 @@ MetricsRegistry::recordJobStarted(std::string_view jobType, std::int64_t queuedD // (logging a warning per call), so skip them rather than spam. jobQueuedDurationHistogram_->Record( static_cast(queuedDurUs), - {{"job_type", std::string(jobType)}}, + {{label::jobType, std::string(jobType)}, {label::handler, handler}}, opentelemetry::context::Context{}); } #else (void)jobType; + (void)jobName; (void)queuedDurUs; (void)enabled_; #endif } void -MetricsRegistry::recordJobFinished(std::string_view jobType, std::int64_t runningDurUs) +MetricsRegistry::recordJobFinished( + std::string_view jobType, + std::string_view jobName, + std::int64_t runningDurUs) { #ifdef XRPL_ENABLE_TELEMETRY if (!enabled_ || !jobFinishedCounter_) return; - jobFinishedCounter_->Add(1, {{"job_type", std::string(jobType)}}); + std::string const handler(sanitiseHandler(jobName)); + jobFinishedCounter_->Add( + 1, {{label::jobType, std::string(jobType)}, {label::handler, handler}}); if (jobRunningDurationHistogram_) { jobRunningDurationHistogram_->Record( static_cast(runningDurUs), - {{"job_type", std::string(jobType)}}, + {{label::jobType, std::string(jobType)}, {label::handler, handler}}, opentelemetry::context::Context{}); } #else (void)jobType; + (void)jobName; (void)runningDurUs; (void)enabled_; #endif @@ -575,7 +637,6 @@ MetricsRegistry::registerAsyncGauges() registerStallEventsCounter(); registerSyncAcquireGauge(); registerCacheHitRateDetailGauge(); - registerJobQueueBacklogGauge(); registerJobQueueSaturationGauge(); registerPeerLedgerSupplyGauge(); registerSlotCensusGauge(); @@ -1826,53 +1887,6 @@ MetricsRegistry::registerCacheHitRateDetailGauge() this); } -void -MetricsRegistry::registerJobQueueBacklogGauge() -{ - // --- Sync diagnostics: which job types are starved right now? --- - // The existing job_* counters and histograms describe jobs that already - // moved. This is instantaneous occupancy, and `deferred` in particular - // has no other exposure: a job held back by its type's concurrency limit - // counts as neither waiting nor running anywhere else. - jobQueueBacklogGauge_ = meter_->CreateInt64ObservableGauge( - metric::jobqBacklog, "JobQueue occupancy per job type (waiting/running/deferred)"); - jobQueueBacklogGauge_->AddCallback( - [](opentelemetry::metrics::ObserverResult result, void* state) { - auto* self = static_cast(state); - if (self->callbacksDetached_.load(std::memory_order_acquire)) - return; - auto& app = self->app_; - - try - { - auto observe = [&](char const* field, std::string const& jobType, int64_t value) { - opentelemetry::nostd::get>>(result) - ->Observe(value, {{label::metric, field}, {label::jobType, jobType}}); - }; - - // One snapshot under one lock acquire, so the three fields of - // a type are mutually consistent rather than read at three - // different instants. - for (auto const& count : app.getJobQueue().getJobTypeCounts()) - { - // The name helper is the single source of the label value, - // the same one the job_*_total counters already use, so the - // two label sets join. - auto const& jobType = JobTypes::name(count.type); - observe(lval::jobq_backlog::waiting, jobType, count.waiting); - observe(lval::jobq_backlog::running, jobType, count.running); - observe(lval::jobq_backlog::deferred, jobType, count.deferred); - } - } - catch (...) // NOLINT(bugprone-empty-catch) - { - // Silently skip if services are not yet ready. - } - }, - this); -} - void MetricsRegistry::registerJobQueueSaturationGauge() { diff --git a/src/xrpld/telemetry/MetricsRegistry.h b/src/xrpld/telemetry/MetricsRegistry.h index fb3830fc0e..ff07fdd10c 100644 --- a/src/xrpld/telemetry/MetricsRegistry.h +++ b/src/xrpld/telemetry/MetricsRegistry.h @@ -28,11 +28,11 @@ * | +-- rpc_method_finished_total * | +-- rpc_method_errored_total * | +-- rpc_method_us (Histogram) - * | +-- job_queued_total - * | +-- job_started_total - * | +-- job_finished_total - * | +-- job_queued_us (Histogram) - * | +-- job_running_us (Histogram) + * | +-- job_queued_total{job_type,handler} + * | +-- job_started_total{job_type,handler} + * | +-- job_finished_total{job_type,handler} + * | +-- job_queued_us{job_type,handler} (Histogram) + * | +-- job_running_us{job_type,handler} (Histogram) * | +-- ledgers_closed_total * | +-- validations_sent_total * | +-- validations_checked_total @@ -64,7 +64,6 @@ * +-- Clock close offset (local clock skew) * +-- Sync state (time to first FULL, network-ledger gate, * | server stall seconds, ledgers behind network) - * +-- JobQueue backlog (waiting/running/deferred per job type) * +-- JobQueue saturation (running tasks vs worker threads vs backlog) * +-- Peer ledger supply (how many peers can serve the needed sequence) * +-- PeerFinder slot census (slots, attempts, fixed peers, address caches) @@ -121,9 +120,10 @@ * // or: mr->recordRpcErrored("server_info", durationUs); * } * - * // In PerfLogImp::jobQueue(): + * // In PerfLogImp::jobQueue(). The second argument is the addJob name; + * // it is sanitised internally into the bounded `handler` label. * if (auto* mr = app_.getMetricsRegistry()) - * mr->recordJobQueued("ledgerData"); + * mr->recordJobQueued("ledgerData", "ProcessLData"); * * // Shutdown: * metricsRegistry_->stop(); @@ -146,9 +146,11 @@ #include +#include #include #include #include +#include #include #include @@ -327,28 +329,115 @@ public: void recordRpcErrored(std::string_view method, std::int64_t durationUs); + /** + * The `handler` label value used for any job name that fails the + * sanitiser's all-ASCII-letters rule. + * + * Public because both sanitiseHandler() and its unit tests must agree + * on the exact fallback token; a test asserting against its own copy + * of the string would not catch a change made here. + * + * Declared as std::string_view rather than the `constexpr char k[]` + * form used for instrument names in MetricsRegistry.cpp: this value is + * *returned* by sanitiseHandler(), whose return type is + * std::string_view, and is compared against std::string_view in tests. + * Matching the type avoids array-to-pointer decay and a needless + * strlen at each use. + */ + static constexpr std::string_view kHandlerOther{"other"}; + + /** + * Reduce a job name to a bounded-cardinality `handler` label value. + * + * A job type can have several producers — both `RcvGetLedger` and + * `RcvGetObjByHash` run as `JtLedgerReq` — so `job_type` alone cannot + * attribute a latency spike to one of them. The job name can, but it + * cannot be used raw: two names embed a ledger sequence number + * (`"Pub" + std::to_string(seq)` in LedgerPersistence.cpp and + * `"OB" + std::to_string(...)` in OrderBookDBImpl.cpp), which would + * mint a fresh Prometheus series for every ledger. + * + * The rule is therefore: keep the name only when it is non-empty and + * every character is an ASCII letter; otherwise return `"other"`. + * Both dynamic names always contain digits, so they always fold to + * `"other"`, while every all-letter name is a compile-time literal. + * The label domain is thus a function of the literals present in the + * source — 43 names plus `"other"` at the time of writing — and + * cannot grow at runtime. A name added later that does not satisfy + * the rule degrades to `"other"` rather than becoming unbounded, + * which is a stronger guarantee than an allowlist that would have to + * be maintained by hand. + * + * Defined inline so unit tests can call it without linking the rest + * of the registry: in a telemetry-enabled build MetricsRegistry.cpp + * is not compiled into the test binary, so an out-of-line definition + * would be unreachable from tests. Being inline also makes it usable + * regardless of XRPL_ENABLE_TELEMETRY. + * + * @param name The job name as passed to JobQueue::addJob. + * @return @p name when it is non-empty and all ASCII letters, else + * kHandlerOther. + * + * @note Pure and reentrant: holds no state, performs no I/O, and is + * safe to call concurrently from any thread. + * @note The letter test is an explicit ASCII range check rather than + * std::isalpha, which classifies by the current C locale. A + * locale-dependent test could admit non-ASCII bytes and so + * weaken the cardinality bound this function exists to provide. + * @note When the name is kept, the returned view aliases @p name, so + * it must not outlive the caller's buffer. The kHandlerOther case + * returns a view of a static constant and is always valid. + * + * Example: + * @code + * sanitiseHandler("RcvGetObjByHash"); // "RcvGetObjByHash" + * sanitiseHandler("Pub94512331"); // kHandlerOther (digits) + * sanitiseHandler(""); // kHandlerOther (empty) + * @endcode + */ + [[nodiscard]] static constexpr std::string_view + sanitiseHandler(std::string_view name) noexcept + { + auto const isAsciiLetter = [](char const c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); + }; + + if (name.empty() || !std::ranges::all_of(name, isAsciiLetter)) + return kHandlerOther; + + return name; + } + /** * Record a job enqueued event. * @param jobType The job type name (e.g. "ledgerData"). + * @param jobName The addJob name, reduced to a bounded `handler` + * label by sanitiseHandler(). Distinguishes producers + * that share a job type. */ void - recordJobQueued(std::string_view jobType); + recordJobQueued(std::string_view jobType, std::string_view jobName); /** * Record a job start event. * @param jobType The job type name. + * @param jobName The addJob name; see recordJobQueued(). * @param queuedDurUs Time the job spent waiting in the queue (us). */ void - recordJobStarted(std::string_view jobType, std::int64_t queuedDurUs); + recordJobStarted(std::string_view jobType, std::string_view jobName, std::int64_t queuedDurUs); /** * Record a job finish event. * @param jobType The job type name. + * @param jobName The addJob name; see recordJobQueued(). * @param runningDurUs Execution time in microseconds. */ void - recordJobFinished(std::string_view jobType, std::int64_t runningDurUs); + recordJobFinished( + std::string_view jobType, + std::string_view jobName, + std::int64_t runningDurUs); // ----------------------------------------------------------------- // External dashboard parity counters (Tasks 7.9-7.14) @@ -507,25 +596,27 @@ private: rpcDurationHistogram_; // --- Synchronous instruments (Job Queue) --- + // All five carry handler="" in addition to + // job_type, so producers that share a job type stay distinguishable. /** - * Counter: job_queued_total{job_type=""} + * Counter: job_queued_total{job_type="",handler=""} */ opentelemetry::nostd::unique_ptr> jobQueuedCounter_; /** - * Counter: job_started_total{job_type=""} + * Counter: job_started_total{job_type="",handler=""} */ opentelemetry::nostd::unique_ptr> jobStartedCounter_; /** - * Counter: job_finished_total{job_type=""} + * Counter: job_finished_total{job_type="",handler=""} */ opentelemetry::nostd::unique_ptr> jobFinishedCounter_; /** - * Histogram: job_queued_duration_us{job_type=""} + * Histogram: job_queued_duration_us{job_type="",handler=""} */ opentelemetry::nostd::unique_ptr> jobQueuedDurationHistogram_; /** - * Histogram: job_running_duration_us{job_type=""} + * Histogram: job_running_duration_us{job_type="",handler=""} */ opentelemetry::nostd::unique_ptr> jobRunningDurationHistogram_; @@ -590,12 +681,6 @@ private: */ opentelemetry::nostd::shared_ptr shamapCacheHitRateGauge_; - /** - * Observable gauge for per-job-type JobQueue occupancy: waiting, running - * and deferred counts, keyed by job type. - */ - opentelemetry::nostd::shared_ptr - jobQueueBacklogGauge_; /** * Observable gauge for global worker-pool saturation: tasks in flight, * configured worker threads, and total jobs queued. @@ -955,42 +1040,6 @@ private: void registerCacheHitRateDetailGauge(); // sync diagnostics: treenode cache - /** - * Register the `jobq_backlog` gauge. - * - * Three series per job type, from one JobQueue::getJobTypeCounts() - * snapshot, under the `metric` and `job_type` attributes: - * - * `waiting` — jobs enqueued and not yet dispatched to a worker. - * `running` — jobs executing on a worker. - * `deferred` — **the signal this gauge exists for.** Jobs held back - * because the type is already at its concurrency limit. The - * sync-critical types run at limits of 3 (`JtLedgerReq`, - * `JtLedgerData` in JobTypes.h), so during a fresh sync those types - * routinely have work denied a worker, and that state appears in - * neither `waiting` nor `running`. - * - * Distinct from the job metrics that already exist. `job_queued_total` / - * `job_started_total` / `job_finished_total` and the `job_queued_us` / - * `job_running_us` histograms are all event-driven and come from - * PerfLogImp: they describe jobs that already moved. This gauge is - * instantaneous occupancy — what is sitting in the queue right now, which - * a rate or a latency quantile cannot express. The StatsD - * `jobq_job_count` gauge is queue-wide only, with no per-type split and - * no deferred count at all. - * - * `job_type` is the JobTypes::name() string, matching the label the job - * counters already use so the two can be joined. Cardinality is bounded - * by the JobType enum (~46 values), and every type is observed on every - * tick, so an idle type reports 0 rather than dropping its series. - * - * @note Pulled on the OTel reader thread (~10 s tick), never on a hot - * path. Takes the JobQueue mutex once per tick for three integer reads - * per type; no per-job cost is added anywhere. - */ - void - registerJobQueueBacklogGauge(); // sync diagnostics: per-type backlog - /** * Register the `jobq_saturation` gauge. * @@ -1004,11 +1053,14 @@ private: * from `[workers]`, node size and hardware concurrency. * `total_waiting` — jobs queued across all types. * - * The reason this is separate from `jobq_backlog`: when the pool itself - * is exhausted, every subsystem waiting behind it looks independently - * slow, and each per-type panel invites the wrong conclusion. A - * `running_tasks / worker_threads` ratio at 1.0 with a non-zero - * `total_waiting` attributes the whole slowdown to pool exhaustion once. + * The reason this is separate from the per-job-type gauges JobQueue + * itself publishes (`jobq__waiting` / `_running` / `_deferred`): + * when the pool itself is exhausted, every subsystem waiting behind it + * looks independently slow, and each per-type panel invites the wrong + * conclusion. A `running_tasks / worker_threads` ratio at 1.0 with a + * non-zero `total_waiting` attributes the whole slowdown to pool + * exhaustion once. Those per-type gauges carry no capacity term at all, + * so no reading there can say whether the pool is the cause. * * @note Pulled on the OTel reader thread (~10 s tick). One atomic load, * one plain int read, and one pass over the per-type counters under the