From ac71480a626aa759598cc8d3991c14b6cb959110 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Mon, 17 Aug 2026 19:36:41 +0100 Subject: [PATCH] fix(telemetry): count mode transitions with tiling buckets, not overlapping ones The transitions panel used increase(...[$__rate_interval]). $__rate_interval is defined as max($__interval + scrape, 4 * scrape), i.e. deliberately one scrape longer than the step so rate() windows overlap and lose no counter increase. That overlap is harmless for rate(), but this panel reads the value as a count of discrete events, and the overlap counts each event in more than one bucket. Measured against a log-derived ground truth of 106 syncing transitions on devnet-otel-usw2-01 over 2026-08-11T11:05Z..2026-08-12T23:04Z, the old query reported 111.3 at a 300s step and 133.7 at a 60s step -- the error grew to +26% as you zoomed in, because the overlap is a larger fraction of a smaller step. Switch to $__interval so the buckets tile exactly, and wrap in round() because increase() extrapolates to the window edges and so reports fractional counts for an integer counter. The same measurement now gives 106 at 300s, 105 at 60s and 107 at 900s. Every state and both nodes land within a few counts of truth at any zoom, and the legend Total is now a meaningful figure. Pin Min step to 1m: the real scrape interval is 60s while the datasource declares 15s, so without a floor $__interval can fall below one sample. Draw as bars with 0 decimals -- the value is a discrete count per bucket, and a line implies interpolation between counts that does not exist. --- .../grafana/dashboards/node-health.json | 31 ++++++++++------ docs/telemetry-runbook.md | 36 +++++++++---------- 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index 35e525d27f..b5231ae7d5 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -274,7 +274,7 @@ }, { "title": "Operating Mode Transitions", - "description": "**What:** Transitions into each operating mode, per interval.\n**How it's computed:** increase() over the per-mode transition counters, so each point is the number of transitions in that bucket and the series stays correct across an xrpld restart (the counters reset to 0).\n**Reading it:** Few transitions is good; a stable node rarely leaves Full. Brief flaps are visible here even when they are too short to appear on Operating Mode (State Timeline), which can only sample state once per scrape.\n**Healthy range:** workload-dependent; low and infrequent transitions.\n**Watch for:** Frequent transitions out of Full, or any into Disconnected/Syncing (flapping).\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp NetworkOPsImp::Stats ctor", + "description": "**What:** Transitions into each operating mode, per interval.\n**How it's computed:** round(increase(...[$__interval])) over the per-mode transition counters. $__interval tiles the buckets exactly, so each bar is the transitions in that bucket and the legend Total is the true count; $__rate_interval would overlap each bucket by one scrape and inflate it (measured +5% at a 36h range, +26% zoomed in). round() removes increase()'s extrapolation, which otherwise reports fractional counts. The series stays correct across an xrpld restart (the counters reset to 0).\n**Reading it:** Few transitions is good; a stable node rarely leaves Full. Brief flaps are visible here even when they are too short to appear on Operating Mode (State Timeline), which can only sample state once per scrape.\n**Healthy range:** workload-dependent; low and infrequent transitions.\n**Watch for:** Frequent transitions out of Full, or any into Disconnected/Syncing (flapping).\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp NetworkOPsImp::Stats ctor", "type": "timeseries", "gridPos": { "h": 10, @@ -287,6 +287,12 @@ "maxHeight": 600, "mode": "multi", "sort": "desc" + }, + "legend": { + "calcs": ["sum", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true } }, "targets": [ @@ -294,35 +300,35 @@ "datasource": { "type": "prometheus" }, - "expr": "increase(state_accounting_full_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", + "expr": "round(increase(state_accounting_full_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__interval]))", "legendFormat": "Full [{{service_instance_id}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "increase(state_accounting_tracking_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", + "expr": "round(increase(state_accounting_tracking_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__interval]))", "legendFormat": "Tracking [{{service_instance_id}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "increase(state_accounting_syncing_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", + "expr": "round(increase(state_accounting_syncing_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__interval]))", "legendFormat": "Syncing [{{service_instance_id}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "increase(state_accounting_connected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", + "expr": "round(increase(state_accounting_connected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__interval]))", "legendFormat": "Connected [{{service_instance_id}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "increase(state_accounting_disconnected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", + "expr": "round(increase(state_accounting_disconnected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__interval]))", "legendFormat": "Disconnected [{{service_instance_id}}]" } ], @@ -335,14 +341,17 @@ "insertNulls": false, "showPoints": "auto", "pointSize": 5, - "lineWidth": 1, - "fillOpacity": 0, - "gradientMode": "none" - } + "lineWidth": 0, + "fillOpacity": 70, + "gradientMode": "none", + "drawStyle": "bars" + }, + "decimals": 0 }, "overrides": [] }, - "id": 6 + "id": 6, + "interval": "1m" }, { "title": "I/O Latency", diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index e915e8cfd7..93f67fabbd 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -723,24 +723,24 @@ Requires `trace_peer=1` in the `[telemetry]` config section. ### Node Health -- System Metrics (`node-health`) -| Panel | Type | PromQL | Labels Used | -| -------------------------------------- | ---------- | --------------------------------------------------------------------------------- | ----------- | -| Validated Ledger Age | stat | `ledgermaster_validated_ledger_age` | — | -| Published Ledger Age | stat | `ledgermaster_published_ledger_age` | — | -| Operating Mode (Time Share) | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))` | — | -| Operating Mode Transitions | timeseries | `increase(state_accounting_*_transitions[$__rate_interval])` | — | -| I/O Latency | timeseries | `histogram_quantile(0.95, ios_latency_bucket)` | — | -| Job Queue Depth | timeseries | `jobq_job_count` | — | -| Ledger Fetch Rate | stat | `rate(ledger_fetches_total[$__rate_interval])` | — | -| Ledger History Mismatches | stat | `rate(ledger_history_mismatch_total[$__rate_interval])` | — | -| Key Jobs Execution Time | timeseries | `acceptledger{quantile="$quantile"}` (+ 10 more key jobs) | `quantile` | -| Key Jobs Dequeue Wait Time | timeseries | `acceptledger_q{quantile="$quantile"}` (+ 10 more) | `quantile` | -| FullBelowCache Size | timeseries | `node_family_full_below_cache_size` | — | -| FullBelowCache Hit Rate | gauge | `node_family_full_below_cache_hit_rate` | — | -| Ledger Publish Gap | stat | `Published_Ledger_Age - Validated_Ledger_Age` | — | -| State Duration Rate (Full vs Tracking) | timeseries | `rate(state_accounting_full_duration[5m]) / 1000000` | — | -| All Jobs Execution Time (Detail) | timeseries | `histogram_quantile($quantile, rate(job_running_us_bucket[5m])) by job_type` — µs | `quantile` | -| All Jobs Dequeue Wait (Detail) | timeseries | `histogram_quantile($quantile, rate(job_queued_us_bucket[5m])) by job_type` — µs | `quantile` | +| Panel | Type | PromQL | Labels Used | +| -------------------------------------- | ---------- | ---------------------------------------------------------------------------------- | ----------- | +| Validated Ledger Age | stat | `ledgermaster_validated_ledger_age` | — | +| Published Ledger Age | stat | `ledgermaster_published_ledger_age` | — | +| Operating Mode (Time Share) | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))` | — | +| Operating Mode Transitions | timeseries | `round(increase(state_accounting_*_transitions[$__interval]))` (bars, Min step 1m) | — | +| I/O Latency | timeseries | `histogram_quantile(0.95, ios_latency_bucket)` | — | +| Job Queue Depth | timeseries | `jobq_job_count` | — | +| Ledger Fetch Rate | stat | `rate(ledger_fetches_total[$__rate_interval])` | — | +| Ledger History Mismatches | stat | `rate(ledger_history_mismatch_total[$__rate_interval])` | — | +| Key Jobs Execution Time | timeseries | `acceptledger{quantile="$quantile"}` (+ 10 more key jobs) | `quantile` | +| Key Jobs Dequeue Wait Time | timeseries | `acceptledger_q{quantile="$quantile"}` (+ 10 more) | `quantile` | +| FullBelowCache Size | timeseries | `node_family_full_below_cache_size` | — | +| FullBelowCache Hit Rate | gauge | `node_family_full_below_cache_hit_rate` | — | +| Ledger Publish Gap | stat | `Published_Ledger_Age - Validated_Ledger_Age` | — | +| State Duration Rate (Full vs Tracking) | timeseries | `rate(state_accounting_full_duration[5m]) / 1000000` | — | +| All Jobs Execution Time (Detail) | timeseries | `histogram_quantile($quantile, rate(job_running_us_bucket[5m])) by job_type` — µs | `quantile` | +| All Jobs Dequeue Wait (Detail) | timeseries | `histogram_quantile($quantile, rate(job_queued_us_bucket[5m])) by job_type` — µs | `quantile` | ### Network Traffic -- System Metrics (`network-traffic`)