From c8f9ab9597051abb469e09ed95e512e520537f3e Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Mon, 17 Aug 2026 18:02:13 +0100 Subject: [PATCH] fix(telemetry): plot operating-mode transitions per interval, not raw counters The Operating Mode Transitions panel queried state_accounting_*_transitions directly. Those are monotonic counters, so the panel drew a slowly rising line and a few transitions per hour were invisible against a total in the hundreds. It also fell off a cliff whenever xrpld restarted and the counters reset to 0, which reads as missing data rather than a restart. Wrap each target in increase(...[$__rate_interval]) so each point is the number of transitions in that bucket and the series survives a counter reset. This is what the sibling panels on the same row (Operating Mode (Time Share), State Duration Rate) already do. Verified against devnet-otel-usw2-01/02 over 2026-08-11T11:01Z..2026-08-12T16:23Z: the fixed expression reports 107 and 123 syncing transitions, matching the counter deltas, and stays continuous across the 12:07 restart where the raw counter dropped 630 -> 1. Brief mode flaps remain invisible on Operating Mode (State Timeline) because a ~2 s dwell cannot be captured by a 60 s scrape; this panel is the place to read them. --- docker/telemetry/grafana/dashboards/node-health.json | 12 ++++++------ docs/telemetry-runbook.md | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index c01f5b4301..35e525d27f 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -274,7 +274,7 @@ }, { "title": "Operating Mode Transitions", - "description": "**What:** Count of transitions into each operating mode.\n**How it's computed:** Per-mode transition counters at their latest value per node.\n**Reading it:** Few transitions is good; a stable node rarely leaves Full.\n**Healthy range:** workload-dependent; low and infrequent transitions.\n**Watch for:** Frequent transitions out of Full, or any into Disconnected/Syncing (flapping).\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp NetworkOPsImp::Stats ctor", + "description": "**What:** Transitions into each operating mode, per interval.\n**How it's computed:** increase() over the per-mode transition counters, so each point is the number of transitions in that bucket and the series stays correct across an xrpld restart (the counters reset to 0).\n**Reading it:** Few transitions is good; a stable node rarely leaves Full. Brief flaps are visible here even when they are too short to appear on Operating Mode (State Timeline), which can only sample state once per scrape.\n**Healthy range:** workload-dependent; low and infrequent transitions.\n**Watch for:** Frequent transitions out of Full, or any into Disconnected/Syncing (flapping).\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp NetworkOPsImp::Stats ctor", "type": "timeseries", "gridPos": { "h": 10, @@ -294,35 +294,35 @@ "datasource": { "type": "prometheus" }, - "expr": "state_accounting_full_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}", + "expr": "increase(state_accounting_full_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", "legendFormat": "Full [{{service_instance_id}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "state_accounting_tracking_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}", + "expr": "increase(state_accounting_tracking_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", "legendFormat": "Tracking [{{service_instance_id}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "state_accounting_syncing_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}", + "expr": "increase(state_accounting_syncing_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", "legendFormat": "Syncing [{{service_instance_id}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "state_accounting_connected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}", + "expr": "increase(state_accounting_connected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", "legendFormat": "Connected [{{service_instance_id}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "state_accounting_disconnected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}", + "expr": "increase(state_accounting_disconnected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", "legendFormat": "Disconnected [{{service_instance_id}}]" } ], diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 80184ca7cc..e915e8cfd7 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -728,7 +728,7 @@ Requires `trace_peer=1` in the `[telemetry]` config section. | Validated Ledger Age | stat | `ledgermaster_validated_ledger_age` | — | | Published Ledger Age | stat | `ledgermaster_published_ledger_age` | — | | Operating Mode (Time Share) | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))` | — | -| Operating Mode Transitions | timeseries | `state_accounting_*_transitions` | — | +| Operating Mode Transitions | timeseries | `increase(state_accounting_*_transitions[$__rate_interval])` | — | | I/O Latency | timeseries | `histogram_quantile(0.95, ios_latency_bucket)` | — | | Job Queue Depth | timeseries | `jobq_job_count` | — | | Ledger Fetch Rate | stat | `rate(ledger_fetches_total[$__rate_interval])` | — |