From 277f7c965545f91c50210bf730e5d331a40e0823 Mon Sep 17 00:00:00 2001
From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
Date: Wed, 12 Aug 2026 16:52:08 +0100
Subject: [PATCH] fix(telemetry): surface stalled consensus rounds with a p99
 series

The "Consensus Round Duration" panel plotted only p50 and p95. At the
devnet rate of ~19 rounds/min, a single stalled round is one sample in
several hundred, so p95 over a rate interval stays near the normal 2s
close time and the stall is invisible.

Measured on devnet-otel-usw2-02 over a 40 min window: 761 rounds, of
which exactly one fell in the 7500-10000ms bucket while the mean never
exceeded 2270ms. The histogram already records the outlier; only the
panel could not show it.

Add a p99 series alongside p50/p95 and retitle the panel accordingly.
No metric or collector change: the existing
consensus_round_duration_ms_bucket histogram already has 20 boundaries
up to 120000ms.
---
 .../grafana/dashboards/ledger-sync-health.json         | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/docker/telemetry/grafana/dashboards/ledger-sync-health.json b/docker/telemetry/grafana/dashboards/ledger-sync-health.json
index 6dfd30f8de..d68c55fe27 100644
--- a/docker/telemetry/grafana/dashboards/ledger-sync-health.json
+++ b/docker/telemetry/grafana/dashboards/ledger-sync-health.json
@@ -3234,9 +3234,17 @@
           },
           "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_work_item) (rate(consensus_round_duration_ms_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"P95 round duration\", \"__name__\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")",
           "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_work_item) (rate(consensus_round_duration_ms_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"P99 round duration\", \"__name__\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")",
+          "refId": "C"
         }
       ],
-      "title": "Consensus Round Duration (p50/p95)",
+      "title": "Consensus Round Duration (p50/p95/p99)",
       "type": "timeseries"
     },
     {