From 946491258f67500b0b2306e78e623ab2d720adfb Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Wed, 22 Jul 2026 12:28:45 +0100 Subject: [PATCH] fix(telemetry): correct Ledger Close Duration on consensus-health dashboard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "Ledger Close Duration" panel measured the consensus.ledger_close span duration, which only wraps the sub-millisecond onClose() prologue — not the ledger close. Repoint it to the consensus.round span (full round, open to accept) so it reflects actual close time (~3-5s on mainnet). The consensus_mode filter is preserved (consensus.round carries that attribute, verified live). The sibling rate panels (Consensus Mode Over Time, Accept vs Close Rate, Validation vs Close Rate) keep using consensus.ledger_close: a rate of that span is a valid per-close event counter, only its duration was wrong. Runbook updated to match. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../grafana/dashboards/consensus-health.json | 4 ++-- docs/telemetry-runbook.md | 24 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json index 17a69e3488..075c33e917 100644 --- a/docker/telemetry/grafana/dashboards/consensus-health.json +++ b/docker/telemetry/grafana/dashboards/consensus-health.json @@ -488,7 +488,7 @@ }, { "title": "Ledger Close Duration", - "description": "###### What this is:\n*Time from when consensus triggers a ledger close to when the close completes, at the 95th percentile.*\n\n###### How it's computed:\n*Per-close durations are aggregated to their 95th percentile per node over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; compare against the round time to see how much of it the close accounts for.*\n\n###### Healthy range:\n*A few to tens of milliseconds; workload-dependent.*\n\n###### Watch for:\n*A rising p95 indicates the close step is becoming a bottleneck, often under heavy transaction volume.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::onClose`", + "description": "###### What this is:\n*Full consensus round duration (open to accept) at the 95th percentile — the time a ledger takes to close.*\n\n###### How it's computed:\n*95th percentile of the consensus.round span duration per node over a 5-minute window.*\n\n###### Reading it:\n*Tracks the network close interval (~3-5s on mainnet); lower and steadier is better.*\n\n###### Healthy range:\n*A few seconds, matching the close cadence; workload-dependent on test networks.*\n\n###### Watch for:\n*A rising p95 means rounds are taking longer to converge.*\n\n###### Note:\n*Uses consensus.round, not consensus.ledger_close: the latter span only wraps the sub-millisecond onClose() prologue and is not the ledger close time.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::onClose (round span)`", "type": "timeseries", "gridPos": { "h": 8, @@ -509,7 +509,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", consensus_mode=~\"$consensus_mode\", span_name=\"consensus.ledger_close\"}[5m]))), \"series\", \"P95 Close Duration\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", consensus_mode=~\"$consensus_mode\", span_name=\"consensus.round\"}[5m]))), \"series\", \"P95 Close (consensus.round)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" } ], "fieldConfig": { diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 594fe9a931..8aa5a5844d 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -976,18 +976,18 @@ Ten dashboards are pre-provisioned in `docker/telemetry/grafana/dashboards/`: ### Consensus Health (`consensus-health`) -| Panel | Type | PromQL | Labels Used | -| ----------------------------- | ---------- | --------------------------------------------------------------------------- | ---------------- | -| Consensus Round Duration | timeseries | `histogram_quantile(0.95 / 0.50, ... {span_name="consensus.accept"})` | — | -| Consensus Proposals Sent Rate | timeseries | `rate(span_calls_total{span_name="consensus.proposal.send"}[5m])` | — | -| Ledger Close Duration | timeseries | `histogram_quantile(0.95, ... {span_name="consensus.ledger_close"})` | — | -| Validation Send Rate | stat | `rate(span_calls_total{span_name="consensus.validation.send"}[5m])` | — | -| Ledger Apply Duration | timeseries | `histogram_quantile(0.95 / 0.50, ... {span_name="consensus.accept.apply"})` | — | -| Close Time Agreement | timeseries | `rate(span_calls_total{span_name="consensus.accept.apply"}[5m])` | — | -| Consensus Mode Over Time | timeseries | `consensus.ledger_close` by `consensus_mode` | `consensus_mode` | -| Accept vs Close Rate | timeseries | `consensus.accept` vs `consensus.ledger_close` rate | — | -| Validation vs Close Rate | timeseries | `consensus.validation.send` vs `consensus.ledger_close` | — | -| Accept Duration Heatmap | heatmap | `consensus.accept` histogram buckets | `le` | +| Panel | Type | PromQL | Labels Used | +| ----------------------------- | ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | +| Consensus Round Duration | timeseries | `histogram_quantile(0.95 / 0.50, ... {span_name="consensus.accept"})` | — | +| Consensus Proposals Sent Rate | timeseries | `rate(span_calls_total{span_name="consensus.proposal.send"}[5m])` | — | +| Ledger Close Duration | timeseries | `histogram_quantile(0.95, ... {span_name="consensus.round"})` (full round, not `consensus.ledger_close` which is only the sub-ms onClose prologue) | `consensus_mode` | +| Validation Send Rate | stat | `rate(span_calls_total{span_name="consensus.validation.send"}[5m])` | — | +| Ledger Apply Duration | timeseries | `histogram_quantile(0.95 / 0.50, ... {span_name="consensus.accept.apply"})` | — | +| Close Time Agreement | timeseries | `rate(span_calls_total{span_name="consensus.accept.apply"}[5m])` | — | +| Consensus Mode Over Time | timeseries | `consensus.ledger_close` by `consensus_mode` | `consensus_mode` | +| Accept vs Close Rate | timeseries | `consensus.accept` vs `consensus.ledger_close` rate | — | +| Validation vs Close Rate | timeseries | `consensus.validation.send` vs `consensus.ledger_close` | — | +| Accept Duration Heatmap | heatmap | `consensus.accept` histogram buckets | `le` | ### Ledger Operations (`ledger-operations`)