From 84ef8cbf3355a752ced56916d377ad4e664343d4 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Thu, 6 Aug 2026 19:01:15 +0100 Subject: [PATCH] fix(telemetry): drop unmeasurable Queue Bypass Ratio panel The Transaction Overview panel "Queue Bypass Ratio (Direct Apply vs Enqueue)" reported a confident 0.50 on every node while the true bypass rate was zero. The two spans it divided are not disjoint alternatives: txq.apply_direct is a child of txq.enqueue. TxQ.cpp creates the apply_direct span as the first statement of tryDirectApply(), ahead of the account, sequence and fee-level guards, and tryDirectApply() is called from inside the live enqueue scope. The span therefore counts attempts, so the denominator direct + enqueue counts each transaction twice and pins the ratio to one half algebraically. Measured on a four-node fleet: 6082443 direct against 6082877 enqueue over the same population, panel output 0.5000170 on three nodes and 0.5000000 on the fourth. Grouping txq.enqueue by txq_status over seven days returns only "rejected" -- no transaction has ever taken the direct-apply path. Remove the panel rather than repoint it. A correct expression using txq_status as the disjoint discriminator would render permanently empty on this fleet, which reads no better than a wrong number. Widen the band partner "TxQ Enqueue Rate by Transaction Type" from 12 to 24 columns so the y=48 band still fills the grid. Every band in all ten dashboards sums to 24 columns; leaving a half-width hole would be the only exception. Panel order and every other panel's position, width and height are unchanged. The runbook already listed txq.apply_direct as available but not paneled, so that row becomes accurate. Rows describing the span itself are untouched -- the span and its metric are unchanged. --- .../dashboards/transaction-overview.json | 42 +------------------ 1 file changed, 1 insertion(+), 41 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json index 4efec5de79..94cc8be828 100644 --- a/docker/telemetry/grafana/dashboards/transaction-overview.json +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -555,7 +555,7 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 12, + "w": 24, "x": 0, "y": 48 }, @@ -589,46 +589,6 @@ "overrides": [] } }, - { - "title": "Queue Bypass Ratio (Direct Apply vs Enqueue)", - "description": "**What:** Fraction of transactions that applied straight to the open ledger versus those that had to be queued.\n**How it's computed:** Ratio of direct applies to direct-plus-queued over a 5-minute window, per node.\n**Reading it:** A high fraction means the network is keeping up without escalation.\n**Healthy range:** Near 1.0 in normal conditions.\n**Watch for:** A falling fraction is the cleanest single signal the network has entered sustained fee escalation.\n**Source:** src/xrpld/app/misc/detail/TxQ.cpp:TxQ::tryDirectApply, TxQ::apply", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 48 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc", - "maxHeight": 600 - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "sum by (service_instance_id) (rate(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"txq.apply_direct\"}[5m])) / clamp_min(sum by (service_instance_id) (rate(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"txq.apply_direct\"}[5m])) + sum by (service_instance_id) (rate(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"txq.enqueue\"}[5m])), 1)", - "legendFormat": "Direct-Apply Fraction [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "custom": { - "axisLabel": "Bypass Fraction", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 3 - } - }, - "overrides": [] - } - }, { "title": "Queue Accept (Drain) Duration per Ledger", "description": "**What:** Time spent draining queued transactions into a newly closed ledger.\n**How it's computed:** 95th and 50th percentile of per-ledger drain durations over a 5-minute window, per node.\n**Reading it:** Rises as the queue holds more transactions to process at close.\n**Healthy range:** A few milliseconds when the queue is light.\n**Watch for:** Rising drain time signals queue pressure at ledger close.\n**Source:** src/xrpld/app/misc/detail/TxQ.cpp:TxQ::accept",