From 24542c46c5653d67cc09b6925ce2aa1f84647f9e Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Fri, 10 Jul 2026 18:33:22 +0100 Subject: [PATCH] feat(telemetry): TxQ accept applied-ratio state timeline Replace the TxQ Accept Status piechart on the Transaction Overview dashboard with a state-timeline showing each node's applied fraction of TxQ accepts (applied / applied+failed) over time, colored by threshold (green >=0.9, yellow >=0.7, red below). Remove the now-orphaned txq_status template variable (the piechart was its only consumer) and document the panel in the telemetry runbook. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../dashboards/transaction-overview.json | 59 ++++++++++--------- docs/telemetry-runbook.md | 21 +++---- 2 files changed, 42 insertions(+), 38 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json index db70919e0a..c5d54e9e4a 100644 --- a/docker/telemetry/grafana/dashboards/transaction-overview.json +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -428,9 +428,9 @@ } }, { - "title": "TxQ Accept Status", - "description": "**What:** Outcomes when queued transactions are considered for a ledger: applied (included), failed (removed), or retried (kept for the next round).\n**How it's computed:** Total counts per outcome over a 5-minute window, shown as proportions.\n**Reading it:** Applied should dominate when the queue is draining healthily.\n**Healthy range:** Mostly applied; workload-dependent.\n**Watch for:** A rising retried or failed share signals queue pressure and fee escalation.\n**Source:** src/xrpld/app/misc/detail/TxQ.cpp:TxQ::accept", - "type": "piechart", + "title": "TxQ Accept: Applied Ratio per Node (State Timeline)", + "description": "**What:** Applied fraction of TxQ accepts per node over time: the share of queued transactions that were included in a ledger versus removed on failure.\n**How it's computed:** Per node, applied accepts divided by applied-plus-failed accepts over a 5-minute window.\n**Reading it:** Green (>=90% applied) is a healthy drain; yellow is degraded; red means accepts are mostly failing.\n**Healthy range:** At or near 100% applied when the queue is draining healthily; workload-dependent.\n**Watch for:** A node dropping into yellow or red, which signals queue pressure, under-bidding, or fee escalation on that node.\n**Source:** src/xrpld/app/misc/detail/TxQ.cpp:TxQ::accept", + "type": "state-timeline", "gridPos": { "h": 8, "w": 8, @@ -438,13 +438,17 @@ "y": 40 }, "options": { + "mergeValues": true, + "showValue": "auto", + "alignValue": "center", + "rowHeight": 0.85, "legend": { - "displayMode": "table", - "placement": "right", - "values": ["value", "percent"] + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, "tooltip": { - "mode": "multi" + "mode": "single" } }, "targets": [ @@ -452,13 +456,30 @@ "datasource": { "type": "prometheus" }, - "expr": "sum by (txq_status) (increase(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"txq.accept_tx\", txq_status=~\"$txq_status\"}[5m]))", - "legendFormat": "{{txq_status}}" + "expr": "sum by (service_instance_id) (increase(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"txq.accept_tx\", txq_status=\"applied\"}[5m]))\n/\nsum by (service_instance_id) (increase(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"txq.accept_tx\", txq_status=~\"applied|failed\"}[5m]))", + "legendFormat": "{{service_instance_id}}" } ], "fieldConfig": { "defaults": { - "unit": "short" + "unit": "percentunit", + "min": 0, + "max": 1, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": 0 }, + { "color": "yellow", "value": 0.7 }, + { "color": "green", "value": 0.9 } + ] + }, + "custom": { + "fillOpacity": 80, + "lineWidth": 0 + } }, "overrides": [] } @@ -987,24 +1008,6 @@ "sort": 1, "label": "Result Code" }, - { - "name": "txq_status", - "type": "query", - "datasource": { - "type": "prometheus" - }, - "query": "label_values(traces_span_metrics_calls_total{span_name=\"txq.accept_tx\", txq_status!=\"\"}, txq_status)", - "refresh": 2, - "includeAll": true, - "multi": true, - "allValue": ".*", - "current": { - "text": "All", - "value": "$__all" - }, - "sort": 1, - "label": "Queue Status" - }, { "name": "stage", "type": "query", diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 6574ceff0a..249ac3cdc8 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -628,16 +628,17 @@ Ten dashboards are pre-provisioned in `docker/telemetry/grafana/dashboards/`: ### Transaction Overview (`transaction-overview`) -| Panel | Type | PromQL | Labels Used | -| --------------------------------- | ---------- | ------------------------------------------------------------------------------------ | ------------- | -| Transaction Processing Rate | timeseries | `rate(traces_span_metrics_calls_total{span_name="tx.process"}[5m])` and `tx.receive` | `span_name` | -| Transaction Processing Latency | timeseries | `histogram_quantile(0.95 / 0.50, ... {span_name="tx.process"})` | — | -| Transaction Path Distribution | piechart | `sum by (local) (rate(traces_span_metrics_calls_total{span_name="tx.process"}[5m]))` | `local` | -| Transaction Receive vs Suppressed | timeseries | `rate(traces_span_metrics_calls_total{span_name="tx.receive"}[5m])` | — | -| TX Processing Duration Heatmap | heatmap | `tx.process` histogram buckets | `le` | -| TX Apply Duration per Ledger | timeseries | p95/p50 of `tx.apply` | — | -| Peer TX Receive Rate | timeseries | `tx.receive` rate | — | -| TX Apply Failed Rate | stat | `tx.apply` with `STATUS_CODE_ERROR` | `status_code` | +| Panel | Type | PromQL | Labels Used | +| ---------------------------------- | -------------- | --------------------------------------------------------------------------------------------------- | ----------------------------------- | +| Transaction Processing Rate | timeseries | `rate(traces_span_metrics_calls_total{span_name="tx.process"}[5m])` and `tx.receive` | `span_name` | +| Transaction Processing Latency | timeseries | `histogram_quantile(0.95 / 0.50, ... {span_name="tx.process"})` | — | +| Transaction Path Distribution | piechart | `sum by (local) (rate(traces_span_metrics_calls_total{span_name="tx.process"}[5m]))` | `local` | +| Transaction Receive vs Suppressed | timeseries | `rate(traces_span_metrics_calls_total{span_name="tx.receive"}[5m])` | — | +| TX Processing Duration Heatmap | heatmap | `tx.process` histogram buckets | `le` | +| TX Apply Duration per Ledger | timeseries | p95/p50 of `tx.apply` | — | +| Peer TX Receive Rate | timeseries | `tx.receive` rate | — | +| TX Apply Failed Rate | stat | `tx.apply` with `STATUS_CODE_ERROR` | `status_code` | +| TxQ Accept: Applied Ratio per Node | state-timeline | applied / (applied+failed) of `traces_span_metrics_calls_total{span_name="txq.accept_tx"}` per node | `txq_status`, `service_instance_id` | ### Consensus Health (`consensus-health`)