From c66f9a391d9b9c7ce4ad3155964c74237be2b1c1 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:05:56 +0100 Subject: [PATCH 1/2] fix(telemetry): state the real noun in dashboard rate units Grafana renders `unit: "ops"` as the literal string "ops/s", so every rate panel read as "operations per second" regardless of what it actually counted. `Ledger Build Rate` showed "0.258 ops/s" where the value is one ledger every 3.9s -- the number was right, the unit was meaningless. Replace the generic units with Grafana custom-suffix units naming the quantity, following the existing `suffix:/hr` and `si:drops` precedent in this repo. Nine of these are `stat` panels with no axis, so the unit string was the only text a reader ever saw. Also switch the two trusted/untrusted piecharts and the transaction path piechart from rate() to increase(): a per-slice "per second" reading is not a share of a total, counts in the window are. Queries are unchanged apart from those three; the values were already correct. Alongside, bring the touched panels up to the dashboard guidelines: tooltip mode/sort/max-height, 30-minute null spanning, and axis labels in title case. Hoist the stat panels above the fold on ledger-operations and rpc-performance. Panels that a later branch in this chain removes are deliberately left alone -- fixing them would only add merge conflicts. --- .../grafana/dashboards/consensus-health.json | 128 ++++++++++++------ .../grafana/dashboards/ledger-operations.json | 60 ++++---- .../grafana/dashboards/peer-network.json | 22 +-- .../grafana/dashboards/rpc-performance.json | 54 +++++--- .../dashboards/transaction-overview.json | 125 ++++++++++------- docs/telemetry-runbook.md | 14 +- 6 files changed, 246 insertions(+), 157 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json index 72eac8871b..3f03f9bbd7 100644 --- a/docker/telemetry/grafana/dashboards/consensus-health.json +++ b/docker/telemetry/grafana/dashboards/consensus-health.json @@ -21,7 +21,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -45,7 +46,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -67,7 +68,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -81,10 +83,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: proposals/s", "custom": { "axisLabel": "Proposals / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -106,7 +108,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -123,7 +126,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -159,7 +162,7 @@ ], "fieldConfig": { "defaults": { - "unit": "ops" + "unit": "suffix: validations/s" }, "overrides": [] } @@ -190,12 +193,19 @@ "legendFormat": "P50 Apply Duration [{{service_instance_id}}]" } ], + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, "fieldConfig": { "defaults": { "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -223,12 +233,19 @@ "legendFormat": "Close Time Correct={{close_time_correct}} [{{service_instance_id}}]" } ], + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: rounds/s", "custom": { "axisLabel": "Rounds / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -250,7 +267,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -264,10 +282,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: closes/s", "custom": { - "axisLabel": "Events / Sec", - "spanNulls": true, + "axisLabel": "Closes / Sec", + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -289,7 +307,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -310,10 +329,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: events/s", "custom": { "axisLabel": "Events / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -335,7 +354,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -356,10 +376,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: events/s", "custom": { "axisLabel": "Events / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -381,7 +401,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "yAxis": { "axisLabel": "Duration (ms)", @@ -400,7 +421,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ms" + "unit": "ms", + "custom": { + "spanNulls": 1800000 + } } } }, @@ -418,6 +442,7 @@ "defaults": { "unit": "dateTimeFromNow", "custom": { + "spanNulls": 1800000, "drawStyle": "points", "pointSize": 6, "showPoints": "always" @@ -428,7 +453,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -461,6 +487,7 @@ "defaults": { "unit": "dateTimeFromNow", "custom": { + "spanNulls": 1800000, "drawStyle": "points", "pointSize": 6, "showPoints": "always" @@ -471,7 +498,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -503,6 +531,7 @@ "fieldConfig": { "defaults": { "custom": { + "spanNulls": 1800000, "drawStyle": "line", "lineInterpolation": "stepAfter", "pointSize": 5, @@ -547,7 +576,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -587,6 +617,7 @@ "fieldConfig": { "defaults": { "custom": { + "spanNulls": 1800000, "drawStyle": "bars", "fillOpacity": 40, "pointSize": 5, @@ -598,7 +629,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -631,6 +663,7 @@ "defaults": { "unit": "short", "custom": { + "spanNulls": 1800000, "fillOpacity": 60 } }, @@ -639,7 +672,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -710,7 +744,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -731,10 +766,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: failures/s", "custom": { "axisLabel": "Failures / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -756,7 +791,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -780,7 +816,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -802,7 +838,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -826,7 +863,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -848,7 +885,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -872,7 +910,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -894,7 +932,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -915,10 +954,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: checks/s", "custom": { "axisLabel": "Checks / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -940,7 +979,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -954,10 +994,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: changes/s", "custom": { - "axisLabel": "Mode Changes / Sec", - "spanNulls": true, + "axisLabel": "Changes / Sec", + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 diff --git a/docker/telemetry/grafana/dashboards/ledger-operations.json b/docker/telemetry/grafana/dashboards/ledger-operations.json index 2cf30b1f16..37673e5b62 100644 --- a/docker/telemetry/grafana/dashboards/ledger-operations.json +++ b/docker/telemetry/grafana/dashboards/ledger-operations.json @@ -14,7 +14,7 @@ "type": "stat", "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, "y": 0 }, @@ -35,7 +35,7 @@ ], "fieldConfig": { "defaults": { - "unit": "ops" + "unit": "suffix: ledgers/s" }, "overrides": [] } @@ -47,13 +47,14 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 0 + "x": 0, + "y": 8 }, "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -77,7 +78,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -92,9 +93,9 @@ "type": "stat", "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 8 + "w": 8, + "x": 8, + "y": 0 }, "options": { "tooltip": { @@ -113,7 +114,7 @@ ], "fieldConfig": { "defaults": { - "unit": "ops" + "unit": "suffix: ledgers/s" }, "overrides": [] } @@ -131,7 +132,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "yAxis": { "axisLabel": "Duration (ms)" @@ -149,7 +151,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ms" + "unit": "ms", + "custom": { + "spanNulls": 1800000 + } }, "overrides": [] } @@ -167,7 +172,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -191,7 +197,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -213,7 +219,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -227,10 +234,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: transactions/s", "custom": { - "axisLabel": "Operations / Sec", - "spanNulls": true, + "axisLabel": "Transactions / Sec", + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -245,9 +252,9 @@ "type": "stat", "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 24 + "w": 8, + "x": 16, + "y": 0 }, "options": { "tooltip": { @@ -266,7 +273,7 @@ ], "fieldConfig": { "defaults": { - "unit": "ops" + "unit": "suffix: ledgers/s" }, "overrides": [] } @@ -277,14 +284,15 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 12, - "x": 12, + "w": 24, + "x": 0, "y": 24 }, "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -308,7 +316,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 diff --git a/docker/telemetry/grafana/dashboards/peer-network.json b/docker/telemetry/grafana/dashboards/peer-network.json index 56f0d756d8..9d9de27f58 100644 --- a/docker/telemetry/grafana/dashboards/peer-network.json +++ b/docker/telemetry/grafana/dashboards/peer-network.json @@ -22,7 +22,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -36,10 +37,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: proposals/s", "custom": { "axisLabel": "Proposals / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -61,7 +62,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -75,10 +77,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: validations/s", "custom": { "axisLabel": "Validations / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -108,13 +110,13 @@ "datasource": { "type": "prometheus" }, - "expr": "sum by (proposal_trusted, service_instance_id) (rate(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", proposal_trusted=~\"$proposal_trusted\", span_name=\"peer.proposal.receive\"}[5m]))", + "expr": "sum by (proposal_trusted, service_instance_id) (increase(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", proposal_trusted=~\"$proposal_trusted\", span_name=\"peer.proposal.receive\"}[5m]))", "legendFormat": "Trusted = {{proposal_trusted}} [{{service_instance_id}}]" } ], "fieldConfig": { "defaults": { - "unit": "ops" + "unit": "short" }, "overrides": [] } @@ -140,13 +142,13 @@ "datasource": { "type": "prometheus" }, - "expr": "sum by (validation_trusted, service_instance_id) (rate(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", validation_trusted=~\"$validation_trusted\", span_name=\"peer.validation.receive\"}[5m]))", + "expr": "sum by (validation_trusted, service_instance_id) (increase(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", validation_trusted=~\"$validation_trusted\", span_name=\"peer.validation.receive\"}[5m]))", "legendFormat": "Trusted = {{validation_trusted}} [{{service_instance_id}}]" } ], "fieldConfig": { "defaults": { - "unit": "ops" + "unit": "short" }, "overrides": [] } diff --git a/docker/telemetry/grafana/dashboards/rpc-performance.json b/docker/telemetry/grafana/dashboards/rpc-performance.json index 9fe962e6a3..b54b98a174 100644 --- a/docker/telemetry/grafana/dashboards/rpc-performance.json +++ b/docker/telemetry/grafana/dashboards/rpc-performance.json @@ -16,12 +16,13 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 24 }, "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -38,7 +39,7 @@ "unit": "reqps", "custom": { "axisLabel": "Requests / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -55,12 +56,13 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 24 }, "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -77,7 +79,7 @@ "unit": "ms", "custom": { "axisLabel": "Latency (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -147,7 +149,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "yAxis": { "axisLabel": "Duration (ms)", @@ -166,7 +169,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ms" + "unit": "ms", + "custom": { + "spanNulls": 1800000 + } } } }, @@ -183,7 +189,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -207,7 +214,7 @@ "unit": "reqps", "custom": { "axisLabel": "Requests / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -229,7 +236,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -250,10 +258,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: commands/s", "custom": { "axisLabel": "Commands / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -270,7 +278,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 0 }, "options": { "tooltip": { @@ -302,7 +310,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 0 }, "options": { "tooltip": { @@ -321,7 +329,7 @@ ], "fieldConfig": { "defaults": { - "unit": "ops" + "unit": "suffix: messages/s" }, "overrides": [] } @@ -339,7 +347,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -358,10 +367,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: requests/s", "custom": { "axisLabel": "Requests / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -383,7 +392,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -404,10 +414,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: requests/s", "custom": { "axisLabel": "Requests / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json index c5d54e9e4a..4efec5de79 100644 --- a/docker/telemetry/grafana/dashboards/transaction-overview.json +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -21,7 +21,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -42,10 +43,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: transactions/s", "custom": { "axisLabel": "Transactions / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -67,7 +68,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -96,7 +98,7 @@ "unit": "ms", "custom": { "axisLabel": "Latency (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -126,10 +128,16 @@ "datasource": { "type": "prometheus" }, - "expr": "sum by (local, service_instance_id) (rate(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", local=~\"$tx_origin\", span_name=\"tx.process\", tx_type=~\"$tx_type\"}[5m]))", + "expr": "sum by (local, service_instance_id) (increase(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", local=~\"$tx_origin\", span_name=\"tx.process\", tx_type=~\"$tx_type\"}[5m]))", "legendFormat": "Local = {{local}} [{{service_instance_id}}]" } - ] + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } }, { "title": "Transaction Receive vs Suppressed", @@ -144,7 +152,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -158,10 +167,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: transactions/s", "custom": { "axisLabel": "Transactions / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -183,7 +192,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "yAxis": { "axisLabel": "Duration (ms)", @@ -202,7 +212,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ms" + "unit": "ms", + "custom": { + "spanNulls": 1800000 + } } } }, @@ -219,7 +232,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -243,7 +257,7 @@ "unit": "ms", "custom": { "axisLabel": "Latency (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -265,7 +279,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -282,7 +297,7 @@ "unit": "ops", "custom": { "axisLabel": "Transactions / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -318,7 +333,7 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: transactions/s", "thresholds": { "steps": [ { @@ -352,7 +367,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -371,10 +387,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: transactions/s", "custom": { - "axisLabel": "TX / Sec", - "spanNulls": true, + "axisLabel": "Transactions / Sec", + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -396,7 +412,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -415,10 +432,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: transactions/s", "custom": { - "axisLabel": "Failed TX / Sec", - "spanNulls": true, + "axisLabel": "Transactions / Sec", + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -448,7 +465,9 @@ "showLegend": true }, "tooltip": { - "mode": "single" + "mode": "multi", + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -477,6 +496,7 @@ ] }, "custom": { + "spanNulls": 1800000, "fillOpacity": 80, "lineWidth": 0 } @@ -497,7 +517,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -519,7 +540,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -541,7 +562,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -555,10 +577,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: enqueues/s", "custom": { "axisLabel": "Enqueues / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -580,7 +602,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -597,7 +620,7 @@ "unit": "percentunit", "custom": { "axisLabel": "Bypass Fraction", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -619,7 +642,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -643,7 +667,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -665,7 +689,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -679,10 +704,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: cleanups/s", "custom": { "axisLabel": "Cleanups / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -704,7 +729,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -726,7 +752,7 @@ "unit": "ops", "custom": { "axisLabel": "Spans / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -748,7 +774,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -770,7 +797,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -792,7 +819,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -811,10 +839,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: failures/s", "custom": { - "axisLabel": "Failed Spans / Sec", - "spanNulls": true, + "axisLabel": "Failures / Sec", + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -836,7 +864,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -858,7 +887,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 2a0ff66091..ef4d3f2dec 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -667,7 +667,7 @@ Ten dashboards are pre-provisioned in `docker/telemetry/grafana/dashboards/`: | ---------------------------------- | -------------- | --------------------------------------------------------------------------------------------------- | ----------------------------------- | | Transaction Processing Rate | timeseries | `rate(traces_span_metrics_calls_total{span_name="tx.process"}[5m])` and `tx.receive` | `span_name` | | Transaction Processing Latency | timeseries | `histogram_quantile(0.95 / 0.50, ... {span_name="tx.process"})` | — | -| Transaction Path Distribution | piechart | `sum by (local) (rate(traces_span_metrics_calls_total{span_name="tx.process"}[5m]))` | `local` | +| Transaction Path Distribution | piechart | `sum by (local) (increase(traces_span_metrics_calls_total{span_name="tx.process"}[5m]))` | `local` | | Transaction Receive vs Suppressed | timeseries | `rate(traces_span_metrics_calls_total{span_name="tx.receive"}[5m])` | — | | TX Processing Duration Heatmap | heatmap | `tx.process` histogram buckets | `le` | | TX Apply Duration per Ledger | timeseries | p95/p50 of `tx.apply` | — | @@ -707,12 +707,12 @@ Ten dashboards are pre-provisioned in `docker/telemetry/grafana/dashboards/`: Requires `trace_peer=1` in the `[telemetry]` config section. -| Panel | Type | PromQL | Labels Used | -| -------------------------------- | ---------- | ------------------------------ | -------------------- | -| Proposal Receive Rate | timeseries | `peer.proposal.receive` rate | — | -| Validation Receive Rate | timeseries | `peer.validation.receive` rate | — | -| Proposals Trusted vs Untrusted | piechart | by `proposal_trusted` | `proposal_trusted` | -| Validations Trusted vs Untrusted | piechart | by `validation_trusted` | `validation_trusted` | +| Panel | Type | PromQL | Labels Used | +| -------------------------------- | ---------- | ------------------------------------------------------------------------- | -------------------- | +| Proposal Receive Rate | timeseries | `peer.proposal.receive` rate | — | +| Validation Receive Rate | timeseries | `peer.validation.receive` rate | — | +| Proposals Trusted vs Untrusted | piechart | `increase()` counts in the selected window, split by `proposal_trusted` | `proposal_trusted` | +| Validations Trusted vs Untrusted | piechart | `increase()` counts in the selected window, split by `validation_trusted` | `validation_trusted` | ### Node Health -- StatsD (`xrpld-statsd-node-health`) From 2095a3d2f17238cad2ee07bc7425b5c4fdf4a0f6 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:06:23 +0100 Subject: [PATCH 2/2] fix(telemetry): correct job latency units and name dashboard rate nouns The four job-latency panels on node-health declared milliseconds while querying `job_running_us` / `job_queued_us`, which record microseconds (MetricsRegistry records the raw value, and the instrument description says microseconds). Every reading was therefore a thousand times too large: the p95 for acceptLedger, 241495us, rendered as "241 sec" instead of 241ms. job-queue.json already read these same metrics as microseconds, so the two dashboards disagreed by 1000x on identical data. Switch node-health to microseconds to match. Also replace the generic `ops` and `cps` units, which Grafana renders as the literal "ops/s" and "counts/s", with custom-suffix units naming what each panel counts -- messages, fetches, calls, mismatches. Two panels plot more than one quantity on a single axis, which no single unit can describe. Give each series its own unit through field overrides: reads per second beside two queue depths on NuDB Read Pressure, and ledgers beside fetches on Ledger Close Rate, the latter on a right-hand axis. State Duration Rate plots a seconds-per-second time share, which can exceed 1.0 and so is not a percentage; label it as the ratio it is. The normalised share already exists as its own panel. Queries are unchanged; the values were already correct. Alongside, bring the touched panels up to the dashboard guidelines and hoist the stat panels above the fold. --- .../grafana/dashboards/ledger-data-sync.json | 85 +++++++++++++++---- .../grafana/dashboards/network-traffic.json | 20 ++--- .../grafana/dashboards/node-health.json | 60 +++++++------ .../dashboards/overlay-traffic-detail.json | 12 +-- .../grafana/dashboards/rpc-pathfinding.json | 48 ++++++----- docs/telemetry-runbook.md | 40 ++++----- 6 files changed, 168 insertions(+), 97 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index 42916eb34a..2f01f834d9 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -496,9 +496,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages In", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -543,9 +543,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages In", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -584,9 +584,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages In", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -637,9 +637,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages In", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -985,8 +985,9 @@ "alignValue": "left", "rowHeight": 0.9, "tooltip": { - "mode": "single", - "sort": "none" + "maxHeight": 600, + "mode": "multi", + "sort": "desc" }, "legend": { "displayMode": "list", @@ -1012,7 +1013,7 @@ "fillOpacity": 80, "lineWidth": 0, "insertNulls": false, - "spanNulls": false + "spanNulls": 1800000 }, "mappings": [ { @@ -1117,16 +1118,33 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "reqps", + "unit": "suffix: ledgers/s", "custom": { - "axisLabel": "Rate (per second)", + "axisLabel": "Ledgers / Sec & Fetches / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^Ledger Fetches/" + }, + "properties": [ + { + "id": "unit", + "value": "suffix: fetches/s" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] }, "id": 19 }, @@ -1376,16 +1394,49 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "short", + "unit": "suffix: reads/s", "custom": { - "axisLabel": "Reads/s & Depth", + "axisLabel": "Reads / Sec & Depth", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^Read Queue/" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^Read Threads Running/" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] }, "id": 24 }, diff --git a/docker/telemetry/grafana/dashboards/network-traffic.json b/docker/telemetry/grafana/dashboards/network-traffic.json index 7323a32bef..724f05f652 100644 --- a/docker/telemetry/grafana/dashboards/network-traffic.json +++ b/docker/telemetry/grafana/dashboards/network-traffic.json @@ -63,7 +63,7 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 24 }, "options": { "tooltip": { @@ -175,9 +175,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -227,9 +227,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -285,9 +285,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -343,9 +343,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -363,7 +363,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 0 }, "options": { "tooltip": { diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index a21bb2d574..44ea4cc5cb 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -113,10 +113,11 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 24 }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -163,7 +164,7 @@ "unit": "percentunit", "custom": { "axisLabel": "Time in mode", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3, @@ -183,10 +184,11 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 24 }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -233,7 +235,7 @@ "unit": "short", "custom": { "axisLabel": "Transitions", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -254,6 +256,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -279,7 +282,7 @@ "unit": "ms", "custom": { "axisLabel": "Latency (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -300,6 +303,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -318,7 +322,7 @@ "unit": "short", "custom": { "axisLabel": "Jobs", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -335,7 +339,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 8 }, "options": { "tooltip": { @@ -354,7 +358,7 @@ ], "fieldConfig": { "defaults": { - "unit": "ops" + "unit": "suffix: fetches/s" }, "overrides": [] } @@ -367,7 +371,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 8 }, "options": { "tooltip": { @@ -386,7 +390,7 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: mismatches/s", "thresholds": { "steps": [ { @@ -427,6 +431,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -512,10 +517,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ms", + "unit": "µs", "custom": { - "axisLabel": "Duration (ms)", - "spanNulls": true, + "axisLabel": "Duration (µs)", + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -536,6 +541,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -621,10 +627,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ms", + "unit": "µs", "custom": { - "axisLabel": "Wait Time (ms)", - "spanNulls": true, + "axisLabel": "Wait Time (µs)", + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -645,6 +651,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -663,7 +670,7 @@ "unit": "short", "custom": { "axisLabel": "Entries", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -782,6 +789,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -804,10 +812,10 @@ ], "fieldConfig": { "defaults": { - "unit": "short", + "unit": "suffix: s/s", "custom": { "axisLabel": "Rate (s/s)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -828,6 +836,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -843,10 +852,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ms", + "unit": "µs", "custom": { - "axisLabel": "Duration (ms)", - "spanNulls": true, + "axisLabel": "Duration (µs)", + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -867,6 +876,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -882,10 +892,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ms", + "unit": "µs", "custom": { - "axisLabel": "Wait Time (ms)", - "spanNulls": true, + "axisLabel": "Wait Time (µs)", + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 diff --git a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json index f86a75b68c..2b1ffc8ea1 100644 --- a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json +++ b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json @@ -67,9 +67,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -195,9 +195,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Count", + "axisLabel": "Messages / Sec & Bytes / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -328,9 +328,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", diff --git a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json index 28f9a51ad5..bae50fa772 100644 --- a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json +++ b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json @@ -54,6 +54,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -81,7 +82,7 @@ "unit": "ms", "custom": { "axisLabel": "Latency (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -98,10 +99,11 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 24 }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -129,7 +131,7 @@ "unit": "decbytes", "custom": { "axisLabel": "Size (Bytes)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -146,10 +148,11 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 24 }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -193,7 +196,7 @@ "unit": "ms", "custom": { "axisLabel": "Latency (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -214,6 +217,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -241,7 +245,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -262,6 +266,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -289,7 +294,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -306,7 +311,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 8 }, "options": { "tooltip": { @@ -326,7 +331,7 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: warnings/s", "thresholds": { "steps": [ { @@ -355,7 +360,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 8 }, "options": { "tooltip": { @@ -375,7 +380,7 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: drops/s", "thresholds": { "steps": [ { @@ -408,6 +413,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -424,10 +430,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: calls/s", "custom": { "axisLabel": "Calls / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -448,6 +454,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -467,7 +474,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -488,6 +495,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -504,10 +512,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: calls/s", "custom": { "axisLabel": "Calls / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -528,6 +536,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -555,7 +564,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -576,6 +585,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -600,10 +610,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: operations/s", "custom": { "axisLabel": "Operations / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 57a5c7ecba..22aded49fc 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -718,24 +718,24 @@ Requires `trace_peer=1` in the `[telemetry]` config section. ### Node Health -- System Metrics (`node-health`) -| Panel | Type | PromQL | Labels Used | -| -------------------------------------- | ---------- | ---------------------------------------------------------- | ----------- | -| Validated Ledger Age | stat | `ledgermaster_validated_ledger_age` | — | -| Published Ledger Age | stat | `ledgermaster_published_ledger_age` | — | -| Operating Mode (Time Share) | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))` | — | -| Operating Mode Transitions | timeseries | `state_accounting_*_transitions` | — | -| I/O Latency | timeseries | `histogram_quantile(0.95, ios_latency_bucket)` | — | -| Job Queue Depth | timeseries | `job_count` | — | -| Ledger Fetch Rate | stat | `rate(ledger_fetches[5m])` | — | -| Ledger History Mismatches | stat | `rate(ledger_history_mismatch[5m])` | — | -| Key Jobs Execution Time | timeseries | `acceptledger{quantile="$quantile"}` (+ 10 more key jobs) | `quantile` | -| Key Jobs Dequeue Wait Time | timeseries | `acceptledger_q{quantile="$quantile"}` (+ 10 more) | `quantile` | -| FullBelowCache Size | timeseries | `node_family_full_below_cache_size` | — | -| FullBelowCache Hit Rate | gauge | `node_family_full_below_cache_hit_rate` | — | -| Ledger Publish Gap | stat | `Published_Ledger_Age - Validated_Ledger_Age` | — | -| State Duration Rate (Full vs Tracking) | timeseries | `rate(state_accounting_full_duration[5m]) / 1000000` | — | -| All Jobs Execution Time (Detail) | timeseries | `{__name__=~"", quantile="$quantile"}` | `quantile` | -| All Jobs Dequeue Wait (Detail) | timeseries | `{__name__=~"_q", quantile="$quantile"}` | `quantile` | +| Panel | Type | PromQL | Labels Used | +| -------------------------------------- | ---------- | --------------------------------------------------------------------------------- | ----------- | +| Validated Ledger Age | stat | `ledgermaster_validated_ledger_age` | — | +| Published Ledger Age | stat | `ledgermaster_published_ledger_age` | — | +| Operating Mode (Time Share) | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))` | — | +| Operating Mode Transitions | timeseries | `state_accounting_*_transitions` | — | +| I/O Latency | timeseries | `histogram_quantile(0.95, ios_latency_bucket)` | — | +| Job Queue Depth | timeseries | `job_count` | — | +| Ledger Fetch Rate | stat | `rate(ledger_fetches_total[$__rate_interval])` | — | +| Ledger History Mismatches | stat | `rate(ledger_history_mismatch_total[$__rate_interval])` | — | +| Key Jobs Execution Time | timeseries | `acceptledger{quantile="$quantile"}` (+ 10 more key jobs) | `quantile` | +| Key Jobs Dequeue Wait Time | timeseries | `acceptledger_q{quantile="$quantile"}` (+ 10 more) | `quantile` | +| FullBelowCache Size | timeseries | `node_family_full_below_cache_size` | — | +| FullBelowCache Hit Rate | gauge | `node_family_full_below_cache_hit_rate` | — | +| Ledger Publish Gap | stat | `Published_Ledger_Age - Validated_Ledger_Age` | — | +| State Duration Rate (Full vs Tracking) | timeseries | `rate(state_accounting_full_duration[5m]) / 1000000` | — | +| All Jobs Execution Time (Detail) | timeseries | `histogram_quantile($quantile, rate(job_running_us_bucket[5m])) by job_type` — µs | `quantile` | +| All Jobs Dequeue Wait (Detail) | timeseries | `histogram_quantile($quantile, rate(job_queued_us_bucket[5m])) by job_type` — µs | `quantile` | ### Network Traffic -- System Metrics (`network-traffic`) @@ -762,8 +762,8 @@ Requires `trace_peer=1` in the `[telemetry]` config section. | RPC Response Time Heatmap | heatmap | `rpc_time_bucket` | — | | Pathfinding Fast Duration | timeseries | `histogram_quantile(0.95, pathfind_fast_bucket)` | — | | Pathfinding Full Duration | timeseries | `histogram_quantile(0.95, pathfind_full_bucket)` | — | -| Resource Warnings Rate | stat | `rate(warn[5m])` | — | -| Resource Drops Rate | stat | `rate(drop[5m])` | — | +| Resource Warnings Rate | stat | `rate(warn_total[$__rate_interval])` | — | +| Resource Drops Rate | stat | `rate(drop_total[$__rate_interval])` | — | ### Span → Metric → Dashboard Summary