From c66f9a391d9b9c7ce4ad3155964c74237be2b1c1 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:05:56 +0100 Subject: [PATCH] fix(telemetry): state the real noun in dashboard rate units Grafana renders `unit: "ops"` as the literal string "ops/s", so every rate panel read as "operations per second" regardless of what it actually counted. `Ledger Build Rate` showed "0.258 ops/s" where the value is one ledger every 3.9s -- the number was right, the unit was meaningless. Replace the generic units with Grafana custom-suffix units naming the quantity, following the existing `suffix:/hr` and `si:drops` precedent in this repo. Nine of these are `stat` panels with no axis, so the unit string was the only text a reader ever saw. Also switch the two trusted/untrusted piecharts and the transaction path piechart from rate() to increase(): a per-slice "per second" reading is not a share of a total, counts in the window are. Queries are unchanged apart from those three; the values were already correct. Alongside, bring the touched panels up to the dashboard guidelines: tooltip mode/sort/max-height, 30-minute null spanning, and axis labels in title case. Hoist the stat panels above the fold on ledger-operations and rpc-performance. Panels that a later branch in this chain removes are deliberately left alone -- fixing them would only add merge conflicts. --- .../grafana/dashboards/consensus-health.json | 128 ++++++++++++------ .../grafana/dashboards/ledger-operations.json | 60 ++++---- .../grafana/dashboards/peer-network.json | 22 +-- .../grafana/dashboards/rpc-performance.json | 54 +++++--- .../dashboards/transaction-overview.json | 125 ++++++++++------- docs/telemetry-runbook.md | 14 +- 6 files changed, 246 insertions(+), 157 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json index 72eac8871b..3f03f9bbd7 100644 --- a/docker/telemetry/grafana/dashboards/consensus-health.json +++ b/docker/telemetry/grafana/dashboards/consensus-health.json @@ -21,7 +21,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -45,7 +46,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -67,7 +68,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -81,10 +83,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: proposals/s", "custom": { "axisLabel": "Proposals / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -106,7 +108,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -123,7 +126,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -159,7 +162,7 @@ ], "fieldConfig": { "defaults": { - "unit": "ops" + "unit": "suffix: validations/s" }, "overrides": [] } @@ -190,12 +193,19 @@ "legendFormat": "P50 Apply Duration [{{service_instance_id}}]" } ], + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, "fieldConfig": { "defaults": { "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -223,12 +233,19 @@ "legendFormat": "Close Time Correct={{close_time_correct}} [{{service_instance_id}}]" } ], + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: rounds/s", "custom": { "axisLabel": "Rounds / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -250,7 +267,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -264,10 +282,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: closes/s", "custom": { - "axisLabel": "Events / Sec", - "spanNulls": true, + "axisLabel": "Closes / Sec", + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -289,7 +307,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -310,10 +329,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: events/s", "custom": { "axisLabel": "Events / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -335,7 +354,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -356,10 +376,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: events/s", "custom": { "axisLabel": "Events / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -381,7 +401,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "yAxis": { "axisLabel": "Duration (ms)", @@ -400,7 +421,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ms" + "unit": "ms", + "custom": { + "spanNulls": 1800000 + } } } }, @@ -418,6 +442,7 @@ "defaults": { "unit": "dateTimeFromNow", "custom": { + "spanNulls": 1800000, "drawStyle": "points", "pointSize": 6, "showPoints": "always" @@ -428,7 +453,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -461,6 +487,7 @@ "defaults": { "unit": "dateTimeFromNow", "custom": { + "spanNulls": 1800000, "drawStyle": "points", "pointSize": 6, "showPoints": "always" @@ -471,7 +498,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -503,6 +531,7 @@ "fieldConfig": { "defaults": { "custom": { + "spanNulls": 1800000, "drawStyle": "line", "lineInterpolation": "stepAfter", "pointSize": 5, @@ -547,7 +576,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -587,6 +617,7 @@ "fieldConfig": { "defaults": { "custom": { + "spanNulls": 1800000, "drawStyle": "bars", "fillOpacity": 40, "pointSize": 5, @@ -598,7 +629,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -631,6 +663,7 @@ "defaults": { "unit": "short", "custom": { + "spanNulls": 1800000, "fillOpacity": 60 } }, @@ -639,7 +672,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -710,7 +744,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -731,10 +766,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: failures/s", "custom": { "axisLabel": "Failures / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -756,7 +791,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -780,7 +816,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -802,7 +838,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -826,7 +863,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -848,7 +885,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -872,7 +910,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -894,7 +932,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -915,10 +954,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: checks/s", "custom": { "axisLabel": "Checks / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -940,7 +979,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -954,10 +994,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: changes/s", "custom": { - "axisLabel": "Mode Changes / Sec", - "spanNulls": true, + "axisLabel": "Changes / Sec", + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 diff --git a/docker/telemetry/grafana/dashboards/ledger-operations.json b/docker/telemetry/grafana/dashboards/ledger-operations.json index 2cf30b1f16..37673e5b62 100644 --- a/docker/telemetry/grafana/dashboards/ledger-operations.json +++ b/docker/telemetry/grafana/dashboards/ledger-operations.json @@ -14,7 +14,7 @@ "type": "stat", "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, "y": 0 }, @@ -35,7 +35,7 @@ ], "fieldConfig": { "defaults": { - "unit": "ops" + "unit": "suffix: ledgers/s" }, "overrides": [] } @@ -47,13 +47,14 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 0 + "x": 0, + "y": 8 }, "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -77,7 +78,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -92,9 +93,9 @@ "type": "stat", "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 8 + "w": 8, + "x": 8, + "y": 0 }, "options": { "tooltip": { @@ -113,7 +114,7 @@ ], "fieldConfig": { "defaults": { - "unit": "ops" + "unit": "suffix: ledgers/s" }, "overrides": [] } @@ -131,7 +132,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "yAxis": { "axisLabel": "Duration (ms)" @@ -149,7 +151,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ms" + "unit": "ms", + "custom": { + "spanNulls": 1800000 + } }, "overrides": [] } @@ -167,7 +172,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -191,7 +197,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -213,7 +219,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -227,10 +234,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: transactions/s", "custom": { - "axisLabel": "Operations / Sec", - "spanNulls": true, + "axisLabel": "Transactions / Sec", + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -245,9 +252,9 @@ "type": "stat", "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 24 + "w": 8, + "x": 16, + "y": 0 }, "options": { "tooltip": { @@ -266,7 +273,7 @@ ], "fieldConfig": { "defaults": { - "unit": "ops" + "unit": "suffix: ledgers/s" }, "overrides": [] } @@ -277,14 +284,15 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 12, - "x": 12, + "w": 24, + "x": 0, "y": 24 }, "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -308,7 +316,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 diff --git a/docker/telemetry/grafana/dashboards/peer-network.json b/docker/telemetry/grafana/dashboards/peer-network.json index 56f0d756d8..9d9de27f58 100644 --- a/docker/telemetry/grafana/dashboards/peer-network.json +++ b/docker/telemetry/grafana/dashboards/peer-network.json @@ -22,7 +22,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -36,10 +37,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: proposals/s", "custom": { "axisLabel": "Proposals / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -61,7 +62,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -75,10 +77,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: validations/s", "custom": { "axisLabel": "Validations / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -108,13 +110,13 @@ "datasource": { "type": "prometheus" }, - "expr": "sum by (proposal_trusted, service_instance_id) (rate(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", proposal_trusted=~\"$proposal_trusted\", span_name=\"peer.proposal.receive\"}[5m]))", + "expr": "sum by (proposal_trusted, service_instance_id) (increase(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", proposal_trusted=~\"$proposal_trusted\", span_name=\"peer.proposal.receive\"}[5m]))", "legendFormat": "Trusted = {{proposal_trusted}} [{{service_instance_id}}]" } ], "fieldConfig": { "defaults": { - "unit": "ops" + "unit": "short" }, "overrides": [] } @@ -140,13 +142,13 @@ "datasource": { "type": "prometheus" }, - "expr": "sum by (validation_trusted, service_instance_id) (rate(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", validation_trusted=~\"$validation_trusted\", span_name=\"peer.validation.receive\"}[5m]))", + "expr": "sum by (validation_trusted, service_instance_id) (increase(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", validation_trusted=~\"$validation_trusted\", span_name=\"peer.validation.receive\"}[5m]))", "legendFormat": "Trusted = {{validation_trusted}} [{{service_instance_id}}]" } ], "fieldConfig": { "defaults": { - "unit": "ops" + "unit": "short" }, "overrides": [] } diff --git a/docker/telemetry/grafana/dashboards/rpc-performance.json b/docker/telemetry/grafana/dashboards/rpc-performance.json index 9fe962e6a3..b54b98a174 100644 --- a/docker/telemetry/grafana/dashboards/rpc-performance.json +++ b/docker/telemetry/grafana/dashboards/rpc-performance.json @@ -16,12 +16,13 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 24 }, "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -38,7 +39,7 @@ "unit": "reqps", "custom": { "axisLabel": "Requests / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -55,12 +56,13 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 24 }, "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -77,7 +79,7 @@ "unit": "ms", "custom": { "axisLabel": "Latency (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -147,7 +149,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "yAxis": { "axisLabel": "Duration (ms)", @@ -166,7 +169,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ms" + "unit": "ms", + "custom": { + "spanNulls": 1800000 + } } } }, @@ -183,7 +189,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -207,7 +214,7 @@ "unit": "reqps", "custom": { "axisLabel": "Requests / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -229,7 +236,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -250,10 +258,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: commands/s", "custom": { "axisLabel": "Commands / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -270,7 +278,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 0 }, "options": { "tooltip": { @@ -302,7 +310,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 0 }, "options": { "tooltip": { @@ -321,7 +329,7 @@ ], "fieldConfig": { "defaults": { - "unit": "ops" + "unit": "suffix: messages/s" }, "overrides": [] } @@ -339,7 +347,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -358,10 +367,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: requests/s", "custom": { "axisLabel": "Requests / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -383,7 +392,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -404,10 +414,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: requests/s", "custom": { "axisLabel": "Requests / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json index c5d54e9e4a..4efec5de79 100644 --- a/docker/telemetry/grafana/dashboards/transaction-overview.json +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -21,7 +21,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -42,10 +43,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: transactions/s", "custom": { "axisLabel": "Transactions / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -67,7 +68,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -96,7 +98,7 @@ "unit": "ms", "custom": { "axisLabel": "Latency (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -126,10 +128,16 @@ "datasource": { "type": "prometheus" }, - "expr": "sum by (local, service_instance_id) (rate(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", local=~\"$tx_origin\", span_name=\"tx.process\", tx_type=~\"$tx_type\"}[5m]))", + "expr": "sum by (local, service_instance_id) (increase(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", local=~\"$tx_origin\", span_name=\"tx.process\", tx_type=~\"$tx_type\"}[5m]))", "legendFormat": "Local = {{local}} [{{service_instance_id}}]" } - ] + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } }, { "title": "Transaction Receive vs Suppressed", @@ -144,7 +152,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -158,10 +167,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: transactions/s", "custom": { "axisLabel": "Transactions / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -183,7 +192,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "yAxis": { "axisLabel": "Duration (ms)", @@ -202,7 +212,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ms" + "unit": "ms", + "custom": { + "spanNulls": 1800000 + } } } }, @@ -219,7 +232,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -243,7 +257,7 @@ "unit": "ms", "custom": { "axisLabel": "Latency (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -265,7 +279,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -282,7 +297,7 @@ "unit": "ops", "custom": { "axisLabel": "Transactions / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -318,7 +333,7 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: transactions/s", "thresholds": { "steps": [ { @@ -352,7 +367,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -371,10 +387,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: transactions/s", "custom": { - "axisLabel": "TX / Sec", - "spanNulls": true, + "axisLabel": "Transactions / Sec", + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -396,7 +412,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -415,10 +432,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: transactions/s", "custom": { - "axisLabel": "Failed TX / Sec", - "spanNulls": true, + "axisLabel": "Transactions / Sec", + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -448,7 +465,9 @@ "showLegend": true }, "tooltip": { - "mode": "single" + "mode": "multi", + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -477,6 +496,7 @@ ] }, "custom": { + "spanNulls": 1800000, "fillOpacity": 80, "lineWidth": 0 } @@ -497,7 +517,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -519,7 +540,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -541,7 +562,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -555,10 +577,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: enqueues/s", "custom": { "axisLabel": "Enqueues / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -580,7 +602,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -597,7 +620,7 @@ "unit": "percentunit", "custom": { "axisLabel": "Bypass Fraction", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -619,7 +642,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -643,7 +667,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -665,7 +689,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 } }, "targets": [ @@ -679,10 +704,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: cleanups/s", "custom": { "axisLabel": "Cleanups / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -704,7 +729,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -726,7 +752,7 @@ "unit": "ops", "custom": { "axisLabel": "Spans / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -748,7 +774,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -770,7 +797,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -792,7 +819,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -811,10 +839,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: failures/s", "custom": { - "axisLabel": "Failed Spans / Sec", - "spanNulls": true, + "axisLabel": "Failures / Sec", + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -836,7 +864,8 @@ "options": { "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "desc", + "maxHeight": 600 }, "legend": { "displayMode": "table", @@ -858,7 +887,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 2a0ff66091..ef4d3f2dec 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -667,7 +667,7 @@ Ten dashboards are pre-provisioned in `docker/telemetry/grafana/dashboards/`: | ---------------------------------- | -------------- | --------------------------------------------------------------------------------------------------- | ----------------------------------- | | Transaction Processing Rate | timeseries | `rate(traces_span_metrics_calls_total{span_name="tx.process"}[5m])` and `tx.receive` | `span_name` | | Transaction Processing Latency | timeseries | `histogram_quantile(0.95 / 0.50, ... {span_name="tx.process"})` | — | -| Transaction Path Distribution | piechart | `sum by (local) (rate(traces_span_metrics_calls_total{span_name="tx.process"}[5m]))` | `local` | +| Transaction Path Distribution | piechart | `sum by (local) (increase(traces_span_metrics_calls_total{span_name="tx.process"}[5m]))` | `local` | | Transaction Receive vs Suppressed | timeseries | `rate(traces_span_metrics_calls_total{span_name="tx.receive"}[5m])` | — | | TX Processing Duration Heatmap | heatmap | `tx.process` histogram buckets | `le` | | TX Apply Duration per Ledger | timeseries | p95/p50 of `tx.apply` | — | @@ -707,12 +707,12 @@ Ten dashboards are pre-provisioned in `docker/telemetry/grafana/dashboards/`: Requires `trace_peer=1` in the `[telemetry]` config section. -| Panel | Type | PromQL | Labels Used | -| -------------------------------- | ---------- | ------------------------------ | -------------------- | -| Proposal Receive Rate | timeseries | `peer.proposal.receive` rate | — | -| Validation Receive Rate | timeseries | `peer.validation.receive` rate | — | -| Proposals Trusted vs Untrusted | piechart | by `proposal_trusted` | `proposal_trusted` | -| Validations Trusted vs Untrusted | piechart | by `validation_trusted` | `validation_trusted` | +| Panel | Type | PromQL | Labels Used | +| -------------------------------- | ---------- | ------------------------------------------------------------------------- | -------------------- | +| Proposal Receive Rate | timeseries | `peer.proposal.receive` rate | — | +| Validation Receive Rate | timeseries | `peer.validation.receive` rate | — | +| Proposals Trusted vs Untrusted | piechart | `increase()` counts in the selected window, split by `proposal_trusted` | `proposal_trusted` | +| Validations Trusted vs Untrusted | piechart | `increase()` counts in the selected window, split by `validation_trusted` | `validation_trusted` | ### Node Health -- StatsD (`xrpld-statsd-node-health`)