From 370130a9f9adc2d842f76bdb6c57e54a4199974a Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:06:56 +0100 Subject: [PATCH] fix(telemetry): correct job latency units and name dashboard rate nouns The two key-job latency panels on node-health declared milliseconds while querying `job_running_us` / `job_queued_us`, which record microseconds. Every reading was a thousand times too large: the p95 for acceptLedger, 241495us, rendered as "241 sec" instead of 241ms. job-queue.json already read the same metrics as microseconds, so the two dashboards disagreed by 1000x on identical data. Also replace the generic `ops`, `cps` and `cpm` units, which Grafana renders as the literal "ops/s", "counts/s" and "counts/min", with custom-suffix units naming what each panel counts -- jobs, ledgers, validations, lookups. The per-minute panels already scale correctly in their queries; only the noun was missing. Normalise the micro sign to U+00B5 throughout. Four axis labels used the visually identical Greek mu, U+03BC, while every unit field used the micro sign. Ledger Close Rate plots ledgers closed beside ledger fetches, which one unit cannot describe; give the fetches series its own unit on a right-hand axis. Drop two field overrides on NodeStore Read Latency that restated the panel unit as a custom suffix -- a suffix is appended verbatim, so it would have suppressed magnitude scaling and left large values unreadable, the same defect fixed above. Queries are unchanged apart from the transaction path piechart, which moves from rate() to increase(): a per-slice "per second" reading is not a share of a total. Alongside, widen the Complete Ledger Ranges table to full width, hoist the stat panels above the fold, and bring the touched panels up to the tooltip and null-spanning guidelines. --- .../grafana/dashboards/consensus-health.json | 5 +- .../grafana/dashboards/fee-market.json | 8 +- .../grafana/dashboards/job-queue.json | 20 ++--- .../grafana/dashboards/ledger-data-sync.json | 39 ++++++--- .../grafana/dashboards/node-health.json | 79 ++++++++++--------- .../dashboards/overlay-traffic-detail.json | 12 ++- .../grafana/dashboards/rpc-performance.json | 12 +-- .../dashboards/transaction-overview.json | 4 +- .../grafana/dashboards/validator-health.json | 22 +++--- docs/telemetry-runbook.md | 52 ++++++------ 10 files changed, 140 insertions(+), 113 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json index 8ae07fe919..ab848401b1 100644 --- a/docker/telemetry/grafana/dashboards/consensus-health.json +++ b/docker/telemetry/grafana/dashboards/consensus-health.json @@ -230,6 +230,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -283,6 +284,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -376,6 +378,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -1006,7 +1009,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: mismatches/s", "custom": { "axisLabel": "Mismatches / Sec", "spanNulls": 1800000, diff --git a/docker/telemetry/grafana/dashboards/fee-market.json b/docker/telemetry/grafana/dashboards/fee-market.json index 7ab2fde491..b8ff971de7 100644 --- a/docker/telemetry/grafana/dashboards/fee-market.json +++ b/docker/telemetry/grafana/dashboards/fee-market.json @@ -332,9 +332,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: transactions/s", "custom": { - "axisLabel": "Expired / Sec", + "axisLabel": "Transactions / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -372,9 +372,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: transactions/s", "custom": { - "axisLabel": "Dropped / Sec", + "axisLabel": "Transactions / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", diff --git a/docker/telemetry/grafana/dashboards/job-queue.json b/docker/telemetry/grafana/dashboards/job-queue.json index fa84c49770..3c7d229316 100644 --- a/docker/telemetry/grafana/dashboards/job-queue.json +++ b/docker/telemetry/grafana/dashboards/job-queue.json @@ -110,12 +110,12 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: jobs/s", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, - "axisLabel": "Operations / Sec", + "axisLabel": "Jobs / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -162,12 +162,12 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: jobs/s", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 5, - "axisLabel": "Operations / Sec", + "axisLabel": "Jobs / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -214,12 +214,12 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: jobs/s", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 5, - "axisLabel": "Operations / Sec", + "axisLabel": "Jobs / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -272,7 +272,7 @@ "drawStyle": "line", "lineWidth": 2, "fillOpacity": 5, - "axisLabel": "Duration (μs)", + "axisLabel": "Duration (µs)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -325,7 +325,7 @@ "drawStyle": "line", "lineWidth": 2, "fillOpacity": 5, - "axisLabel": "Duration (μs)", + "axisLabel": "Duration (µs)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -377,7 +377,7 @@ "drawStyle": "line", "lineWidth": 1, "fillOpacity": 5, - "axisLabel": "Duration (μs)", + "axisLabel": "Duration (µs)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -419,7 +419,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cpm", + "unit": "suffix: overflows/min", "thresholds": { "steps": [ { diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index 51f4ec9f40..f598ba5808 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -1117,16 +1117,33 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "reqps", + "unit": "suffix: ledgers/s", "custom": { - "axisLabel": "Rate (per second)", + "axisLabel": "Ledgers / Sec & Fetches / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^Ledger Fetches/" + }, + "properties": [ + { + "id": "unit", + "value": "suffix: fetches/s" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] }, "id": 19 }, @@ -1826,9 +1843,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "ops", + "unit": "suffix: events/s", "custom": { - "axisLabel": "Events (Per Second)", + "axisLabel": "Events / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -1882,9 +1899,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "ops", + "unit": "suffix: outcomes/s", "custom": { - "axisLabel": "Outcomes (Per Second)", + "axisLabel": "Outcomes / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -1931,9 +1948,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "ops", + "unit": "suffix: acquisitions/s", "custom": { - "axisLabel": "Discarded Acquisitions (Per Second)", + "axisLabel": "Acquisitions / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -1980,9 +1997,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "ops", + "unit": "suffix: events/s", "custom": { - "axisLabel": "Ledger Lane Events (Per Second)", + "axisLabel": "Events / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index 0350b6d4a1..a0bb801fe0 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -526,7 +526,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: operations/s", "custom": { "axisLabel": "Operations / Sec", "drawStyle": "line", @@ -1196,7 +1196,7 @@ "type": "timeseries", "gridPos": { "x": 0, - "y": 88, + "y": 96, "w": 12, "h": 8 }, @@ -1240,7 +1240,7 @@ "type": "timeseries", "gridPos": { "x": 12, - "y": 88, + "y": 96, "w": 12, "h": 8 }, @@ -1285,8 +1285,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 96 + "x": 0, + "y": 88 }, "options": { "tooltip": { @@ -1319,8 +1319,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 104 + "x": 12, + "y": 88 }, "options": { "tooltip": { @@ -1353,7 +1353,7 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, + "x": 0, "y": 104 }, "options": { @@ -1407,9 +1407,9 @@ "type": "table", "gridPos": { "h": 8, - "w": 12, + "w": 24, "x": 0, - "y": 113 + "y": 121 }, "options": { "showHeader": true @@ -1440,8 +1440,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 113 + "x": 0, + "y": 129 }, "options": { "tooltip": { @@ -1494,9 +1494,9 @@ "type": "stat", "gridPos": { "h": 8, - "w": 12, + "w": 24, "x": 0, - "y": 121 + "y": 113 }, "options": { "tooltip": { @@ -1516,7 +1516,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cpm", + "unit": "suffix: fetches/min", "custom": {} }, "overrides": [] @@ -1530,7 +1530,7 @@ "h": 8, "w": 12, "x": 12, - "y": 121 + "y": 129 }, "options": { "tooltip": { @@ -1571,7 +1571,7 @@ "h": 1, "w": 24, "x": 0, - "y": 129 + "y": 137 }, "collapsed": false, "panels": [] @@ -1584,7 +1584,7 @@ "h": 8, "w": 6, "x": 0, - "y": 130 + "y": 138 }, "options": { "tooltip": { @@ -1618,7 +1618,7 @@ "h": 8, "w": 6, "x": 6, - "y": 130 + "y": 138 }, "options": { "tooltip": { @@ -1652,7 +1652,7 @@ "h": 8, "w": 6, "x": 12, - "y": 130 + "y": 138 }, "options": { "tooltip": { @@ -1686,7 +1686,7 @@ "h": 8, "w": 6, "x": 18, - "y": 130 + "y": 138 }, "options": { "tooltip": { @@ -1748,7 +1748,7 @@ "h": 8, "w": 24, "x": 0, - "y": 138 + "y": 146 }, "options": { "tooltip": { @@ -1768,9 +1768,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: transactions/s", "custom": { - "axisLabel": "Transactions", + "axisLabel": "Transactions / Sec", "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, @@ -1886,9 +1886,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ms", + "unit": "µs", "custom": { - "axisLabel": "Duration (ms)", + "axisLabel": "Duration (µs)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -1986,9 +1986,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ms", + "unit": "µs", "custom": { - "axisLabel": "Wait Time (ms)", + "axisLabel": "Wait Time (µs)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -2290,7 +2290,7 @@ "h": 8, "w": 12, "x": 0, - "y": 130 + "y": 138 }, "options": { "tooltip": { @@ -2330,7 +2330,7 @@ "h": 8, "w": 12, "x": 12, - "y": 130 + "y": 138 }, "options": { "tooltip": { @@ -2350,7 +2350,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: acquisitions/s", "custom": { "axisLabel": "Acquisitions / Sec", "spanNulls": 1800000, @@ -2370,7 +2370,7 @@ "h": 8, "w": 18, "x": 0, - "y": 138 + "y": 146 }, "options": { "mergeValues": true, @@ -2378,8 +2378,9 @@ "alignValue": "center", "rowHeight": 0.9, "tooltip": { - "mode": "single", - "sort": "none" + "maxHeight": 600, + "mode": "multi", + "sort": "desc" } }, "targets": [ @@ -2444,7 +2445,9 @@ ], "custom": { "fillOpacity": 80, - "lineWidth": 0 + "lineWidth": 0, + "spanNulls": 1800000, + "insertNulls": false } }, "overrides": [] @@ -2458,7 +2461,7 @@ "h": 8, "w": 6, "x": 18, - "y": 138 + "y": 146 }, "options": { "reduceOptions": { @@ -2519,7 +2522,7 @@ "h": 1, "w": 24, "x": 0, - "y": 146 + "y": 154 }, "collapsed": false, "panels": [] @@ -2532,7 +2535,7 @@ "h": 8, "w": 24, "x": 0, - "y": 147 + "y": 155 }, "options": { "tooltip": { diff --git a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json index f7bf0823c6..adc685ce6f 100644 --- a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json +++ b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json @@ -590,7 +590,7 @@ "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", "unit": "µs", "custom": { - "axisLabel": "Duration (μs)", + "axisLabel": "Duration (µs)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -639,7 +639,11 @@ ], "fieldConfig": { "defaults": { - "unit": "short" + "unit": "short", + "custom": { + "spanNulls": 1800000, + "insertNulls": false + } }, "overrides": [] } @@ -672,7 +676,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: lookups/s", "custom": { "axisLabel": "Lookups / Sec", "spanNulls": 1800000, @@ -712,7 +716,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: rejections/s", "custom": { "axisLabel": "Rejections / Sec", "spanNulls": 1800000, diff --git a/docker/telemetry/grafana/dashboards/rpc-performance.json b/docker/telemetry/grafana/dashboards/rpc-performance.json index 5d436d2427..230dcbe0fb 100644 --- a/docker/telemetry/grafana/dashboards/rpc-performance.json +++ b/docker/telemetry/grafana/dashboards/rpc-performance.json @@ -498,12 +498,12 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: calls/s", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, - "axisLabel": "Operations / Sec", + "axisLabel": "Calls / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -550,12 +550,12 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: calls/s", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 5, - "axisLabel": "Operations / Sec", + "axisLabel": "Calls / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -602,12 +602,12 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "ops", + "unit": "suffix: errors/s", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 5, - "axisLabel": "Operations / Sec", + "axisLabel": "Errors / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json index 2a4fc54292..c14b31013d 100644 --- a/docker/telemetry/grafana/dashboards/transaction-overview.json +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -297,7 +297,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps" + "unit": "short" }, "overrides": [] }, @@ -313,7 +313,7 @@ "datasource": { "type": "prometheus" }, - "expr": "label_replace(sum by (local, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", local=~\"$tx_origin\", span_name=\"tx.process\", tx_type=~\"$tx_type\"}[$__rate_interval])), \"series\", \"Local $1\", \"local\", \"(.*)\")" + "expr": "label_replace(sum by (local, service_instance_id, xrpl_branch, xrpl_node_role) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", local=~\"$tx_origin\", span_name=\"tx.process\", tx_type=~\"$tx_type\"}[$__rate_interval])), \"series\", \"Local $1\", \"local\", \"(.*)\")" } ], "id": 7 diff --git a/docker/telemetry/grafana/dashboards/validator-health.json b/docker/telemetry/grafana/dashboards/validator-health.json index 7c53c2d593..e1f1abf1f4 100644 --- a/docker/telemetry/grafana/dashboards/validator-health.json +++ b/docker/telemetry/grafana/dashboards/validator-health.json @@ -297,7 +297,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cpm", + "unit": "suffix: validations/min", "thresholds": { "steps": [ { @@ -347,7 +347,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cpm", + "unit": "suffix: validations/min", "custom": {} }, "overrides": [] @@ -569,9 +569,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cpm", + "unit": "suffix: validations/min", "custom": { - "axisLabel": "Per Minute", + "axisLabel": "Validations / Min", "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, @@ -657,7 +657,7 @@ "h": 8, "w": 24, "x": 0, - "y": 91 + "y": 123 }, "options": { "tooltip": { @@ -703,7 +703,7 @@ "h": 8, "w": 24, "x": 0, - "y": 99 + "y": 91 }, "options": { "tooltip": { @@ -737,7 +737,7 @@ "h": 8, "w": 24, "x": 0, - "y": 107 + "y": 99 }, "options": { "tooltip": { @@ -757,7 +757,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "suffix:/hr", + "unit": "suffix: changes/hr", "thresholds": { "steps": [ { @@ -787,7 +787,7 @@ "h": 8, "w": 24, "x": 0, - "y": 115 + "y": 107 }, "options": { "tooltip": { @@ -807,7 +807,7 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cpm", + "unit": "suffix: ledgers/min", "thresholds": { "steps": [ { @@ -837,7 +837,7 @@ "h": 8, "w": 24, "x": 0, - "y": 123 + "y": 115 }, "options": { "tooltip": { diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index aaee9385fe..b41fbffa7b 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -1095,32 +1095,32 @@ Requires `trace_peer=1` in the `[telemetry]` config section. ### Node Health -- System Metrics (`node-health`) -| Panel | Type | PromQL | Labels Used | -| -------------------------------------- | ---------- | ---------------------------------------------------------- | ---------------- | -| Validated Ledger Age | stat | `ledgermaster_validated_ledger_age` | — | -| Published Ledger Age | stat | `ledgermaster_published_ledger_age` | — | -| Operating Mode (Time Share) | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))` | — | -| Operating Mode Transitions | timeseries | `state_accounting_*_transitions` | — | -| I/O Latency | timeseries | `histogram_quantile(0.95, ios_latency_bucket)` | — | -| Job Queue Depth | timeseries | `job_count` | — | -| Ledger Fetch Rate | stat | `rate(ledger_fetches[5m])` | — | -| Ledger History Mismatches | stat | `rate(ledger_history_mismatch[5m])` | — | -| Key Jobs Execution Time | timeseries | `acceptledger{quantile="$quantile"}` (+ 10 more key jobs) | `quantile` | -| Key Jobs Dequeue Wait Time | timeseries | `acceptledger_q{quantile="$quantile"}` (+ 10 more) | `quantile` | -| FullBelowCache Size | timeseries | `node_family_full_below_cache_size` | — | -| FullBelowCache Hit Rate | gauge | `node_family_full_below_cache_hit_rate` | — | -| Ledger Publish Gap | stat | `Published_Ledger_Age - Validated_Ledger_Age` | — | -| State Duration Rate (Full vs Tracking) | timeseries | `rate(state_accounting_full_duration[5m]) / 1000000` | — | -| All Jobs Execution Time (Detail) | timeseries | `{__name__=~"", quantile="$quantile"}` | `quantile` | -| All Jobs Dequeue Wait (Detail) | timeseries | `{__name__=~"_q", quantile="$quantile"}` | `quantile` | -| Server State | stat | `server_info{metric="server_state"}` | `metric` | -| Uptime | stat | `server_info{metric="uptime"}` | `metric` | -| Peer Count | stat | `server_info{metric="peers"}` | `metric` | -| Validated Ledger Seq | stat | `server_info{metric="validated_ledger_seq"}` | `metric` | -| Build Version | stat | `build_info` | `version` | -| Complete Ledger Ranges | table | `complete_ledgers` | `bound`, `index` | -| Database Sizes | timeseries | `db_metrics{metric=~"db_kb_.*"}` | `metric` | -| Historical Fetch Rate | stat | `db_metrics{metric="historical_perminute"}` | `metric` | +| Panel | Type | PromQL | Labels Used | +| -------------------------------- | ---------- | ---------------------------------------------------------- | ---------------- | +| Validated Ledger Age | stat | `ledgermaster_validated_ledger_age` | — | +| Published Ledger Age | stat | `ledgermaster_published_ledger_age` | — | +| Operating Mode (Time Share) | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))` | — | +| Operating Mode Transitions | timeseries | `state_accounting_*_transitions` | — | +| I/O Latency | timeseries | `histogram_quantile(0.95, ios_latency_bucket)` | — | +| Job Queue Depth | timeseries | `job_count` | — | +| Ledger Fetch Rate | stat | `rate(ledger_fetches[5m])` | — | +| Ledger History Mismatches | stat | `rate(ledger_history_mismatch[5m])` | — | +| Key Jobs Execution Time | timeseries | `acceptledger{quantile="$quantile"}` (+ 10 more key jobs) | `quantile` | +| Key Jobs Dequeue Wait Time | timeseries | `acceptledger_q{quantile="$quantile"}` (+ 10 more) | `quantile` | +| FullBelowCache Size | timeseries | `node_family_full_below_cache_size` | — | +| FullBelowCache Hit Rate | gauge | `node_family_full_below_cache_hit_rate` | — | +| Ledger Publish Gap | stat | `Published_Ledger_Age - Validated_Ledger_Age` | — | +| State Duration Rate (All States) | timeseries | `rate(state_accounting__duration[5m]) / 1000000` | — | +| All Jobs Execution Time (Detail) | timeseries | `{__name__=~"", quantile="$quantile"}` | `quantile` | +| All Jobs Dequeue Wait (Detail) | timeseries | `{__name__=~"_q", quantile="$quantile"}` | `quantile` | +| Server State | stat | `server_info{metric="server_state"}` | `metric` | +| Uptime | stat | `server_info{metric="uptime"}` | `metric` | +| Peer Count | stat | `server_info{metric="peers"}` | `metric` | +| Validated Ledger Seq | stat | `server_info{metric="validated_ledger_seq"}` | `metric` | +| Build Version | stat | `build_info` | `version` | +| Complete Ledger Ranges | table | `complete_ledgers` | `bound`, `index` | +| Database Sizes | timeseries | `db_metrics{metric=~"db_kb_.*"}` | `metric` | +| Historical Fetch Rate | stat | `db_metrics{metric="historical_perminute"}` | `metric` | ### Network Traffic -- System Metrics (`network-traffic`)