From d06faec407b372cdc2747515618093e26827065e Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Mon, 13 Jul 2026 20:29:52 +0100 Subject: [PATCH] fix(telemetry): split mixed-unit node-health panels into single-unit panels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three panels mixed incompatible units on one Y-axis (count vs ms, ops vs percent), causing the small-magnitude series to flatline to zero. - Last Close Info -> Last Close — Proposers (short) + Converge Time (ms) - NodeStore Read/Write Totals -> Throughput (ops) + Cache Hit Ratio (%) - NodeStore Read Threads & Duration -> Read Threads (short) + Busy Ratio (%) Each new panel has a single unit; per-series unit overrides removed. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../grafana/dashboards/node-health.json | 305 +++++++++++------- 1 file changed, 183 insertions(+), 122 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index 37936c7e3d..966f3cd1c3 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -412,14 +412,14 @@ "panels": [] }, { - "title": "NodeStore Read/Write Totals", + "title": "NodeStore Throughput", "description": "###### What this is:\n*Cumulative object-store read, cache-hit, and write operation counts.*\n\n###### How it's computed:\n*Current values of the read, cache-hit, and write counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope reflects store activity.*\n\n###### Healthy range:\n*Reads and writes rising smoothly, with cache-hits tracking a good fraction of reads.*\n\n###### Watch for:\n*A sudden surge in reads or writes signals heavy back-end I/O, from sync, replay, or query load.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`", "type": "timeseries", "gridPos": { - "h": 8, - "w": 12, "x": 0, - "y": 33 + "y": 33, + "w": 12, + "h": 8 }, "options": { "tooltip": { @@ -433,29 +433,24 @@ "type": "prometheus" }, "expr": "rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])", - "legendFormat": "Reads Total [{{service_instance_id}}]" + "legendFormat": "Reads Total [{{service_instance_id}}]", + "refId": "A" }, { "datasource": { "type": "prometheus" }, "expr": "rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval])", - "legendFormat": "Reads Hit (cache) [{{service_instance_id}}]" + "legendFormat": "Reads Hit (cache) [{{service_instance_id}}]", + "refId": "B" }, { "datasource": { "type": "prometheus" }, "expr": "rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_writes\"}[$__rate_interval])", - "legendFormat": "Writes Total [{{service_instance_id}}]" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval]) / rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])", - "legendFormat": "Cache Hit Ratio [{{service_instance_id}}]" + "legendFormat": "Writes Total [{{service_instance_id}}]", + "refId": "C" } ], "fieldConfig": { @@ -471,24 +466,50 @@ "mode": "palette-classic" } }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/Cache Hit Ratio/" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "custom.axisPlacement", - "value": "right" - } - ] + "overrides": [] + } + }, + { + "title": "NodeStore Cache Hit Ratio", + "description": "###### What this is:\n*Cumulative object-store read, cache-hit, and write operation counts.*\n\n###### How it's computed:\n*Current values of the read, cache-hit, and write counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope reflects store activity.*\n\n###### Healthy range:\n*Reads and writes rising smoothly, with cache-hits tracking a good fraction of reads.*\n\n###### Watch for:\n*A sudden surge in reads or writes signals heavy back-end I/O, from sync, replay, or query load.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`", + "type": "timeseries", + "gridPos": { + "x": 12, + "y": 33, + "w": 12, + "h": 8 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval]) / rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])", + "legendFormat": "Cache Hit Ratio [{{service_instance_id}}]", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "axisLabel": "Operations / Sec", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10 + }, + "color": { + "mode": "palette-classic" } - ] + }, + "overrides": [] } }, { @@ -496,10 +517,10 @@ "description": "###### What this is:\n*Instantaneous write-load score and read-queue depth of the object store.*\n\n###### How it's computed:\n*Current values of the write-load and read-queue gauges, plotted over time.*\n\n###### Reading it:\n*Lower is better for both; short, flat lines are healthy.*\n\n###### Healthy range:\n*Write load near zero and read queue in low double digits or less.*\n\n###### Watch for:\n*High write load means back-end pressure; a high read queue means the prefetch threads are saturated.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`", "type": "timeseries", "gridPos": { - "h": 8, + "x": 0, + "y": 41, "w": 12, - "x": 12, - "y": 33 + "h": 8 }, "options": { "tooltip": { @@ -560,10 +581,10 @@ "description": "###### What this is:\n*Cumulative bytes read from and written to the object-store back end.*\n\n###### How it's computed:\n*Current values of the bytes-read and bytes-written counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope shows throughput.*\n\n###### Healthy range:\n*Smooth growth consistent with ledger and query activity.*\n\n###### Watch for:\n*A sharp acceleration indicates a heavy I/O phase such as sync, replay, or large queries.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`", "type": "timeseries", "gridPos": { - "h": 8, + "x": 12, + "y": 41, "w": 12, - "x": 0, - "y": 41 + "h": 8 }, "options": { "tooltip": { @@ -604,14 +625,14 @@ } }, { - "title": "NodeStore Read Threads & Duration", + "title": "NodeStore Read Threads", "description": "###### What this is:\n*Read-thread utilization, bundled read count, and cumulative read time for the object store.*\n\n###### How it's computed:\n*Current values of the running/total read-thread gauges, read-bundle gauge, and cumulative read-duration counter, plotted as lines.*\n\n###### Reading it:\n*Running threads well below total means spare capacity; a rising duration line reflects time spent in read I/O.*\n\n###### Healthy range:\n*Running threads below the total count most of the time.*\n\n###### Watch for:\n*Running threads pinned at the total for long periods means read I/O is saturated.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`", "type": "timeseries", "gridPos": { - "h": 8, + "x": 0, + "y": 49, "w": 12, - "x": 12, - "y": 41 + "h": 8 }, "options": { "tooltip": { @@ -625,28 +646,24 @@ "type": "prometheus" }, "expr": "nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_threads_running\"}", - "legendFormat": "Read Threads Running [{{service_instance_id}}]" + "legendFormat": "Read Threads Running [{{service_instance_id}}]", + "refId": "A" }, { "datasource": { "type": "prometheus" }, "expr": "nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_threads_total\"}", - "legendFormat": "Read Threads Total [{{service_instance_id}}]" + "legendFormat": "Read Threads Total [{{service_instance_id}}]", + "refId": "B" }, { "datasource": { "type": "prometheus" }, "expr": "nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_request_bundle\"}", - "legendFormat": "Read Request Bundle [{{service_instance_id}}]" - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_duration_us\"}[$__rate_interval]) / 1e6", - "legendFormat": "Read Busy Ratio [{{service_instance_id}}]" + "legendFormat": "Read Request Bundle [{{service_instance_id}}]", + "refId": "C" } ], "fieldConfig": { @@ -662,24 +679,49 @@ "mode": "palette-classic" } }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/Read Busy Ratio/" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "custom.axisPlacement", - "value": "right" - } - ] + "overrides": [] + } + }, + { + "title": "NodeStore Read Busy Ratio", + "description": "###### What this is:\n*Read-thread utilization, bundled read count, and cumulative read time for the object store.*\n\n###### How it's computed:\n*Current values of the running/total read-thread gauges, read-bundle gauge, and cumulative read-duration counter, plotted as lines.*\n\n###### Reading it:\n*Running threads well below total means spare capacity; a rising duration line reflects time spent in read I/O.*\n\n###### Healthy range:\n*Running threads below the total count most of the time.*\n\n###### Watch for:\n*Running threads pinned at the total for long periods means read I/O is saturated.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`", + "type": "timeseries", + "gridPos": { + "x": 12, + "y": 49, + "w": 12, + "h": 8 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_duration_us\"}[$__rate_interval]) / 1e6", + "legendFormat": "Read Busy Ratio [{{service_instance_id}}]", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "axisLabel": "Threads / Bundle", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10 + }, + "color": { + "mode": "palette-classic" } - ] + }, + "overrides": [] } }, { @@ -689,7 +731,7 @@ "h": 1, "w": 24, "x": 0, - "y": 49 + "y": 57 }, "collapsed": false, "panels": [] @@ -702,7 +744,7 @@ "h": 8, "w": 12, "x": 0, - "y": 50 + "y": 58 }, "options": { "tooltip": { @@ -763,7 +805,7 @@ "h": 8, "w": 12, "x": 12, - "y": 50 + "y": 58 }, "options": { "tooltip": { @@ -824,7 +866,7 @@ "h": 1, "w": 24, "x": 0, - "y": 58 + "y": 66 }, "collapsed": false, "panels": [] @@ -837,7 +879,7 @@ "h": 8, "w": 24, "x": 0, - "y": 59 + "y": 67 }, "options": { "tooltip": { @@ -882,7 +924,7 @@ "h": 1, "w": 24, "x": 0, - "y": 67 + "y": 75 }, "collapsed": false, "panels": [] @@ -895,7 +937,7 @@ "h": 8, "w": 6, "x": 0, - "y": 68 + "y": 76 }, "options": { "tooltip": { @@ -975,7 +1017,7 @@ "h": 8, "w": 6, "x": 6, - "y": 68 + "y": 76 }, "options": { "tooltip": { @@ -1008,7 +1050,7 @@ "h": 8, "w": 6, "x": 12, - "y": 68 + "y": 76 }, "options": { "tooltip": { @@ -1041,7 +1083,7 @@ "h": 8, "w": 6, "x": 18, - "y": 68 + "y": 76 }, "options": { "tooltip": { @@ -1067,14 +1109,14 @@ } }, { - "title": "Last Close Info", + "title": "Last Close \u2014 Proposers", "description": "###### What this is:\n*Proposer count and convergence time from the last closed consensus round.*\n\n###### How it's computed:\n*Current values of the last-close proposer-count and convergence-time gauges, plotted as lines.*\n\n###### Reading it:\n*A healthy proposer count with a low convergence time is good.*\n\n###### Healthy range:\n*Convergence time of a few seconds with the expected number of proposers.*\n\n###### Watch for:\n*A falling proposer count or rising convergence time signals degrading consensus conditions.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`", "type": "timeseries", "gridPos": { - "h": 8, - "w": 12, "x": 0, - "y": 76 + "y": 84, + "w": 12, + "h": 8 }, "options": { "tooltip": { @@ -1088,14 +1130,8 @@ "type": "prometheus" }, "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"last_close_proposers\"}", - "legendFormat": "Proposers [{{service_instance_id}}]" - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"last_close_converge_time_ms\"}", - "legendFormat": "Converge Time ms [{{service_instance_id}}]" + "legendFormat": "Proposers [{{service_instance_id}}]", + "refId": "A" } ], "fieldConfig": { @@ -1111,24 +1147,49 @@ "mode": "palette-classic" } }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/Converge Time ms/" - }, - "properties": [ - { - "id": "unit", - "value": "ms" - }, - { - "id": "custom.axisPlacement", - "value": "right" - } - ] + "overrides": [] + } + }, + { + "title": "Last Close \u2014 Converge Time", + "description": "###### What this is:\n*Proposer count and convergence time from the last closed consensus round.*\n\n###### How it's computed:\n*Current values of the last-close proposer-count and convergence-time gauges, plotted as lines.*\n\n###### Reading it:\n*A healthy proposer count with a low convergence time is good.*\n\n###### Healthy range:\n*Convergence time of a few seconds with the expected number of proposers.*\n\n###### Watch for:\n*A falling proposer count or rising convergence time signals degrading consensus conditions.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`", + "type": "timeseries", + "gridPos": { + "x": 12, + "y": 84, + "w": 12, + "h": 8 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"last_close_converge_time_ms\"}", + "legendFormat": "Converge Time ms [{{service_instance_id}}]", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "axisLabel": "Proposers", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10 + }, + "color": { + "mode": "palette-classic" } - ] + }, + "overrides": [] } }, { @@ -1139,7 +1200,7 @@ "h": 8, "w": 12, "x": 12, - "y": 76 + "y": 92 }, "options": { "tooltip": { @@ -1173,7 +1234,7 @@ "h": 8, "w": 12, "x": 0, - "y": 84 + "y": 100 }, "options": { "tooltip": { @@ -1206,7 +1267,7 @@ "h": 8, "w": 12, "x": 12, - "y": 84 + "y": 100 }, "options": { "tooltip": { @@ -1246,7 +1307,7 @@ "h": 1, "w": 24, "x": 0, - "y": 92 + "y": 108 }, "collapsed": false, "panels": [] @@ -1259,7 +1320,7 @@ "h": 8, "w": 12, "x": 0, - "y": 93 + "y": 109 }, "options": { "showHeader": true @@ -1291,7 +1352,7 @@ "h": 8, "w": 12, "x": 12, - "y": 93 + "y": 109 }, "options": { "tooltip": { @@ -1346,7 +1407,7 @@ "h": 8, "w": 12, "x": 0, - "y": 101 + "y": 117 }, "options": { "tooltip": { @@ -1379,7 +1440,7 @@ "h": 8, "w": 12, "x": 12, - "y": 101 + "y": 117 }, "options": { "tooltip": { @@ -1418,7 +1479,7 @@ "h": 1, "w": 24, "x": 0, - "y": 109 + "y": 125 }, "collapsed": false, "panels": [] @@ -1431,7 +1492,7 @@ "h": 8, "w": 6, "x": 0, - "y": 110 + "y": 126 }, "options": { "tooltip": { @@ -1464,7 +1525,7 @@ "h": 8, "w": 6, "x": 6, - "y": 110 + "y": 126 }, "options": { "tooltip": { @@ -1497,7 +1558,7 @@ "h": 8, "w": 6, "x": 12, - "y": 110 + "y": 126 }, "options": { "tooltip": { @@ -1530,7 +1591,7 @@ "h": 8, "w": 6, "x": 18, - "y": 110 + "y": 126 }, "options": { "tooltip": { @@ -1591,7 +1652,7 @@ "h": 8, "w": 24, "x": 0, - "y": 118 + "y": 134 }, "options": { "tooltip": {