fix(telemetry): split mixed-unit node-health panels into single-unit panels

Three panels mixed incompatible units on one Y-axis (count vs ms, ops vs
percent), causing the small-magnitude series to flatline to zero.

- Last Close Info -> Last Close — Proposers (short) + Converge Time (ms)
- NodeStore Read/Write Totals -> Throughput (ops) + Cache Hit Ratio (%)
- NodeStore Read Threads & Duration -> Read Threads (short) + Busy Ratio (%)

Each new panel has a single unit; per-series unit overrides removed.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Pratik Mankawde
2026-07-13 20:29:52 +01:00
parent 7f71ffe06e
commit d06faec407

View File

@@ -412,14 +412,14 @@
"panels": []
},
{
"title": "NodeStore Read/Write Totals",
"title": "NodeStore Throughput",
"description": "###### What this is:\n*Cumulative object-store read, cache-hit, and write operation counts.*\n\n###### How it's computed:\n*Current values of the read, cache-hit, and write counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope reflects store activity.*\n\n###### Healthy range:\n*Reads and writes rising smoothly, with cache-hits tracking a good fraction of reads.*\n\n###### Watch for:\n*A sudden surge in reads or writes signals heavy back-end I/O, from sync, replay, or query load.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 33
"y": 33,
"w": 12,
"h": 8
},
"options": {
"tooltip": {
@@ -433,29 +433,24 @@
"type": "prometheus"
},
"expr": "rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])",
"legendFormat": "Reads Total [{{service_instance_id}}]"
"legendFormat": "Reads Total [{{service_instance_id}}]",
"refId": "A"
},
{
"datasource": {
"type": "prometheus"
},
"expr": "rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval])",
"legendFormat": "Reads Hit (cache) [{{service_instance_id}}]"
"legendFormat": "Reads Hit (cache) [{{service_instance_id}}]",
"refId": "B"
},
{
"datasource": {
"type": "prometheus"
},
"expr": "rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_writes\"}[$__rate_interval])",
"legendFormat": "Writes Total [{{service_instance_id}}]"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval]) / rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])",
"legendFormat": "Cache Hit Ratio [{{service_instance_id}}]"
"legendFormat": "Writes Total [{{service_instance_id}}]",
"refId": "C"
}
],
"fieldConfig": {
@@ -471,24 +466,50 @@
"mode": "palette-classic"
}
},
"overrides": [
{
"matcher": {
"id": "byRegexp",
"options": "/Cache Hit Ratio/"
},
"properties": [
{
"id": "unit",
"value": "percentunit"
},
{
"id": "custom.axisPlacement",
"value": "right"
}
]
"overrides": []
}
},
{
"title": "NodeStore Cache Hit Ratio",
"description": "###### What this is:\n*Cumulative object-store read, cache-hit, and write operation counts.*\n\n###### How it's computed:\n*Current values of the read, cache-hit, and write counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope reflects store activity.*\n\n###### Healthy range:\n*Reads and writes rising smoothly, with cache-hits tracking a good fraction of reads.*\n\n###### Watch for:\n*A sudden surge in reads or writes signals heavy back-end I/O, from sync, replay, or query load.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`",
"type": "timeseries",
"gridPos": {
"x": 12,
"y": 33,
"w": 12,
"h": 8
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval]) / rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])",
"legendFormat": "Cache Hit Ratio [{{service_instance_id}}]",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"custom": {
"axisLabel": "Operations / Sec",
"drawStyle": "line",
"lineWidth": 2,
"fillOpacity": 10
},
"color": {
"mode": "palette-classic"
}
]
},
"overrides": []
}
},
{
@@ -496,10 +517,10 @@
"description": "###### What this is:\n*Instantaneous write-load score and read-queue depth of the object store.*\n\n###### How it's computed:\n*Current values of the write-load and read-queue gauges, plotted over time.*\n\n###### Reading it:\n*Lower is better for both; short, flat lines are healthy.*\n\n###### Healthy range:\n*Write load near zero and read queue in low double digits or less.*\n\n###### Watch for:\n*High write load means back-end pressure; a high read queue means the prefetch threads are saturated.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`",
"type": "timeseries",
"gridPos": {
"h": 8,
"x": 0,
"y": 41,
"w": 12,
"x": 12,
"y": 33
"h": 8
},
"options": {
"tooltip": {
@@ -560,10 +581,10 @@
"description": "###### What this is:\n*Cumulative bytes read from and written to the object-store back end.*\n\n###### How it's computed:\n*Current values of the bytes-read and bytes-written counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope shows throughput.*\n\n###### Healthy range:\n*Smooth growth consistent with ledger and query activity.*\n\n###### Watch for:\n*A sharp acceleration indicates a heavy I/O phase such as sync, replay, or large queries.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`",
"type": "timeseries",
"gridPos": {
"h": 8,
"x": 12,
"y": 41,
"w": 12,
"x": 0,
"y": 41
"h": 8
},
"options": {
"tooltip": {
@@ -604,14 +625,14 @@
}
},
{
"title": "NodeStore Read Threads & Duration",
"title": "NodeStore Read Threads",
"description": "###### What this is:\n*Read-thread utilization, bundled read count, and cumulative read time for the object store.*\n\n###### How it's computed:\n*Current values of the running/total read-thread gauges, read-bundle gauge, and cumulative read-duration counter, plotted as lines.*\n\n###### Reading it:\n*Running threads well below total means spare capacity; a rising duration line reflects time spent in read I/O.*\n\n###### Healthy range:\n*Running threads below the total count most of the time.*\n\n###### Watch for:\n*Running threads pinned at the total for long periods means read I/O is saturated.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`",
"type": "timeseries",
"gridPos": {
"h": 8,
"x": 0,
"y": 49,
"w": 12,
"x": 12,
"y": 41
"h": 8
},
"options": {
"tooltip": {
@@ -625,28 +646,24 @@
"type": "prometheus"
},
"expr": "nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_threads_running\"}",
"legendFormat": "Read Threads Running [{{service_instance_id}}]"
"legendFormat": "Read Threads Running [{{service_instance_id}}]",
"refId": "A"
},
{
"datasource": {
"type": "prometheus"
},
"expr": "nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_threads_total\"}",
"legendFormat": "Read Threads Total [{{service_instance_id}}]"
"legendFormat": "Read Threads Total [{{service_instance_id}}]",
"refId": "B"
},
{
"datasource": {
"type": "prometheus"
},
"expr": "nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_request_bundle\"}",
"legendFormat": "Read Request Bundle [{{service_instance_id}}]"
},
{
"datasource": {
"type": "prometheus"
},
"expr": "rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_duration_us\"}[$__rate_interval]) / 1e6",
"legendFormat": "Read Busy Ratio [{{service_instance_id}}]"
"legendFormat": "Read Request Bundle [{{service_instance_id}}]",
"refId": "C"
}
],
"fieldConfig": {
@@ -662,24 +679,49 @@
"mode": "palette-classic"
}
},
"overrides": [
{
"matcher": {
"id": "byRegexp",
"options": "/Read Busy Ratio/"
},
"properties": [
{
"id": "unit",
"value": "percentunit"
},
{
"id": "custom.axisPlacement",
"value": "right"
}
]
"overrides": []
}
},
{
"title": "NodeStore Read Busy Ratio",
"description": "###### What this is:\n*Read-thread utilization, bundled read count, and cumulative read time for the object store.*\n\n###### How it's computed:\n*Current values of the running/total read-thread gauges, read-bundle gauge, and cumulative read-duration counter, plotted as lines.*\n\n###### Reading it:\n*Running threads well below total means spare capacity; a rising duration line reflects time spent in read I/O.*\n\n###### Healthy range:\n*Running threads below the total count most of the time.*\n\n###### Watch for:\n*Running threads pinned at the total for long periods means read I/O is saturated.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`",
"type": "timeseries",
"gridPos": {
"x": 12,
"y": 49,
"w": 12,
"h": 8
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus"
},
"expr": "rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_duration_us\"}[$__rate_interval]) / 1e6",
"legendFormat": "Read Busy Ratio [{{service_instance_id}}]",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"custom": {
"axisLabel": "Threads / Bundle",
"drawStyle": "line",
"lineWidth": 2,
"fillOpacity": 10
},
"color": {
"mode": "palette-classic"
}
]
},
"overrides": []
}
},
{
@@ -689,7 +731,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 49
"y": 57
},
"collapsed": false,
"panels": []
@@ -702,7 +744,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 50
"y": 58
},
"options": {
"tooltip": {
@@ -763,7 +805,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 50
"y": 58
},
"options": {
"tooltip": {
@@ -824,7 +866,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 58
"y": 66
},
"collapsed": false,
"panels": []
@@ -837,7 +879,7 @@
"h": 8,
"w": 24,
"x": 0,
"y": 59
"y": 67
},
"options": {
"tooltip": {
@@ -882,7 +924,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 67
"y": 75
},
"collapsed": false,
"panels": []
@@ -895,7 +937,7 @@
"h": 8,
"w": 6,
"x": 0,
"y": 68
"y": 76
},
"options": {
"tooltip": {
@@ -975,7 +1017,7 @@
"h": 8,
"w": 6,
"x": 6,
"y": 68
"y": 76
},
"options": {
"tooltip": {
@@ -1008,7 +1050,7 @@
"h": 8,
"w": 6,
"x": 12,
"y": 68
"y": 76
},
"options": {
"tooltip": {
@@ -1041,7 +1083,7 @@
"h": 8,
"w": 6,
"x": 18,
"y": 68
"y": 76
},
"options": {
"tooltip": {
@@ -1067,14 +1109,14 @@
}
},
{
"title": "Last Close Info",
"title": "Last Close \u2014 Proposers",
"description": "###### What this is:\n*Proposer count and convergence time from the last closed consensus round.*\n\n###### How it's computed:\n*Current values of the last-close proposer-count and convergence-time gauges, plotted as lines.*\n\n###### Reading it:\n*A healthy proposer count with a low convergence time is good.*\n\n###### Healthy range:\n*Convergence time of a few seconds with the expected number of proposers.*\n\n###### Watch for:\n*A falling proposer count or rising convergence time signals degrading consensus conditions.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 76
"y": 84,
"w": 12,
"h": 8
},
"options": {
"tooltip": {
@@ -1088,14 +1130,8 @@
"type": "prometheus"
},
"expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"last_close_proposers\"}",
"legendFormat": "Proposers [{{service_instance_id}}]"
},
{
"datasource": {
"type": "prometheus"
},
"expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"last_close_converge_time_ms\"}",
"legendFormat": "Converge Time ms [{{service_instance_id}}]"
"legendFormat": "Proposers [{{service_instance_id}}]",
"refId": "A"
}
],
"fieldConfig": {
@@ -1111,24 +1147,49 @@
"mode": "palette-classic"
}
},
"overrides": [
{
"matcher": {
"id": "byRegexp",
"options": "/Converge Time ms/"
},
"properties": [
{
"id": "unit",
"value": "ms"
},
{
"id": "custom.axisPlacement",
"value": "right"
}
]
"overrides": []
}
},
{
"title": "Last Close \u2014 Converge Time",
"description": "###### What this is:\n*Proposer count and convergence time from the last closed consensus round.*\n\n###### How it's computed:\n*Current values of the last-close proposer-count and convergence-time gauges, plotted as lines.*\n\n###### Reading it:\n*A healthy proposer count with a low convergence time is good.*\n\n###### Healthy range:\n*Convergence time of a few seconds with the expected number of proposers.*\n\n###### Watch for:\n*A falling proposer count or rising convergence time signals degrading consensus conditions.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`",
"type": "timeseries",
"gridPos": {
"x": 12,
"y": 84,
"w": 12,
"h": 8
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus"
},
"expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"last_close_converge_time_ms\"}",
"legendFormat": "Converge Time ms [{{service_instance_id}}]",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "ms",
"custom": {
"axisLabel": "Proposers",
"drawStyle": "line",
"lineWidth": 2,
"fillOpacity": 10
},
"color": {
"mode": "palette-classic"
}
]
},
"overrides": []
}
},
{
@@ -1139,7 +1200,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 76
"y": 92
},
"options": {
"tooltip": {
@@ -1173,7 +1234,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 84
"y": 100
},
"options": {
"tooltip": {
@@ -1206,7 +1267,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 84
"y": 100
},
"options": {
"tooltip": {
@@ -1246,7 +1307,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 92
"y": 108
},
"collapsed": false,
"panels": []
@@ -1259,7 +1320,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 93
"y": 109
},
"options": {
"showHeader": true
@@ -1291,7 +1352,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 93
"y": 109
},
"options": {
"tooltip": {
@@ -1346,7 +1407,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 101
"y": 117
},
"options": {
"tooltip": {
@@ -1379,7 +1440,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 101
"y": 117
},
"options": {
"tooltip": {
@@ -1418,7 +1479,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 109
"y": 125
},
"collapsed": false,
"panels": []
@@ -1431,7 +1492,7 @@
"h": 8,
"w": 6,
"x": 0,
"y": 110
"y": 126
},
"options": {
"tooltip": {
@@ -1464,7 +1525,7 @@
"h": 8,
"w": 6,
"x": 6,
"y": 110
"y": 126
},
"options": {
"tooltip": {
@@ -1497,7 +1558,7 @@
"h": 8,
"w": 6,
"x": 12,
"y": 110
"y": 126
},
"options": {
"tooltip": {
@@ -1530,7 +1591,7 @@
"h": 8,
"w": 6,
"x": 18,
"y": 110
"y": 126
},
"options": {
"tooltip": {
@@ -1591,7 +1652,7 @@
"h": 8,
"w": 24,
"x": 0,
"y": 118
"y": 134
},
"options": {
"tooltip": {