From 6f9319e3192809588a07dbb09c8fbfdb9d76d2ad Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Mon, 13 Jul 2026 20:29:38 +0100 Subject: [PATCH 1/3] refactor(telemetry): split ledger-data-sync panels by data type Each family panel (7-8 series) was overloaded. Split by Ledger/Transaction/ Account State so each panel shows only related series. Get+Share stay together within a type. - Ledger Data Exchange: 8 targets -> 3 panels - Ledger Share/Get Traffic: 8 targets -> 3 panels - GetObject Traffic by Type: 8 targets -> 3 panels - GetObject Messages by Type: 7 targets -> 3 + specials panel - GetObject Aggregate & Special: renamed, kept as one - Overlay Traffic Heatmap: unchanged Co-Authored-By: Claude Opus 4.6 (1M context) --- .../grafana/dashboards/ledger-data-sync.json | 579 +++++++++++++----- 1 file changed, 441 insertions(+), 138 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index 9a0569bc7b..11868f5e15 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -10,7 +10,7 @@ "links": [], "panels": [ { - "title": "Ledger Data Exchange (Bytes In)", + "title": "Ledger Data — Ledger", "description": "###### What this is:\n*Inbound bytes for ledger-data message categories, split into aggregate get/share plus the transaction-set, transaction-node, and account-state-node sub-types the node receives from peers.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Normally low and flat once synced. Account-state-node traffic dominates during state sync; transaction-set-candidate traffic dominates during consensus catch-up.*\n\n###### Healthy range:\n*workload-dependent; low and steady on a synced node.*\n\n###### Watch for:\n*Sustained high account-state or tx-node inbound bytes on a node that should be caught up (repeated re-sync, missing history), or a single peer driving all traffic.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", "type": "timeseries", "gridPos": { @@ -39,7 +39,40 @@ }, "expr": "rate(ledger_data_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", "legendFormat": "Ledger Data Share [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": true, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } }, + "overrides": [] + }, + "id": 1 + }, + { + "title": "Ledger Data — Transaction", + "description": "###### What this is:\n*Inbound bytes for ledger-data message categories, split into aggregate get/share plus the transaction-set, transaction-node, and account-state-node sub-types the node receives from peers.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Normally low and flat once synced. Account-state-node traffic dominates during state sync; transaction-set-candidate traffic dominates during consensus catch-up.*\n\n###### Healthy range:\n*workload-dependent; low and steady on a synced node.*\n\n###### Watch for:\n*Sustained high account-state or tx-node inbound bytes on a node that should be caught up (repeated re-sync, missing history), or a single peer driving all traffic.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ { "datasource": { "type": "prometheus" @@ -67,7 +100,40 @@ }, "expr": "rate(ledger_data_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", "legendFormat": "TX Node Share [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": true, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } }, + "overrides": [] + }, + "id": 2 + }, + { + "title": "Ledger Data — Account State", + "description": "###### What this is:\n*Inbound bytes for ledger-data message categories, split into aggregate get/share plus the transaction-set, transaction-node, and account-state-node sub-types the node receives from peers.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Normally low and flat once synced. Account-state-node traffic dominates during state sync; transaction-set-candidate traffic dominates during consensus catch-up.*\n\n###### Healthy range:\n*workload-dependent; low and steady on a synced node.*\n\n###### Watch for:\n*Sustained high account-state or tx-node inbound bytes on a node that should be caught up (repeated re-sync, missing history), or a single peer driving all traffic.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ { "datasource": { "type": "prometheus" @@ -95,17 +161,18 @@ } }, "overrides": [] - } + }, + "id": 3 }, { - "title": "Ledger Share/Get Traffic (Bytes)", + "title": "Ledger Traffic — Ledger", "description": "###### What this is:\n*Inbound bytes for the older ledger share/get message categories and their tx-set, tx-node, and account-state sub-types.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Usually small; these legacy categories carry ledger-fetch traffic for peers using the older protocol.*\n\n###### Healthy range:\n*workload-dependent; low on a synced node.*\n\n###### Watch for:\n*Large sustained volumes indicating heavy fetch load or a peer repeatedly requesting the same data.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", "type": "timeseries", "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 8 }, "options": { "tooltip": { @@ -114,13 +181,6 @@ } }, "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "rate(ledger_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", - "legendFormat": "Ledger Share In [{{service_instance_id}}]" - }, { "datasource": { "type": "prometheus" @@ -132,43 +192,8 @@ "datasource": { "type": "prometheus" }, - "expr": "rate(ledger_transaction_set_candidate_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", - "legendFormat": "TX Set Candidate Share [{{service_instance_id}}]" - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "rate(ledger_transaction_set_candidate_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", - "legendFormat": "TX Set Candidate Get [{{service_instance_id}}]" - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "rate(ledger_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", - "legendFormat": "TX Node Share [{{service_instance_id}}]" - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "rate(ledger_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", - "legendFormat": "TX Node Get [{{service_instance_id}}]" - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "rate(ledger_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", - "legendFormat": "Account State Share [{{service_instance_id}}]" - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "rate(ledger_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", - "legendFormat": "Account State Get [{{service_instance_id}}]" + "expr": "rate(ledger_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", + "legendFormat": "Ledger Share In [{{service_instance_id}}]" } ], "fieldConfig": { @@ -183,17 +208,126 @@ } }, "overrides": [] - } + }, + "id": 4 }, { - "title": "GetObject Traffic by Type (Bytes In)", + "title": "Ledger Traffic — Transaction", + "description": "###### What this is:\n*Inbound bytes for the older ledger share/get message categories and their tx-set, tx-node, and account-state sub-types.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Usually small; these legacy categories carry ledger-fetch traffic for peers using the older protocol.*\n\n###### Healthy range:\n*workload-dependent; low on a synced node.*\n\n###### Watch for:\n*Large sustained volumes indicating heavy fetch load or a peer repeatedly requesting the same data.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "rate(ledger_transaction_set_candidate_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", + "legendFormat": "TX Set Candidate Get [{{service_instance_id}}]" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "rate(ledger_transaction_set_candidate_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", + "legendFormat": "TX Set Candidate Share [{{service_instance_id}}]" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "rate(ledger_transaction_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", + "legendFormat": "TX Node Get [{{service_instance_id}}]" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "rate(ledger_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", + "legendFormat": "TX Node Share [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": true, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 5 + }, + { + "title": "Ledger Traffic — Account State", + "description": "###### What this is:\n*Inbound bytes for the older ledger share/get message categories and their tx-set, tx-node, and account-state sub-types.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Usually small; these legacy categories carry ledger-fetch traffic for peers using the older protocol.*\n\n###### Healthy range:\n*workload-dependent; low on a synced node.*\n\n###### Watch for:\n*Large sustained volumes indicating heavy fetch load or a peer repeatedly requesting the same data.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "rate(ledger_account_state_node_get_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", + "legendFormat": "Account State Get [{{service_instance_id}}]" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "rate(ledger_account_state_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", + "legendFormat": "Account State Share [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": true, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 6 + }, + { + "title": "GetObject — Ledger", "description": "###### What this is:\n*Inbound bytes for object-fetch traffic broken down by object type: ledger headers, individual transactions, transaction-tree nodes, and state-tree nodes.*\n\n###### How it's computed:\n*Per-type inbound byte rate per node.*\n\n###### Reading it:\n*Small during steady state; grows when the node fetches missing tree nodes.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*A large share on state/tx nodes for long periods (persistent gap-filling), meaning the node keeps catching up.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", "type": "timeseries", "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 24 }, "options": { "tooltip": { @@ -215,7 +349,40 @@ }, "expr": "rate(getobject_ledger_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", "legendFormat": "Ledger Share [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": true, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } }, + "overrides": [] + }, + "id": 7 + }, + { + "title": "GetObject — Transaction", + "description": "###### What this is:\n*Inbound bytes for object-fetch traffic broken down by object type: ledger headers, individual transactions, transaction-tree nodes, and state-tree nodes.*\n\n###### How it's computed:\n*Per-type inbound byte rate per node.*\n\n###### Reading it:\n*Small during steady state; grows when the node fetches missing tree nodes.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*A large share on state/tx nodes for long periods (persistent gap-filling), meaning the node keeps catching up.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ { "datasource": { "type": "prometheus" @@ -243,7 +410,40 @@ }, "expr": "rate(getobject_transaction_node_share_bytes_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", "legendFormat": "TX Node Share [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "axisLabel": "Bytes In", + "spanNulls": true, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } }, + "overrides": [] + }, + "id": 8 + }, + { + "title": "GetObject — Account State", + "description": "###### What this is:\n*Inbound bytes for object-fetch traffic broken down by object type: ledger headers, individual transactions, transaction-tree nodes, and state-tree nodes.*\n\n###### How it's computed:\n*Per-type inbound byte rate per node.*\n\n###### Reading it:\n*Small during steady state; grows when the node fetches missing tree nodes.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*A large share on state/tx nodes for long periods (persistent gap-filling), meaning the node keeps catching up.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ { "datasource": { "type": "prometheus" @@ -271,17 +471,199 @@ } }, "overrides": [] - } + }, + "id": 9 }, { - "title": "GetObject Aggregate & Special Types (Bytes In)", + "title": "GetObject Messages — Ledger", + "description": "###### What this is:\n*Count of individual object-fetch request/response messages per object type.*\n\n###### How it's computed:\n*Per-type inbound message rate per node.*\n\n###### Reading it:\n*Many messages with few bytes means small piecemeal fetches; few messages with many bytes means large batch transfers.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High message counts with tiny payloads sustained over time (inefficient per-node fetching).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "rate(getobject_ledger_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", + "legendFormat": "Ledger Get [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cps", + "custom": { + "axisLabel": "Messages In", + "spanNulls": true, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 10 + }, + { + "title": "GetObject Messages — Transaction", + "description": "###### What this is:\n*Count of individual object-fetch request/response messages per object type.*\n\n###### How it's computed:\n*Per-type inbound message rate per node.*\n\n###### Reading it:\n*Many messages with few bytes means small piecemeal fetches; few messages with many bytes means large batch transfers.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High message counts with tiny payloads sustained over time (inefficient per-node fetching).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "rate(getobject_transaction_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", + "legendFormat": "Transaction Get [{{service_instance_id}}]" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "rate(getobject_transaction_node_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", + "legendFormat": "TX Node Get [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cps", + "custom": { + "axisLabel": "Messages In", + "spanNulls": true, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 11 + }, + { + "title": "GetObject Messages — Account State", + "description": "###### What this is:\n*Count of individual object-fetch request/response messages per object type.*\n\n###### How it's computed:\n*Per-type inbound message rate per node.*\n\n###### Reading it:\n*Many messages with few bytes means small piecemeal fetches; few messages with many bytes means large batch transfers.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High message counts with tiny payloads sustained over time (inefficient per-node fetching).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "rate(getobject_account_state_node_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", + "legendFormat": "Account State Get [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cps", + "custom": { + "axisLabel": "Messages In", + "spanNulls": true, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 12 + }, + { + "title": "GetObject Messages — Specials", + "description": "###### What this is:\n*Count of individual object-fetch request/response messages per object type.*\n\n###### How it's computed:\n*Per-type inbound message rate per node.*\n\n###### Reading it:\n*Many messages with few bytes means small piecemeal fetches; few messages with many bytes means large batch transfers.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High message counts with tiny payloads sustained over time (inefficient per-node fetching).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "rate(getobject_cas_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", + "legendFormat": "CAS Get [{{service_instance_id}}]" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "rate(getobject_fetch_pack_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", + "legendFormat": "Fetch Pack Get [{{service_instance_id}}]" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "rate(getobject_transactions_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", + "legendFormat": "Transactions Get [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cps", + "custom": { + "axisLabel": "Messages In", + "spanNulls": true, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + }, + "id": 13 + }, + { + "title": "GetObject — Specials", "description": "###### What this is:\n*Aggregate object-fetch inbound bytes plus special buckets: content-addressed storage fetches, bulk fetch-pack downloads used during catch-up, and bulk transaction fetches.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Fetch-pack rises sharply while catching up a range of ledgers; near zero when fully synced.*\n\n###### Healthy range:\n*workload-dependent; low when synced.*\n\n###### Watch for:\n*Continuous fetch-pack traffic (node never fully catches up) or unexpectedly high content-store volume.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", "type": "timeseries", "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 48 }, "options": { "tooltip": { @@ -352,88 +734,8 @@ } }, "overrides": [] - } - }, - { - "title": "GetObject Messages by Type", - "description": "###### What this is:\n*Count of individual object-fetch request/response messages per object type.*\n\n###### How it's computed:\n*Per-type inbound message rate per node.*\n\n###### Reading it:\n*Many messages with few bytes means small piecemeal fetches; few messages with many bytes means large batch transfers.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*High message counts with tiny payloads sustained over time (inefficient per-node fetching).*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "rate(getobject_ledger_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", - "legendFormat": "Ledger Get [{{service_instance_id}}]" - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "rate(getobject_transaction_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", - "legendFormat": "Transaction Get [{{service_instance_id}}]" - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "rate(getobject_transaction_node_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", - "legendFormat": "TX Node Get [{{service_instance_id}}]" - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "rate(getobject_account_state_node_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", - "legendFormat": "Account State Get [{{service_instance_id}}]" - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "rate(getobject_cas_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", - "legendFormat": "CAS Get [{{service_instance_id}}]" - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "rate(getobject_fetch_pack_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", - "legendFormat": "Fetch Pack Get [{{service_instance_id}}]" - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "rate(getobject_transactions_get_messages_in{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])", - "legendFormat": "Transactions Get [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "cps", - "custom": { - "axisLabel": "Messages In", - "spanNulls": true, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 3 - } - }, - "overrides": [] - } + "id": 14 }, { "title": "Overlay Traffic Heatmap (All Categories, Bytes In)", @@ -442,8 +744,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 16 + "x": 0, + "y": 56 }, "options": { "tooltip": { @@ -489,7 +791,8 @@ } }, "overrides": [] - } + }, + "id": 15 } ], "schemaVersion": 39, From d06faec407b372cdc2747515618093e26827065e Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Mon, 13 Jul 2026 20:29:52 +0100 Subject: [PATCH 2/3] fix(telemetry): split mixed-unit node-health panels into single-unit panels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three panels mixed incompatible units on one Y-axis (count vs ms, ops vs percent), causing the small-magnitude series to flatline to zero. - Last Close Info -> Last Close — Proposers (short) + Converge Time (ms) - NodeStore Read/Write Totals -> Throughput (ops) + Cache Hit Ratio (%) - NodeStore Read Threads & Duration -> Read Threads (short) + Busy Ratio (%) Each new panel has a single unit; per-series unit overrides removed. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../grafana/dashboards/node-health.json | 305 +++++++++++------- 1 file changed, 183 insertions(+), 122 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index 37936c7e3d..966f3cd1c3 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -412,14 +412,14 @@ "panels": [] }, { - "title": "NodeStore Read/Write Totals", + "title": "NodeStore Throughput", "description": "###### What this is:\n*Cumulative object-store read, cache-hit, and write operation counts.*\n\n###### How it's computed:\n*Current values of the read, cache-hit, and write counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope reflects store activity.*\n\n###### Healthy range:\n*Reads and writes rising smoothly, with cache-hits tracking a good fraction of reads.*\n\n###### Watch for:\n*A sudden surge in reads or writes signals heavy back-end I/O, from sync, replay, or query load.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`", "type": "timeseries", "gridPos": { - "h": 8, - "w": 12, "x": 0, - "y": 33 + "y": 33, + "w": 12, + "h": 8 }, "options": { "tooltip": { @@ -433,29 +433,24 @@ "type": "prometheus" }, "expr": "rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])", - "legendFormat": "Reads Total [{{service_instance_id}}]" + "legendFormat": "Reads Total [{{service_instance_id}}]", + "refId": "A" }, { "datasource": { "type": "prometheus" }, "expr": "rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval])", - "legendFormat": "Reads Hit (cache) [{{service_instance_id}}]" + "legendFormat": "Reads Hit (cache) [{{service_instance_id}}]", + "refId": "B" }, { "datasource": { "type": "prometheus" }, "expr": "rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_writes\"}[$__rate_interval])", - "legendFormat": "Writes Total [{{service_instance_id}}]" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval]) / rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])", - "legendFormat": "Cache Hit Ratio [{{service_instance_id}}]" + "legendFormat": "Writes Total [{{service_instance_id}}]", + "refId": "C" } ], "fieldConfig": { @@ -471,24 +466,50 @@ "mode": "palette-classic" } }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/Cache Hit Ratio/" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "custom.axisPlacement", - "value": "right" - } - ] + "overrides": [] + } + }, + { + "title": "NodeStore Cache Hit Ratio", + "description": "###### What this is:\n*Cumulative object-store read, cache-hit, and write operation counts.*\n\n###### How it's computed:\n*Current values of the read, cache-hit, and write counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope reflects store activity.*\n\n###### Healthy range:\n*Reads and writes rising smoothly, with cache-hits tracking a good fraction of reads.*\n\n###### Watch for:\n*A sudden surge in reads or writes signals heavy back-end I/O, from sync, replay, or query load.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`", + "type": "timeseries", + "gridPos": { + "x": 12, + "y": 33, + "w": 12, + "h": 8 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval]) / rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])", + "legendFormat": "Cache Hit Ratio [{{service_instance_id}}]", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "axisLabel": "Operations / Sec", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10 + }, + "color": { + "mode": "palette-classic" } - ] + }, + "overrides": [] } }, { @@ -496,10 +517,10 @@ "description": "###### What this is:\n*Instantaneous write-load score and read-queue depth of the object store.*\n\n###### How it's computed:\n*Current values of the write-load and read-queue gauges, plotted over time.*\n\n###### Reading it:\n*Lower is better for both; short, flat lines are healthy.*\n\n###### Healthy range:\n*Write load near zero and read queue in low double digits or less.*\n\n###### Watch for:\n*High write load means back-end pressure; a high read queue means the prefetch threads are saturated.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`", "type": "timeseries", "gridPos": { - "h": 8, + "x": 0, + "y": 41, "w": 12, - "x": 12, - "y": 33 + "h": 8 }, "options": { "tooltip": { @@ -560,10 +581,10 @@ "description": "###### What this is:\n*Cumulative bytes read from and written to the object-store back end.*\n\n###### How it's computed:\n*Current values of the bytes-read and bytes-written counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope shows throughput.*\n\n###### Healthy range:\n*Smooth growth consistent with ledger and query activity.*\n\n###### Watch for:\n*A sharp acceleration indicates a heavy I/O phase such as sync, replay, or large queries.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`", "type": "timeseries", "gridPos": { - "h": 8, + "x": 12, + "y": 41, "w": 12, - "x": 0, - "y": 41 + "h": 8 }, "options": { "tooltip": { @@ -604,14 +625,14 @@ } }, { - "title": "NodeStore Read Threads & Duration", + "title": "NodeStore Read Threads", "description": "###### What this is:\n*Read-thread utilization, bundled read count, and cumulative read time for the object store.*\n\n###### How it's computed:\n*Current values of the running/total read-thread gauges, read-bundle gauge, and cumulative read-duration counter, plotted as lines.*\n\n###### Reading it:\n*Running threads well below total means spare capacity; a rising duration line reflects time spent in read I/O.*\n\n###### Healthy range:\n*Running threads below the total count most of the time.*\n\n###### Watch for:\n*Running threads pinned at the total for long periods means read I/O is saturated.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`", "type": "timeseries", "gridPos": { - "h": 8, + "x": 0, + "y": 49, "w": 12, - "x": 12, - "y": 41 + "h": 8 }, "options": { "tooltip": { @@ -625,28 +646,24 @@ "type": "prometheus" }, "expr": "nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_threads_running\"}", - "legendFormat": "Read Threads Running [{{service_instance_id}}]" + "legendFormat": "Read Threads Running [{{service_instance_id}}]", + "refId": "A" }, { "datasource": { "type": "prometheus" }, "expr": "nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_threads_total\"}", - "legendFormat": "Read Threads Total [{{service_instance_id}}]" + "legendFormat": "Read Threads Total [{{service_instance_id}}]", + "refId": "B" }, { "datasource": { "type": "prometheus" }, "expr": "nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_request_bundle\"}", - "legendFormat": "Read Request Bundle [{{service_instance_id}}]" - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_duration_us\"}[$__rate_interval]) / 1e6", - "legendFormat": "Read Busy Ratio [{{service_instance_id}}]" + "legendFormat": "Read Request Bundle [{{service_instance_id}}]", + "refId": "C" } ], "fieldConfig": { @@ -662,24 +679,49 @@ "mode": "palette-classic" } }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/Read Busy Ratio/" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "custom.axisPlacement", - "value": "right" - } - ] + "overrides": [] + } + }, + { + "title": "NodeStore Read Busy Ratio", + "description": "###### What this is:\n*Read-thread utilization, bundled read count, and cumulative read time for the object store.*\n\n###### How it's computed:\n*Current values of the running/total read-thread gauges, read-bundle gauge, and cumulative read-duration counter, plotted as lines.*\n\n###### Reading it:\n*Running threads well below total means spare capacity; a rising duration line reflects time spent in read I/O.*\n\n###### Healthy range:\n*Running threads below the total count most of the time.*\n\n###### Watch for:\n*Running threads pinned at the total for long periods means read I/O is saturated.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`", + "type": "timeseries", + "gridPos": { + "x": 12, + "y": 49, + "w": 12, + "h": 8 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_duration_us\"}[$__rate_interval]) / 1e6", + "legendFormat": "Read Busy Ratio [{{service_instance_id}}]", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "axisLabel": "Threads / Bundle", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10 + }, + "color": { + "mode": "palette-classic" } - ] + }, + "overrides": [] } }, { @@ -689,7 +731,7 @@ "h": 1, "w": 24, "x": 0, - "y": 49 + "y": 57 }, "collapsed": false, "panels": [] @@ -702,7 +744,7 @@ "h": 8, "w": 12, "x": 0, - "y": 50 + "y": 58 }, "options": { "tooltip": { @@ -763,7 +805,7 @@ "h": 8, "w": 12, "x": 12, - "y": 50 + "y": 58 }, "options": { "tooltip": { @@ -824,7 +866,7 @@ "h": 1, "w": 24, "x": 0, - "y": 58 + "y": 66 }, "collapsed": false, "panels": [] @@ -837,7 +879,7 @@ "h": 8, "w": 24, "x": 0, - "y": 59 + "y": 67 }, "options": { "tooltip": { @@ -882,7 +924,7 @@ "h": 1, "w": 24, "x": 0, - "y": 67 + "y": 75 }, "collapsed": false, "panels": [] @@ -895,7 +937,7 @@ "h": 8, "w": 6, "x": 0, - "y": 68 + "y": 76 }, "options": { "tooltip": { @@ -975,7 +1017,7 @@ "h": 8, "w": 6, "x": 6, - "y": 68 + "y": 76 }, "options": { "tooltip": { @@ -1008,7 +1050,7 @@ "h": 8, "w": 6, "x": 12, - "y": 68 + "y": 76 }, "options": { "tooltip": { @@ -1041,7 +1083,7 @@ "h": 8, "w": 6, "x": 18, - "y": 68 + "y": 76 }, "options": { "tooltip": { @@ -1067,14 +1109,14 @@ } }, { - "title": "Last Close Info", + "title": "Last Close \u2014 Proposers", "description": "###### What this is:\n*Proposer count and convergence time from the last closed consensus round.*\n\n###### How it's computed:\n*Current values of the last-close proposer-count and convergence-time gauges, plotted as lines.*\n\n###### Reading it:\n*A healthy proposer count with a low convergence time is good.*\n\n###### Healthy range:\n*Convergence time of a few seconds with the expected number of proposers.*\n\n###### Watch for:\n*A falling proposer count or rising convergence time signals degrading consensus conditions.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`", "type": "timeseries", "gridPos": { - "h": 8, - "w": 12, "x": 0, - "y": 76 + "y": 84, + "w": 12, + "h": 8 }, "options": { "tooltip": { @@ -1088,14 +1130,8 @@ "type": "prometheus" }, "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"last_close_proposers\"}", - "legendFormat": "Proposers [{{service_instance_id}}]" - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"last_close_converge_time_ms\"}", - "legendFormat": "Converge Time ms [{{service_instance_id}}]" + "legendFormat": "Proposers [{{service_instance_id}}]", + "refId": "A" } ], "fieldConfig": { @@ -1111,24 +1147,49 @@ "mode": "palette-classic" } }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/Converge Time ms/" - }, - "properties": [ - { - "id": "unit", - "value": "ms" - }, - { - "id": "custom.axisPlacement", - "value": "right" - } - ] + "overrides": [] + } + }, + { + "title": "Last Close \u2014 Converge Time", + "description": "###### What this is:\n*Proposer count and convergence time from the last closed consensus round.*\n\n###### How it's computed:\n*Current values of the last-close proposer-count and convergence-time gauges, plotted as lines.*\n\n###### Reading it:\n*A healthy proposer count with a low convergence time is good.*\n\n###### Healthy range:\n*Convergence time of a few seconds with the expected number of proposers.*\n\n###### Watch for:\n*A falling proposer count or rising convergence time signals degrading consensus conditions.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`", + "type": "timeseries", + "gridPos": { + "x": 12, + "y": 84, + "w": 12, + "h": 8 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"last_close_converge_time_ms\"}", + "legendFormat": "Converge Time ms [{{service_instance_id}}]", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "axisLabel": "Proposers", + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10 + }, + "color": { + "mode": "palette-classic" } - ] + }, + "overrides": [] } }, { @@ -1139,7 +1200,7 @@ "h": 8, "w": 12, "x": 12, - "y": 76 + "y": 92 }, "options": { "tooltip": { @@ -1173,7 +1234,7 @@ "h": 8, "w": 12, "x": 0, - "y": 84 + "y": 100 }, "options": { "tooltip": { @@ -1206,7 +1267,7 @@ "h": 8, "w": 12, "x": 12, - "y": 84 + "y": 100 }, "options": { "tooltip": { @@ -1246,7 +1307,7 @@ "h": 1, "w": 24, "x": 0, - "y": 92 + "y": 108 }, "collapsed": false, "panels": [] @@ -1259,7 +1320,7 @@ "h": 8, "w": 12, "x": 0, - "y": 93 + "y": 109 }, "options": { "showHeader": true @@ -1291,7 +1352,7 @@ "h": 8, "w": 12, "x": 12, - "y": 93 + "y": 109 }, "options": { "tooltip": { @@ -1346,7 +1407,7 @@ "h": 8, "w": 12, "x": 0, - "y": 101 + "y": 117 }, "options": { "tooltip": { @@ -1379,7 +1440,7 @@ "h": 8, "w": 12, "x": 12, - "y": 101 + "y": 117 }, "options": { "tooltip": { @@ -1418,7 +1479,7 @@ "h": 1, "w": 24, "x": 0, - "y": 109 + "y": 125 }, "collapsed": false, "panels": [] @@ -1431,7 +1492,7 @@ "h": 8, "w": 6, "x": 0, - "y": 110 + "y": 126 }, "options": { "tooltip": { @@ -1464,7 +1525,7 @@ "h": 8, "w": 6, "x": 6, - "y": 110 + "y": 126 }, "options": { "tooltip": { @@ -1497,7 +1558,7 @@ "h": 8, "w": 6, "x": 12, - "y": 110 + "y": 126 }, "options": { "tooltip": { @@ -1530,7 +1591,7 @@ "h": 8, "w": 6, "x": 18, - "y": 110 + "y": 126 }, "options": { "tooltip": { @@ -1591,7 +1652,7 @@ "h": 8, "w": 24, "x": 0, - "y": 118 + "y": 134 }, "options": { "tooltip": { From 674c9a8496d3f1ab707832cf6afc59f26d9abf7a Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Mon, 13 Jul 2026 20:48:51 +0100 Subject: [PATCH 3/3] fix(telemetry): correct NodeStore Cache Hit Ratio PromQL vector matching The Cache Hit Ratio panel divided two nodestore_state series that carry different `metric` label values (node_reads_hit / node_reads_total). Without ignoring(metric) the two vectors share no common label set, so the division matched nothing and the panel showed "No data". Add `ignoring(metric)` to the division. Verified live against mainnet: now returns a valid ratio (~0.11-0.23). Pre-existing bug, surfaced when the panel was split out from NodeStore Read/Write Totals. Co-Authored-By: Claude Opus 4.6 (1M context) --- docker/telemetry/grafana/dashboards/node-health.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index 966f3cd1c3..869bd1b4c4 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -491,7 +491,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval]) / rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])", + "expr": "rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval]) / ignoring(metric) rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])", "legendFormat": "Cache Hit Ratio [{{service_instance_id}}]", "refId": "A" }