From 9c54020322a475b94f8c4eaa17d77959b12ae634 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Fri, 24 Jul 2026 20:50:39 +0100 Subject: [PATCH] fix(telemetry): plot all five states in State Duration Rate panel The panel plotted only Full and Tracking, so the states a node actually passes through while catching up were invisible. On the dev box's cold sync the node spent ~19 minutes in Connected and ~3 seconds in Syncing, while Tracking totalled 2 microseconds -- the one non-Full line the panel did draw was the least informative of the five. Add Syncing, Connected and Disconnected series (state-ladder order, matching the Operating Mode Transitions panel) and rename the panel to "State Duration Rate (All States)". Because the node is always in exactly one state, the five rates sum to ~1.0, so the panel now reads as "which state is time going into right now", and a handover between two lines marks a state change with its width showing the dwell time. Description expanded to the full section set (keywords, computation boundary, references) used by the other panels on this dashboard. Verified live: rate(state_accounting_connected_duration)/1e6 reaches 1.0 then 0.387 across the catch-up window and syncing reaches 0.0103 -- both previously undrawable. --- .../grafana/dashboards/node-health.json | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index 31bde919b7..d36a9e779f 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -2139,8 +2139,8 @@ } }, { - "title": "State Duration Rate (Full vs Tracking)", - "description": "###### What this is:\n*Rate of change of time spent in Full and Tracking modes, normalized to seconds per second.*\n\n###### How it's computed:\n*Per-second rate of the Full and Tracking duration counters, scaled to seconds.*\n\n###### Reading it:\n*The Full line near 1.0 means the node gains a second of Full-mode time each wall-clock second.*\n\n###### Healthy range:\n*Full rate close to 1.0, Tracking near 0.*\n\n###### Watch for:\n*A Full rate below 1.0 means the node is spending time in other modes.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`", + "title": "State Duration Rate (All States)", + "description": "###### What this is:\n*Which operating mode the node is accumulating time in right now, one line per state, normalized to seconds per second.*\n\n###### How it's computed:\n*Per-second rate of each state's duration counter, scaled from microseconds to seconds. The five lines sum to about 1.0 because the node is always in exactly one state.*\n\n###### Reading it:\n*The line sitting near 1.0 is the state the node is currently in; the others sit at 0. A handover between two lines marks a state change, and its width is how long that state lasted.*\n\n###### Healthy range:\n*Full near 1.0 with every other line at 0.*\n\n###### Watch for:\n*Time accumulating in Connected or Syncing means the node is catching up rather than serving; repeated handovers mean it is flapping.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* — the node's sync level: Disconnected, Connected, Syncing, Tracking, Full.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Server states](https://xrpl.org/docs/concepts/networks-and-servers/rippled-server-states)", "type": "timeseries", "gridPos": { "h": 8, @@ -2167,6 +2167,24 @@ "type": "prometheus" }, "expr": "label_replace(rate(state_accounting_tracking_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / 1000000, \"series\", \"Tracking Mode Rate\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(state_accounting_syncing_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / 1000000, \"series\", \"Syncing Mode Rate\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(state_accounting_connected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / 1000000, \"series\", \"Connected Mode Rate\", \"\", \"\")" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(rate(state_accounting_disconnected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / 1000000, \"series\", \"Disconnected Mode Rate\", \"\", \"\")" } ], "fieldConfig": {