From 1d5697d7d3fb883e7e7b3b19cb3b7e8153e3c200 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:31:54 +0100 Subject: [PATCH 1/4] style(telemetry): uniform dashboard layout, stable panel ids, row grouping These dashboards were hand-authored over time and had drifted apart: panel heights spanned ten different values, nine dashboards had no row grouping, line-chart styling was inconsistent, and no panel carried an id, so Grafana assigned them positionally at load and every panelId deep link was only as stable as the panel order. Per panel, in document order: - id written as 1..N so panelId links address a specific panel - gridPos quantized to at most two panels across: charts h=8, stats/gauges h=4, tables/logs h=12 full width. Panels are paired only with an equal-height neighbour, so no row is left with a ragged half-empty cell. - line charts lineWidth=1, fillOpacity=0, pointSize=5, gradientMode=none - repeat xrpl_network_type (horizontal, maxPerRow=2) with a [$xrpl_network_type] title suffix, on the panel types where overlaying two networks in one panel reads as noise (stat/gauge/bargauge/table/state-timeline). Line charts keep their networks as separate series, which is the point of a line chart. - decimals 0 where the value counts discrete things (threads, peers, queue depths); a fractional thread count is meaningless. - rows category rows added where a dashboard had none Panels whose legend sits on the right stay full width: a side legend needs the horizontal room, and squeezing it to half width clips the series names. Edits were made as raw-text replacements, not a json.dump round-trip, so formatting and escaping of untouched lines are byte-identical. Verified per dashboard: panel count unchanged, every targets block byte-identical, all descriptions unchanged, ids exactly 1..N, and no two panels overlapping on the grid. The repo dashboard lint and the OTel naming check both pass. --- .../grafana/dashboards/consensus-health.json | 297 +++++++++++++----- .../grafana/dashboards/ledger-data-sync.json | 204 ++++++++---- .../grafana/dashboards/ledger-operations.json | 134 ++++++-- .../grafana/dashboards/network-traffic.json | 151 ++++++--- .../grafana/dashboards/node-health.json | 188 +++++++---- .../dashboards/overlay-traffic-detail.json | 64 +++- .../grafana/dashboards/peer-network.json | 56 +++- .../grafana/dashboards/rpc-pathfinding.json | 181 ++++++++--- .../grafana/dashboards/rpc-performance.json | 107 ++++--- .../dashboards/transaction-overview.json | 237 ++++++++++---- 10 files changed, 1179 insertions(+), 440 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json index 78fcc21d55..0c127ddcb3 100644 --- a/docker/telemetry/grafana/dashboards/consensus-health.json +++ b/docker/telemetry/grafana/dashboards/consensus-health.json @@ -8,6 +8,19 @@ "id": null, "links": [], "panels": [ + { + "title": "Consensus Rounds", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [], + "id": 1 + }, { "title": "Consensus Round Duration", "description": "**What:** Time to process an accepted ledger, from the moment consensus is reached through applying the transaction set and finalising state.\n**How it's computed:** 95th and 50th percentile of per-round durations over a 5-minute window, per node.\n**Reading it:** Flat p50 with a modest p95 gap is normal; both track network transaction load.\n**Healthy range:** Roughly 3-6 seconds on mainnet.\n**Watch for:** A rising p95 that pulls away from p50 means occasional slow rounds; sustained growth precedes ledger-age alarms.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::makeAcceptSpan", @@ -16,7 +29,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 1 }, "options": { "tooltip": { @@ -51,11 +64,28 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 2 + }, + { + "title": "Proposals & Validations", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "collapsed": false, + "panels": [], + "id": 3 }, { "title": "Consensus Proposals Sent Rate", @@ -64,8 +94,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 0 + "x": 0, + "y": 10 }, "options": { "tooltip": { @@ -92,11 +122,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 4 }, { "title": "Ledger Close Duration", @@ -105,8 +139,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 8 + "x": 12, + "y": 10 }, "options": { "tooltip": { @@ -133,21 +167,25 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 5 }, { - "title": "Validation Send Rate", + "title": "Validation Send Rate [$xrpl_network_type]", "description": "**What:** How often this node issues a validation confirming it has fully validated a ledger.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should closely track the ledger close rate on a healthy validator.\n**Healthy range:** About one per ledger (~0.25/s on mainnet).\n**Watch for:** Validations lagging closes suggests the node is falling behind on validation.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::createValidationSpan", "type": "stat", "gridPos": { - "h": 8, + "h": 4, "w": 12, - "x": 12, - "y": 8 + "x": 0, + "y": 18 }, "options": { "tooltip": { @@ -170,7 +208,11 @@ "unit": "suffix: validations/s" }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 6 }, { "title": "Ledger Apply Duration (doAccept)", @@ -180,7 +222,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 22 }, "targets": [ { @@ -215,11 +257,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 7 }, { "title": "Close Time Agreement", @@ -229,7 +275,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 22 }, "targets": [ { @@ -256,11 +302,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 8 }, { "title": "Consensus Mode Over Time", @@ -270,7 +320,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 30 }, "options": { "tooltip": { @@ -297,11 +347,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 9 }, { "title": "Accept vs Close Rate", @@ -311,7 +365,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 30 }, "options": { "tooltip": { @@ -346,11 +400,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 10 }, { "title": "Validation vs Close Rate", @@ -360,7 +418,7 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 38 }, "options": { "tooltip": { @@ -395,11 +453,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 11 }, { "title": "Consensus Accept Duration Heatmap", @@ -409,7 +471,7 @@ "h": 8, "w": 12, "x": 12, - "y": 32 + "y": 38 }, "options": { "tooltip": { @@ -440,7 +502,21 @@ "spanNulls": 1800000 } } - } + }, + "id": 12 + }, + { + "title": "Close Time Analysis", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, + "collapsed": false, + "panels": [], + "id": 13 }, { "title": "Close Time: Raw Proposals (Per Node)", @@ -450,7 +526,7 @@ "h": 8, "w": 12, "x": 0, - "y": 40 + "y": 47 }, "fieldConfig": { "defaults": { @@ -458,8 +534,11 @@ "custom": { "spanNulls": 1800000, "drawStyle": "points", - "pointSize": 6, - "showPoints": "always" + "pointSize": 5, + "showPoints": "always", + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] @@ -485,7 +564,8 @@ "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && span.close_time_correct=~\"$close_time_correct\"} | select(span.close_time_self)", "refId": "A" } - ] + ], + "id": 14 }, { "title": "Close Time: Effective / Quantized", @@ -495,7 +575,7 @@ "h": 8, "w": 12, "x": 12, - "y": 40 + "y": 47 }, "fieldConfig": { "defaults": { @@ -503,8 +583,11 @@ "custom": { "spanNulls": 1800000, "drawStyle": "points", - "pointSize": 6, - "showPoints": "always" + "pointSize": 5, + "showPoints": "always", + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] @@ -530,7 +613,8 @@ "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && span.close_time_correct=~\"$close_time_correct\"} | select(span.close_time)", "refId": "A" } - ] + ], + "id": 15 }, { "title": "Close Time Vote Bins & Resolution", @@ -540,7 +624,7 @@ "h": 8, "w": 12, "x": 0, - "y": 48 + "y": 55 }, "fieldConfig": { "defaults": { @@ -549,7 +633,10 @@ "drawStyle": "line", "lineInterpolation": "stepAfter", "pointSize": 5, - "showPoints": "auto" + "showPoints": "auto", + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [ @@ -616,7 +703,8 @@ "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && span.close_time_correct=~\"$close_time_correct\"} | select(span.close_resolution_ms)", "refId": "B" } - ] + ], + "id": 16 }, { "title": "Close Time Resolution Direction", @@ -626,16 +714,18 @@ "h": 8, "w": 12, "x": 12, - "y": 48 + "y": 55 }, "fieldConfig": { "defaults": { "custom": { "spanNulls": 1800000, "drawStyle": "bars", - "fillOpacity": 40, + "fillOpacity": 0, "pointSize": 5, - "showPoints": "auto" + "showPoints": "auto", + "lineWidth": 1, + "gradientMode": "none" } }, "overrides": [] @@ -661,7 +751,8 @@ "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && span.close_time_correct=~\"$close_time_correct\" && span.resolution_direction=~\"$resolution_direction\"} | select(span.resolution_direction)", "refId": "A" } - ] + ], + "id": 17 }, { "title": "Close Time Bin Distribution", @@ -669,9 +760,9 @@ "type": "barchart", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 56 + "y": 63 }, "fieldConfig": { "defaults": { @@ -707,7 +798,21 @@ "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && span.close_time_correct=~\"$close_time_correct\"} | select(span.close_time, span.close_time_vote_bins)", "refId": "A" } - ] + ], + "id": 18 + }, + { + "title": "Outcomes & Failures", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 71 + }, + "collapsed": false, + "panels": [], + "id": 19 }, { "title": "Consensus Outcome Distribution", @@ -715,9 +820,9 @@ "type": "piechart", "gridPos": { "h": 8, - "w": 8, + "w": 24, "x": 0, - "y": 64 + "y": 72 }, "options": { "legend": { @@ -744,7 +849,8 @@ "unit": "short" }, "overrides": [] - } + }, + "id": 20 }, { "title": "Consensus Failures Over Time", @@ -752,9 +858,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 16, - "x": 8, - "y": 64 + "w": 12, + "x": 0, + "y": 80 }, "options": { "tooltip": { @@ -789,11 +895,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 21 }, { "title": "Consensus Round Duration (Full Round)", @@ -802,8 +912,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 72 + "x": 12, + "y": 80 }, "options": { "tooltip": { @@ -838,11 +948,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 22 }, { "title": "Consensus Phase Duration (Open vs Establish)", @@ -851,8 +965,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 72 + "x": 0, + "y": 88 }, "options": { "tooltip": { @@ -887,11 +1001,28 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 23 + }, + { + "title": "Timing", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 96 + }, + "collapsed": false, + "panels": [], + "id": 24 }, { "title": "Position Update Duration", @@ -901,7 +1032,7 @@ "h": 8, "w": 12, "x": 0, - "y": 80 + "y": 97 }, "options": { "tooltip": { @@ -936,11 +1067,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 25 }, { "title": "Consensus Stall Rate", @@ -950,7 +1085,7 @@ "h": 8, "w": 12, "x": 12, - "y": 80 + "y": 97 }, "options": { "tooltip": { @@ -985,11 +1120,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 26 }, { "title": "Consensus Mode-Change Rate by Target Mode", @@ -997,9 +1136,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 88 + "y": 105 }, "options": { "tooltip": { @@ -1026,11 +1165,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 27 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index 5f02984c72..3532137aef 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -49,7 +49,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] @@ -108,7 +111,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] @@ -155,7 +161,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] @@ -202,7 +211,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] @@ -261,7 +273,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] @@ -308,7 +323,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] @@ -355,7 +373,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] @@ -414,7 +435,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] @@ -461,7 +485,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] @@ -502,7 +529,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] @@ -549,7 +579,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] @@ -590,7 +623,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] @@ -643,7 +679,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] @@ -720,7 +759,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] @@ -728,12 +770,12 @@ "id": 14 }, { - "title": "Overlay Traffic Heatmap (All Categories, Bytes In)", + "title": "Overlay Traffic Heatmap (All Categories, Bytes In) [$xrpl_network_type]", "description": "###### What this is:\n*All overlay traffic categories ranked by inbound bytes, giving an at-a-glance view of which message types consume the most receive bandwidth.*\n\n###### How it's computed:\n*Top categories by latest inbound byte value across all traffic categories. Each bar is labelled with its traffic category followed by the node identity; the shared `_bytes_in` suffix is dropped from the category name because the panel already reports inbound bytes.*\n\n###### Reading it:\n*The longest bars are the biggest bandwidth consumers; on a synced node transactions, proposals, and validations usually lead.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*A single ledger-data or fetch category dominating (ongoing sync) or an unexpected category topping the list.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", "type": "bargauge", "gridPos": { - "h": 18, - "w": 24, + "h": 4, + "w": 12, "x": 0, "y": 56 }, @@ -783,7 +825,10 @@ }, "overrides": [] }, - "id": 15 + "id": 15, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "title": "Sync Diagnostics", @@ -793,7 +838,7 @@ "h": 1, "w": 24, "x": 0, - "y": 74 + "y": 60 }, "id": 16, "panels": [] @@ -806,7 +851,7 @@ "h": 8, "w": 12, "x": 0, - "y": 75 + "y": 61 }, "options": { "tooltip": { @@ -833,7 +878,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" }, "mappings": [ { @@ -928,7 +976,7 @@ "h": 8, "w": 12, "x": 12, - "y": 75 + "y": 61 }, "options": { "tooltip": { @@ -962,7 +1010,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] @@ -970,14 +1021,14 @@ "id": 18 }, { - "title": "Time Spent Per State", + "title": "Time Spent Per State [$xrpl_network_type]", "description": "###### What this is:\n*Operating mode as a colour-coded timeline. Each band's width is the time spent in that state, so short-lived states show as thin slivers instead of vanishing.*\n\n###### How it's computed:\n*server_info{metric=\"server_state\"} (gauge), the raw OperatingMode 0-4. Uses server_state rather than state_value because state_value folds 5 and 6 onto FULL, which would split one Full band into three colours.*\n\n###### Reading it:\n*One green band across the window = healthy. Red/orange/yellow bands show when and for how long the node was degraded.*\n\n###### Healthy range:\n*Continuously green (Full), brief orange/yellow/blue only after a restart.*\n\n###### Watch for:\n*Repeated thin bands = the node is oscillating. This is sampled every 10s, so a state shorter than one sample can still be missed.*\n\n###### Source:\n[app/misc/NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::setMode`", "type": "state-timeline", "gridPos": { - "h": 6, - "w": 24, + "h": 8, + "w": 12, "x": 0, - "y": 83 + "y": 69 }, "options": { "mergeValues": true, @@ -1080,7 +1131,10 @@ }, "overrides": [] }, - "id": 27 + "id": 19, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "title": "Ledger Close Rate", @@ -1089,8 +1143,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 89 + "x": 12, + "y": 69 }, "options": { "tooltip": { @@ -1124,7 +1178,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [ @@ -1146,7 +1203,7 @@ } ] }, - "id": 19 + "id": 20 }, { "title": "Job Queue Wait p95 By Type", @@ -1155,8 +1212,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 89 + "x": 0, + "y": 77 }, "options": { "tooltip": { @@ -1218,12 +1275,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 20 + "id": 21 }, { "title": "NuDB Read Latency", @@ -1232,8 +1292,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 97 + "x": 12, + "y": 77 }, "options": { "tooltip": { @@ -1260,12 +1320,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 21 + "id": 22 }, { "title": "I/O Scheduler Latency p95", @@ -1274,8 +1337,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 97 + "x": 0, + "y": 85 }, "options": { "tooltip": { @@ -1302,12 +1365,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 22 + "id": 23 }, { "title": "NuDB Read Found Ratio", @@ -1316,8 +1382,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 105 + "x": 12, + "y": 85 }, "options": { "tooltip": { @@ -1344,12 +1410,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 23 + "id": 24 }, { "title": "NuDB Read Pressure", @@ -1358,8 +1427,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 105 + "x": 0, + "y": 93 }, "options": { "tooltip": { @@ -1400,7 +1469,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [ @@ -1438,7 +1510,7 @@ } ] }, - "id": 24 + "id": 25 }, { "title": "Job Queue Depth", @@ -1447,8 +1519,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 113 + "x": 12, + "y": 93 }, "options": { "tooltip": { @@ -1475,12 +1547,16 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 - } + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + }, + "decimals": 0 }, "overrides": [] }, - "id": 25 + "id": 26 }, { "title": "Load Factor & Peers", @@ -1489,8 +1565,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 113 + "x": 0, + "y": 101 }, "options": { "tooltip": { @@ -1531,12 +1607,16 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 - } + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + }, + "decimals": 0 }, "overrides": [] }, - "id": 26 + "id": 27 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/ledger-operations.json b/docker/telemetry/grafana/dashboards/ledger-operations.json index 281731ad2a..4c04a97c66 100644 --- a/docker/telemetry/grafana/dashboards/ledger-operations.json +++ b/docker/telemetry/grafana/dashboards/ledger-operations.json @@ -9,14 +9,27 @@ "links": [], "panels": [ { - "title": "Ledger Build Rate", + "title": "Ledger Build & Validate", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [], + "id": 1 + }, + { + "title": "Ledger Build Rate [$xrpl_network_type]", "description": "**What:** How often the node constructs a new ledger from its parent, applying transactions and writing state.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should match the consensus close cadence.\n**Healthy range:** About 0.25/s on mainnet (~4s rounds).\n**Watch for:** A drop below the close rate means the node is not keeping up with ledger construction.\n**Source:** src/xrpld/app/ledger/detail/BuildLedger.cpp:buildLedgerImpl", "type": "stat", "gridPos": { - "h": 8, - "w": 8, + "h": 4, + "w": 12, "x": 0, - "y": 0 + "y": 1 }, "options": { "tooltip": { @@ -39,7 +52,11 @@ "unit": "suffix: ledgers/s" }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 2 }, { "title": "Ledger Build Duration", @@ -49,7 +66,7 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 5 }, "options": { "tooltip": { @@ -84,21 +101,25 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 3 }, { - "title": "Ledger Validation Rate", + "title": "Ledger Validation Rate [$xrpl_network_type]", "description": "**What:** How often ledgers reach the trusted-validation quorum and become fully validated.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should match the build and close rates on a healthy node.\n**Healthy range:** About 0.25/s on mainnet.\n**Watch for:** A shortfall means ledgers are not accumulating enough trusted validations to be accepted.\n**Source:** src/xrpld/app/ledger/detail/LedgerMaster.cpp:LedgerMaster::checkAccept", "type": "stat", "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 0 + "h": 4, + "w": 12, + "x": 0, + "y": 13 }, "options": { "tooltip": { @@ -121,7 +142,11 @@ "unit": "suffix: ledgers/s" }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 4 }, { "title": "Ledger Build Duration Heatmap", @@ -130,8 +155,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 8 + "x": 0, + "y": 17 }, "options": { "tooltip": { @@ -162,7 +187,21 @@ } }, "overrides": [] - } + }, + "id": 5 + }, + { + "title": "Transaction Apply", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "collapsed": false, + "panels": [], + "id": 6 }, { "title": "Transaction Apply Duration", @@ -172,7 +211,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 26 }, "options": { "tooltip": { @@ -207,11 +246,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 7 }, { "title": "Transaction Apply Rate", @@ -221,7 +264,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 26 }, "options": { "tooltip": { @@ -248,21 +291,38 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 8 }, { - "title": "Ledger Store Rate", + "title": "Store & Close Timing", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "collapsed": false, + "panels": [], + "id": 9 + }, + { + "title": "Ledger Store Rate [$xrpl_network_type]", "description": "**What:** How often completed ledgers are written into the local ledger history.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should match the build rate under normal operation.\n**Healthy range:** About 0.25/s on mainnet.\n**Watch for:** A store rate below the build rate means ledgers are being built but not persisted.\n**Source:** src/xrpld/app/ledger/detail/LedgerMaster.cpp:LedgerMaster::storeLedger", "type": "stat", "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 0 + "h": 4, + "w": 12, + "x": 0, + "y": 35 }, "options": { "tooltip": { @@ -285,7 +345,11 @@ "unit": "suffix: ledgers/s" }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 10 }, { "title": "Build vs Close Duration", @@ -293,9 +357,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 24 + "y": 39 }, "options": { "tooltip": { @@ -330,11 +394,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 11 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/network-traffic.json b/docker/telemetry/grafana/dashboards/network-traffic.json index 724f05f652..5b2e0d809f 100644 --- a/docker/telemetry/grafana/dashboards/network-traffic.json +++ b/docker/telemetry/grafana/dashboards/network-traffic.json @@ -9,6 +9,19 @@ "id": null, "links": [], "panels": [ + { + "title": "Peer Connectivity", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [], + "id": 1 + }, { "title": "Active Peers", "description": "###### What this is:\n*Number of active inbound and outbound peer connections the node currently holds.*\n\n###### How it's computed:\n*Current value of the inbound and outbound active-peer counts per node.*\n\n###### Reading it:\n*Outbound is what the node dials out; inbound is what others open to it. Both should be stable.*\n\n###### Healthy range:\n*roughly 10-21 outbound and 0-85 inbound on mainnet, depending on config.*\n\n###### Watch for:\n*Outbound dropping toward zero (isolation) or inbound pinned at the limit with churn (connection pressure).*\n\n###### Source:\n[PeerfinderManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/peerfinder/detail/PeerfinderManager.cpp)\n\n###### Function:\n`Logic Stats ctor`", @@ -17,7 +30,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 1 }, "options": { "tooltip": { @@ -49,11 +62,16 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 - } + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + }, + "decimals": 0 }, "overrides": [] - } + }, + "id": 2 }, { "title": "Peer Disconnects", @@ -63,7 +81,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 1 }, "options": { "tooltip": { @@ -89,11 +107,16 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 - } + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + }, + "decimals": 0 }, "overrides": [] - } + }, + "id": 3 }, { "title": "Total Network Bytes", @@ -103,7 +126,7 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 9 }, "options": { "tooltip": { @@ -135,11 +158,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 4 }, { "title": "Total Network Messages", @@ -149,7 +176,7 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 9 }, "options": { "tooltip": { @@ -181,11 +208,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 5 }, { "title": "Transaction Traffic", @@ -195,7 +226,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 17 }, "options": { "tooltip": { @@ -233,11 +264,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 6 }, { "title": "Proposal Traffic", @@ -247,7 +282,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 17 }, "options": { "tooltip": { @@ -291,11 +326,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 7 }, { "title": "Validation Traffic", @@ -305,7 +344,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 25 }, "options": { "tooltip": { @@ -349,21 +388,38 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 8 }, { - "title": "Overlay Traffic by Category (Bytes In)", + "title": "Traffic by Category", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "collapsed": false, + "panels": [], + "id": 9 + }, + { + "title": "Overlay Traffic by Category (Bytes In) [$xrpl_network_type]", "description": "###### What this is:\n*Top overlay traffic categories ranked by inbound bytes, excluding the all-traffic total.*\n\n###### How it's computed:\n*Top categories by inbound byte rate per node.*\n\n###### Reading it:\n*Shows which message types dominate receive bandwidth right now.*\n\n###### Healthy range:\n*workload-dependent; transactions, proposals, and validations typically lead on a synced node.*\n\n###### Watch for:\n*A fetch or ledger-data category topping the list (sync activity) or an unexpected category dominating.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", "type": "bargauge", "gridPos": { - "h": 8, + "h": 4, "w": 12, - "x": 12, - "y": 0 + "x": 0, + "y": 34 }, "options": { "tooltip": { @@ -650,7 +706,11 @@ ] } ] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 10 }, { "title": "Duplicate Traffic (Wasted Bandwidth)", @@ -660,7 +720,7 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 38 }, "options": { "tooltip": { @@ -716,11 +776,28 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 11 + }, + { + "title": "Detail", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, + "collapsed": false, + "panels": [], + "id": 12 }, { "title": "All Traffic Categories (Detail)", @@ -729,8 +806,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 32 + "x": 0, + "y": 47 }, "options": { "tooltip": { @@ -756,11 +833,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 13 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index 44ea4cc5cb..f0a560c73a 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -10,11 +10,11 @@ "links": [], "panels": [ { - "title": "Validated Ledger Age", + "title": "Validated Ledger Age [$xrpl_network_type]", "description": "**What:** Seconds since the most recently validated ledger, i.e. how far behind the network the node is.\n**How it's computed:** Current value of the validated-ledger-age gauge per node.\n**Reading it:** Lower is better; a healthy node stays within a few ledger-close intervals.\n**Healthy range:** under ~10s (the network closes a ledger every 3-5s).\n**Watch for:** Above 20s or climbing steadily (the node is falling behind or has lost sync).\n**Source:** src/xrpld/app/ledger/LedgerMaster.h LedgerMaster::Stats ctor", "type": "stat", "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, "y": 0 @@ -55,14 +55,18 @@ } }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 1 }, { - "title": "Published Ledger Age", + "title": "Published Ledger Age [$xrpl_network_type]", "description": "**What:** Seconds since the most recently published ledger (the ledger exposed to clients and subscribers).\n**How it's computed:** Current value of the published-ledger-age gauge per node.\n**Reading it:** Should track validated-ledger age closely.\n**Healthy range:** under ~10s.\n**Watch for:** Published age growing while validated age stays low (publish-pipeline backlog).\n**Source:** src/xrpld/app/ledger/LedgerMaster.h LedgerMaster::Stats ctor", "type": "stat", "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 12, "y": 0 @@ -103,7 +107,11 @@ } }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 2 }, { "title": "Operating Mode (Time Share)", @@ -113,7 +121,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 4 }, "options": { "tooltip": { @@ -167,14 +175,18 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3, + "pointSize": 5, "stacking": { "mode": "normal" - } + }, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 3 }, { "title": "Operating Mode Transitions", @@ -184,7 +196,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 4 }, "options": { "tooltip": { @@ -238,11 +250,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 4 }, { "title": "I/O Latency", @@ -252,7 +268,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 12 }, "options": { "tooltip": { @@ -285,11 +301,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 5 }, { "title": "Job Queue Depth", @@ -299,7 +319,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 12 }, "options": { "tooltip": { @@ -325,21 +345,26 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 - } + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + }, + "decimals": 0 }, "overrides": [] - } + }, + "id": 6 }, { - "title": "Ledger Fetch Rate", + "title": "Ledger Fetch Rate [$xrpl_network_type]", "description": "**What:** Rate at which the node requests ledgers from peers.\n**How it's computed:** Per-second rate of the ledger-fetches counter over 5 minutes.\n**Reading it:** Near zero when synced; rises when catching up or backfilling history.\n**Healthy range:** workload-dependent; low on a synced node.\n**Watch for:** Sustained high fetch rate on a node that should be current (repeatedly missing ledgers).\n**Source:** src/xrpld/app/ledger/detail/InboundLedgers.cpp InboundLedgersImp ctor", "type": "stat", "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 8 + "y": 20 }, "options": { "tooltip": { @@ -361,17 +386,21 @@ "unit": "suffix: fetches/s" }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 7 }, { - "title": "Ledger History Mismatches", + "title": "Ledger History Mismatches [$xrpl_network_type]", "description": "**What:** Rate of ledger-history hash mismatches: built ledgers whose hash disagrees with the validated hash.\n**How it's computed:** Per-second rate of the history-mismatch counter over 5 minutes.\n**Reading it:** Should be flat at zero.\n**Healthy range:** zero.\n**Watch for:** Any non-zero value (consensus divergence, corrupted history, or database issues).\n**Source:** src/xrpld/telemetry/MetricsRegistry.cpp incrementLedgerHistoryMismatch (caller LedgerHistory.cpp)", "type": "stat", "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 12, - "y": 8 + "y": 20 }, "options": { "tooltip": { @@ -405,7 +434,11 @@ } }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 8 }, { "title": "--- Extended Metrics (Recovered from Phase 6) ---", @@ -414,10 +447,11 @@ "h": 1, "w": 24, "x": 0, - "y": 32 + "y": 24 }, "collapsed": false, - "panels": [] + "panels": [], + "id": 9 }, { "title": "Key Jobs Execution Time", @@ -427,7 +461,7 @@ "h": 8, "w": 12, "x": 0, - "y": 33 + "y": 25 }, "options": { "tooltip": { @@ -523,11 +557,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 10 }, { "title": "Key Jobs Dequeue Wait Time", @@ -537,7 +575,7 @@ "h": 8, "w": 12, "x": 12, - "y": 33 + "y": 25 }, "options": { "tooltip": { @@ -633,11 +671,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 11 }, { "title": "FullBelowCache Size", @@ -647,7 +689,7 @@ "h": 8, "w": 12, "x": 0, - "y": 41 + "y": 33 }, "options": { "tooltip": { @@ -673,20 +715,24 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 12 }, { - "title": "FullBelowCache Hit Rate", + "title": "FullBelowCache Hit Rate [$xrpl_network_type]", "description": "**What:** Hit-rate percentage of the FullBelowCache.\n**How it's computed:** Current value of the cache hit-rate gauge (0-100%).\n**Reading it:** Higher is better; means the node reuses knowledge of complete subtrees.\n**Healthy range:** above ~50% in steady state.\n**Watch for:** Low hit rate during steady state (cache too small or constant re-acquisition).\n**Source:** include/xrpl/basics/TaggedCache.h TaggedCache::Stats ctor", "type": "gauge", "gridPos": { - "h": 8, + "h": 4, "w": 12, - "x": 12, + "x": 0, "y": 41 }, "options": { @@ -727,17 +773,21 @@ } }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 13 }, { - "title": "Ledger Publish Gap", + "title": "Ledger Publish Gap [$xrpl_network_type]", "description": "**What:** Difference between published and validated ledger ages: how far the publish pipeline trails validation.\n**How it's computed:** Published-ledger age minus validated-ledger age, in seconds.\n**Reading it:** Near zero means publishing keeps up with validation.\n**Healthy range:** within a few seconds of zero.\n**Watch for:** A growing gap (publish backlog, stale data for subscribers).\n**Source:** src/xrpld/app/ledger/LedgerMaster.h LedgerMaster::Stats ctor", "type": "stat", "gridPos": { - "h": 8, + "h": 4, "w": 12, - "x": 0, - "y": 49 + "x": 12, + "y": 41 }, "options": { "tooltip": { @@ -775,7 +825,11 @@ } }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 14 }, { "title": "State Duration Rate (Full vs Tracking)", @@ -784,8 +838,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 49 + "x": 0, + "y": 45 }, "options": { "tooltip": { @@ -818,11 +872,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 15 }, { "title": "All Jobs Execution Time (Detail)", @@ -830,9 +888,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, - "x": 0, - "y": 57 + "w": 12, + "x": 12, + "y": 45 }, "options": { "tooltip": { @@ -858,11 +916,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 16 }, { "title": "All Jobs Dequeue Wait (Detail)", @@ -870,9 +932,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 65 + "y": 53 }, "options": { "tooltip": { @@ -898,11 +960,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 17 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json index 2b1ffc8ea1..458cc5e087 100644 --- a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json +++ b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json @@ -73,11 +73,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 1 }, { "title": "Overhead Traffic Breakdown (Bytes)", @@ -143,11 +147,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 2 }, { "title": "Validator List Traffic", @@ -201,7 +209,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [ @@ -222,7 +233,8 @@ ] } ] - } + }, + "id": 3 }, { "title": "Set Get/Share Traffic (Bytes)", @@ -276,11 +288,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 4 }, { "title": "Have/Requested Transactions (Messages)", @@ -334,11 +350,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 5 }, { "title": "Unknown / Unclassified Traffic", @@ -392,7 +412,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [ @@ -413,7 +436,8 @@ ] } ] - } + }, + "id": 6 }, { "title": "Proof Path Traffic", @@ -467,11 +491,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 7 }, { "title": "Replay Delta Traffic", @@ -525,11 +553,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 8 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/peer-network.json b/docker/telemetry/grafana/dashboards/peer-network.json index 74739c657b..279ab600e2 100644 --- a/docker/telemetry/grafana/dashboards/peer-network.json +++ b/docker/telemetry/grafana/dashboards/peer-network.json @@ -9,6 +9,19 @@ "id": null, "links": [], "panels": [ + { + "title": "Proposal & Validation Receive", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [], + "id": 1 + }, { "title": "Peer Proposal Receive Rate", "description": "**What:** Rate of consensus proposals arriving from connected network peers.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Reflects how many validators' proposals reach this node each round.\n**Healthy range:** Workload-dependent; scales with peer count and network activity.\n**Watch for:** A sudden drop means loss of proposal flow; a sharp sustained spike from a single peer can indicate flooding.\n**Source:** src/xrpld/overlay/detail/PeerImp.cpp:PeerImp::onMessage(TMProposeSet)", @@ -17,7 +30,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 1 }, "options": { "tooltip": { @@ -44,11 +57,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 2 }, { "title": "Peer Validation Receive Rate", @@ -58,7 +75,7 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 1 }, "options": { "tooltip": { @@ -85,11 +102,28 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 3 + }, + { + "title": "Trust Split", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "collapsed": false, + "panels": [], + "id": 4 }, { "title": "Proposals Trusted vs Untrusted", @@ -99,7 +133,7 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 10 }, "options": { "tooltip": { @@ -122,7 +156,8 @@ "unit": "short" }, "overrides": [] - } + }, + "id": 5 }, { "title": "Validations Trusted vs Untrusted", @@ -132,7 +167,7 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 10 }, "options": { "tooltip": { @@ -155,7 +190,8 @@ "unit": "short" }, "overrides": [] - } + }, + "id": 6 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json index bae50fa772..094f864531 100644 --- a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json +++ b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json @@ -10,11 +10,11 @@ "links": [], "panels": [ { - "title": "RPC Request Rate", + "title": "RPC Request Rate [$xrpl_network_type]", "description": "**What:** Rate of RPC requests handled, counting every HTTP and WebSocket call.\n**How it's computed:** Per-second rate of the RPC-requests counter over 5 minutes.\n**Reading it:** Tracks client demand on the node.\n**Healthy range:** workload-dependent.\n**Watch for:** Sudden sustained spikes (client surge or abusive polling) or a drop to zero (endpoint unavailable).\n**Source:** src/xrpld/rpc/detail/ServerHandler.cpp ServerHandler ctor", "type": "stat", "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, "y": 0 @@ -40,7 +40,24 @@ "unit": "reqps" }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 1 + }, + { + "title": "RPC Response", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "collapsed": false, + "panels": [], + "id": 2 }, { "title": "RPC Response Time", @@ -49,8 +66,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 0 + "x": 0, + "y": 5 }, "options": { "tooltip": { @@ -85,11 +102,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 3 }, { "title": "RPC Response Size", @@ -98,8 +119,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 24 + "x": 12, + "y": 5 }, "options": { "tooltip": { @@ -134,11 +155,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 4 }, { "title": "RPC Response Time Distribution", @@ -147,8 +172,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 24 + "x": 0, + "y": 13 }, "options": { "tooltip": { @@ -199,11 +224,28 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 5 + }, + { + "title": "Pathfinding", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "collapsed": false, + "panels": [], + "id": 6 }, { "title": "Pathfinding Fast Duration", @@ -213,7 +255,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 22 }, "options": { "tooltip": { @@ -248,11 +290,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 7 }, { "title": "Pathfinding Full Duration", @@ -262,7 +308,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 22 }, "options": { "tooltip": { @@ -297,21 +343,25 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 8 }, { - "title": "Resource Warnings Rate", + "title": "Resource Warnings Rate [$xrpl_network_type]", "description": "**What:** Rate of resource-limit warnings raised when a peer or client exceeds its usage warning threshold.\n**How it's computed:** Per-second rate of the warn counter over 5 minutes.\n**Reading it:** Occasional warnings are normal under load; a rising rate flags aggressive clients.\n**Healthy range:** workload-dependent; low.\n**Watch for:** A climbing warning rate (clients approaching limits, a precursor to drops).\n**Source:** include/xrpl/resource/detail/Logic.h Logic::Stats ctor", "type": "stat", "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 8 + "y": 30 }, "options": { "tooltip": { @@ -350,17 +400,21 @@ } }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 9 }, { - "title": "Resource Drops Rate", + "title": "Resource Drops Rate [$xrpl_network_type]", "description": "**What:** Rate of resource drops: consumers disconnected or blocked for excessive usage.\n**How it's computed:** Per-second rate of the drop counter over 5 minutes.\n**Reading it:** Non-zero means the node is actively rejecting abusive connections.\n**Healthy range:** at or near zero.\n**Watch for:** Sustained non-zero drops (ongoing abuse or a misbehaving client/peer being throttled).\n**Source:** include/xrpl/resource/detail/Logic.h Logic::Stats ctor", "type": "stat", "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 12, - "y": 8 + "y": 30 }, "options": { "tooltip": { @@ -399,7 +453,24 @@ } }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 10 + }, + { + "title": "gRPC (Spans)", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "collapsed": false, + "panels": [], + "id": 11 }, { "title": "gRPC Request Rate by Method (Spans)", @@ -409,7 +480,7 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 35 }, "options": { "tooltip": { @@ -436,11 +507,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 12 }, { "title": "gRPC Latency P95 by Method (Spans)", @@ -450,7 +525,7 @@ "h": 8, "w": 12, "x": 12, - "y": 32 + "y": 35 }, "options": { "tooltip": { @@ -477,11 +552,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 13 }, { "title": "gRPC Error Rate by Status (Spans)", @@ -491,7 +570,7 @@ "h": 8, "w": 12, "x": 0, - "y": 40 + "y": 43 }, "options": { "tooltip": { @@ -518,11 +597,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 14 }, { "title": "Pathfinding Compute Duration (Spans)", @@ -532,7 +615,7 @@ "h": 8, "w": 12, "x": 12, - "y": 40 + "y": 43 }, "options": { "tooltip": { @@ -567,11 +650,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 15 }, { "title": "Pathfinding Request & Discovery Rate (Spans)", @@ -579,9 +666,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 48 + "y": 51 }, "options": { "tooltip": { @@ -616,11 +703,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 16 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/rpc-performance.json b/docker/telemetry/grafana/dashboards/rpc-performance.json index 27be31f54d..f2f19ce578 100644 --- a/docker/telemetry/grafana/dashboards/rpc-performance.json +++ b/docker/telemetry/grafana/dashboards/rpc-performance.json @@ -16,7 +16,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 0 }, "options": { "tooltip": { @@ -43,11 +43,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 1 }, { "title": "RPC Latency P95 by Command", @@ -57,7 +61,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 0 }, "options": { "tooltip": { @@ -84,18 +88,22 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 2 }, { - "title": "RPC Error Rate", + "title": "RPC Error Rate [$xrpl_network_type]", "description": "**What:** Percentage of each RPC command's calls that ended in an error.\n**How it's computed:** Error calls divided by total calls per command over a 5-minute window, per node.\n**Reading it:** Green below 1%, yellow 1-5%, red above 5%.\n**Healthy range:** Below 1% for healthy commands.\n**Watch for:** A sustained error spike on one command is consistent with a client probing or misusing that endpoint.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", "type": "bargauge", "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, "y": 8 @@ -137,7 +145,11 @@ } }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 3 }, { "title": "RPC Latency Heatmap", @@ -146,8 +158,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 8 + "x": 0, + "y": 12 }, "options": { "tooltip": { @@ -178,7 +190,8 @@ "spanNulls": 1800000 } } - } + }, + "id": 4 }, { "title": "Overall RPC Throughput", @@ -187,8 +200,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 16 + "x": 12, + "y": 12 }, "options": { "tooltip": { @@ -223,11 +236,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 5 }, { "title": "RPC Success vs Error", @@ -236,8 +253,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 16 + "x": 0, + "y": 20 }, "options": { "tooltip": { @@ -272,21 +289,25 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 6 }, { - "title": "Top Commands by Volume", + "title": "Top Commands by Volume [$xrpl_network_type]", "description": "**What:** The ten most-called RPC commands by total invocations recently.\n**How it's computed:** Ranked total counts per command over a 5-minute window, per node.\n**Reading it:** Identifies the hottest API endpoints driving node load.\n**Healthy range:** Workload-dependent.\n**Watch for:** An unexpected command dominating the ranking can indicate scripted abuse or a misbehaving client.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", "type": "bargauge", "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 0 + "y": 28 }, "options": { "tooltip": { @@ -309,17 +330,21 @@ "unit": "none" }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 7 }, { - "title": "WebSocket Message Rate", + "title": "WebSocket Message Rate [$xrpl_network_type]", "description": "**What:** Rate of incoming WebSocket RPC messages the server processes.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Non-zero only when clients connect over WebSocket rather than HTTP.\n**Healthy range:** Zero is normal for HTTP-only deployments; otherwise workload-dependent.\n**Watch for:** An unexpected surge points to a chatty or abusive WebSocket client.\n**Source:** src/xrpld/rpc/detail/ServerHandler.cpp:ServerHandler::processSession", "type": "stat", "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 12, - "y": 0 + "y": 28 }, "options": { "tooltip": { @@ -342,7 +367,11 @@ "unit": "suffix: messages/s" }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 8 }, { "title": "RPC Resource Cost by Command", @@ -350,7 +379,7 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 12, + "w": 24, "x": 0, "y": 32 }, @@ -384,11 +413,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 9 }, { "title": "Batch vs Single RPC Requests", @@ -397,8 +430,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 32 + "x": 0, + "y": 40 }, "options": { "tooltip": { @@ -433,11 +466,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 10 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json index b6ece294c5..27e7254194 100644 --- a/docker/telemetry/grafana/dashboards/transaction-overview.json +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -51,11 +51,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 1 }, { "title": "Transaction Processing Latency by Type", @@ -63,9 +67,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 0 + "w": 24, + "x": 0, + "y": 8 }, "options": { "tooltip": { @@ -105,11 +109,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 2 }, { "title": "Transaction Path Distribution", @@ -119,7 +127,7 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 16 }, "options": { "tooltip": { @@ -142,7 +150,8 @@ "unit": "short" }, "overrides": [] - } + }, + "id": 3 }, { "title": "Transaction Receive vs Suppressed", @@ -152,7 +161,7 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 16 }, "options": { "tooltip": { @@ -179,11 +188,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 4 }, { "title": "Transaction Processing Duration Heatmap", @@ -193,7 +206,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 24 }, "options": { "tooltip": { @@ -224,7 +237,8 @@ "spanNulls": 1800000 } } - } + }, + "id": 5 }, { "title": "Transaction Apply Duration per Ledger", @@ -234,7 +248,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 24 }, "options": { "tooltip": { @@ -269,21 +283,25 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 6 }, { - "title": "Transaction Apply Failed Rate", + "title": "Transaction Apply Failed Rate [$xrpl_network_type]", "description": "**What:** Rate of transactor-stage applies that returned a non-success result while building a ledger.\n**How it's computed:** Per-second rate of tx.transactor spans at stage=apply whose ter_result is not tesSUCCESS, per node.\n**Reading it:** Green below 1/s, yellow 1-5/s, red above 5/s.\n**Healthy range:** A steady low non-zero rate is normal (e.g. tefPAST_SEQ, tecUNFUNDED and other benign conflicts).\n**Watch for:** A sustained jump above the normal baseline can indicate malformed transaction floods or a systemic issue.\n**Source:** src/libxrpl/tx/Transactor.cpp:Transactor::operator()", "type": "stat", "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 24 + "y": 32 }, "options": { "tooltip": { @@ -323,7 +341,24 @@ } }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 7 + }, + { + "title": "Throughput & Results", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "collapsed": false, + "panels": [], + "id": 8 }, { "title": "Transaction Rate by Type", @@ -331,9 +366,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 12, + "w": 24, "x": 0, - "y": 32 + "y": 37 }, "options": { "tooltip": { @@ -365,11 +400,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 9 }, { "title": "Transaction Results by Type", @@ -377,9 +416,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 32 + "w": 24, + "x": 0, + "y": 45 }, "options": { "tooltip": { @@ -411,21 +450,25 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 10 }, { - "title": "TxQ Accept: Applied Ratio per Node (State Timeline)", + "title": "TxQ Accept: Applied Ratio per Node (State Timeline) [$xrpl_network_type]", "description": "**What:** Applied fraction of TxQ accepts per node over time: the share of queued transactions that were included in a ledger versus removed on failure.\n**How it's computed:** Per node, applied accepts divided by applied-plus-failed accepts over a 5-minute window.\n**Reading it:** Green (>=90% applied) is a healthy drain; yellow is degraded; red means accepts are mostly failing.\n**Healthy range:** At or near 100% applied when the queue is draining healthily; workload-dependent.\n**Watch for:** A node dropping into yellow or red, which signals queue pressure, under-bidding, or fee escalation on that node.\n**Source:** src/xrpld/app/misc/detail/TxQ.cpp:TxQ::accept", "type": "state-timeline", "gridPos": { "h": 8, - "w": 8, + "w": 12, "x": 0, - "y": 40 + "y": 53 }, "options": { "mergeValues": true, @@ -476,7 +519,11 @@ } }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 11 }, { "title": "Transactor Duration by Type (p95)", @@ -484,9 +531,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 16, - "x": 8, - "y": 40 + "w": 24, + "x": 0, + "y": 61 }, "options": { "tooltip": { @@ -518,11 +565,28 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 12 + }, + { + "title": "Transaction Queue", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 69 + }, + "collapsed": false, + "panels": [], + "id": 13 }, { "title": "TxQ Enqueue Rate by Transaction Type", @@ -530,9 +594,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 48 + "y": 70 }, "options": { "tooltip": { @@ -559,11 +623,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 14 }, { "title": "Queue Accept (Drain) Duration per Ledger", @@ -572,8 +640,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 56 + "x": 12, + "y": 70 }, "options": { "tooltip": { @@ -608,11 +676,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 15 }, { "title": "Queue Cleanup Rate (Expired Entries)", @@ -621,8 +693,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 56 + "x": 0, + "y": 78 }, "options": { "tooltip": { @@ -649,11 +721,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 16 }, { "title": "Tx Apply Pipeline Rate by Stage", @@ -661,9 +737,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 12, + "w": 24, "x": 0, - "y": 64 + "y": 86 }, "options": { "tooltip": { @@ -695,11 +771,28 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 17 + }, + { + "title": "Apply Pipeline", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 94 + }, + "collapsed": false, + "panels": [], + "id": 18 }, { "title": "Tx Apply Pipeline Latency by Stage (p95)", @@ -707,9 +800,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 64 + "w": 24, + "x": 0, + "y": 95 }, "options": { "tooltip": { @@ -741,11 +834,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 19 }, { "title": "Tx Apply Pipeline Failure Rate by Stage", @@ -755,7 +852,7 @@ "h": 8, "w": 24, "x": 0, - "y": 80 + "y": 103 }, "options": { "tooltip": { @@ -787,11 +884,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 20 }, { "title": "Tx Apply Pipeline Latency by Type and Stage (p95)", @@ -801,7 +902,7 @@ "h": 8, "w": 24, "x": 0, - "y": 72 + "y": 111 }, "options": { "tooltip": { @@ -833,11 +934,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 21 } ], "schemaVersion": 39, From 92f9b88a217fc6d8fe6ac61f6c3d32ffa4048329 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:39:58 +0100 Subject: [PATCH 2/4] style(telemetry): uniform layout and stable panel ids for phase-9 dashboards Applies the same normalization the phase-7-owned dashboards received, to the five introduced on this branch: fee-market, job-queue, log-derived-insights, peer-quality and validator-health. - id written as 1..N so panelId deep links address a specific panel rather than whichever panel happens to sit in that position - gridPos at most two panels across; charts h=8, stats/gauges h=4, tables/logs h=12 full width. Panels pair only with an equal-height neighbour, so no row keeps a ragged empty cell. validator-health and peer-quality were stacked single-file at full width, which is why they were the longest to scroll. - line charts lineWidth=1, fillOpacity=0, pointSize=5, gradientMode=none - repeat xrpl_network_type (horizontal, maxPerRow=2) with a [$xrpl_network_type] title suffix on stat/gauge/bargauge/ table/state-timeline panels, where two networks overlaid in one panel reads as noise. Line charts keep their networks as separate series. - decimals 0 on panels counting discrete things (peers, quorum, connection mix) - a fractional peer count is meaningless. - rows category rows added to fee-market, job-queue and peer-quality Panels with a right-hand legend stay full width; a side legend needs the room. Raw-text edits, so untouched lines keep their formatting and escaping byte for byte. Verified per dashboard against origin/phase9: panel count unchanged, no panel lost, every targets block byte-identical, descriptions unchanged, ids exactly 1..N, and no two panels overlapping on the grid. --- .../grafana/dashboards/fee-market.json | 133 +++++-- .../grafana/dashboards/job-queue.json | 136 +++++-- .../dashboards/log-derived-insights.json | 354 ++++++++++-------- .../grafana/dashboards/peer-quality.json | 134 +++++-- .../grafana/dashboards/validator-health.json | 262 ++++++++----- 5 files changed, 663 insertions(+), 356 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/fee-market.json b/docker/telemetry/grafana/dashboards/fee-market.json index 8d1cc6bfd8..ab4b502f04 100644 --- a/docker/telemetry/grafana/dashboards/fee-market.json +++ b/docker/telemetry/grafana/dashboards/fee-market.json @@ -39,6 +39,19 @@ "id": null, "links": [], "panels": [ + { + "title": "Queue Depth & Throughput", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [], + "id": 1 + }, { "title": "Transaction Queue Depth", "description": "###### What this is:\n*Transactions currently waiting in the transaction queue versus the queue's maximum capacity.*\n\n###### How it's computed:\n*Instantaneous gauge readings of current queue count and configured max size.*\n\n###### Reading it:\n*Queue depth well below capacity is normal; depth approaching capacity means the node is saturating.*\n\n###### Healthy range:\n*Depth near 0 in quiet periods; workload-dependent under load.*\n\n###### Watch for:\n*Depth pinned at capacity for sustained periods, which signals demand exceeding throughput or a fee-spam burst.*\n\n###### Keywords:\n- **Transaction queue (TxQ)** *(per node)* \u2014 holds transactions that meet local cost but not the open-ledger cost, to include in a later ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerTxqGauge`\n\n###### References:\n[Transaction queue (TxQ)](https://xrpl.org/docs/concepts/transactions/transaction-queue) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-queue-txq)", @@ -47,7 +60,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 1 }, "options": { "tooltip": { @@ -79,16 +92,20 @@ "custom": { "axisLabel": "Transactions", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, - "spanNulls": 1800000 + "lineWidth": 1, + "fillOpacity": 0, + "spanNulls": 1800000, + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" - } + }, + "decimals": 0 }, "overrides": [] - } + }, + "id": 2 }, { "title": "Transactions Per Ledger", @@ -98,7 +115,7 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 1 }, "options": { "tooltip": { @@ -130,16 +147,32 @@ "custom": { "axisLabel": "Transactions", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, - "spanNulls": 1800000 + "lineWidth": 1, + "fillOpacity": 0, + "spanNulls": 1800000, + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" } }, "overrides": [] - } + }, + "id": 3 + }, + { + "title": "Fee Escalation", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "collapsed": false, + "panels": [], + "id": 4 }, { "title": "Fee Escalation Levels", @@ -147,9 +180,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 8 + "y": 10 }, "options": { "tooltip": { @@ -195,20 +228,23 @@ "custom": { "axisLabel": "Fee Level", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 5, + "lineWidth": 1, + "fillOpacity": 0, "scaleDistribution": { "type": "log", "log": 2 }, - "spanNulls": 1800000 + "spanNulls": 1800000, + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" } }, "overrides": [] - } + }, + "id": 5 }, { "title": "Load Factor Breakdown", @@ -217,8 +253,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 16 + "x": 12, + "y": 10 }, "options": { "tooltip": { @@ -264,9 +300,11 @@ "custom": { "axisLabel": "Multiplier", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 5, - "spanNulls": 1800000 + "lineWidth": 1, + "fillOpacity": 0, + "spanNulls": 1800000, + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -289,7 +327,8 @@ } }, "overrides": [] - } + }, + "id": 6 }, { "title": "Load Factor Components", @@ -298,8 +337,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 16 + "x": 0, + "y": 18 }, "options": { "tooltip": { @@ -338,16 +377,32 @@ "custom": { "axisLabel": "Multiplier", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 5, - "spanNulls": 1800000 + "lineWidth": 1, + "fillOpacity": 0, + "spanNulls": 1800000, + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" } }, "overrides": [] - } + }, + "id": 7 + }, + { + "title": "Queue Rejections", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "collapsed": false, + "panels": [], + "id": 8 }, { "title": "Queue Abandonment Rate (Expired)", @@ -357,7 +412,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 27 }, "options": { "tooltip": { @@ -384,11 +439,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 9 }, { "title": "Queue Admission Rejections (Dropped)", @@ -398,7 +457,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 27 }, "options": { "tooltip": { @@ -425,11 +484,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 10 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/job-queue.json b/docker/telemetry/grafana/dashboards/job-queue.json index c7e745e7e9..f509b39f60 100644 --- a/docker/telemetry/grafana/dashboards/job-queue.json +++ b/docker/telemetry/grafana/dashboards/job-queue.json @@ -10,14 +10,27 @@ "links": [], "panels": [ { - "title": "Current Job Latency (p99 Gauge)", + "title": "Current State", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [], + "id": 1 + }, + { + "title": "Current Job Latency (p99 Gauge) [$xrpl_network_type]", "description": "###### What this is:\n*At-a-glance p99 of how long jobs wait in the queue and how long they run once started.*\n\n###### How it's computed:\n*99th percentile derived from the job wait-time and run-time histograms over the last 5 minutes.*\n\n###### Reading it:\n*Lower is better; green under 100ms, yellow to 1s, red beyond 1s.*\n\n###### Healthy range:\n*Wait and exec p99 under 100ms.*\n\n###### Watch for:\n*p99 wait climbing into the red, meaning worker threads are saturated and jobs are backing up.*\n\n###### Keywords:\n- **Job queue / job type** *(per node)* \u2014 xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n- **Deferred job** *(per node)* \u2014 a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobStarted / recordJobFinished`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type)", "type": "gauge", "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 0 + "y": 1 }, "options": { "reduceOptions": { @@ -68,7 +81,23 @@ }, "overrides": [] }, - "id": 1 + "id": 2, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, + { + "title": "Throughput", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "collapsed": false, + "panels": [], + "id": 3 }, { "title": "Job Throughput Rate (Per Second)", @@ -76,9 +105,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 8 + "y": 6 }, "options": { "tooltip": { @@ -113,13 +142,14 @@ "unit": "suffix: jobs/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Jobs / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -127,7 +157,7 @@ }, "overrides": [] }, - "id": 2 + "id": 4 }, { "title": "Per-Job-Type Queued Rate", @@ -137,7 +167,7 @@ "h": 8, "w": 24, "x": 0, - "y": 16 + "y": 14 }, "options": { "tooltip": { @@ -166,12 +196,13 @@ "custom": { "drawStyle": "line", "lineWidth": 1, - "fillOpacity": 5, + "fillOpacity": 0, "axisLabel": "Jobs / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -179,7 +210,7 @@ }, "overrides": [] }, - "id": 3 + "id": 5 }, { "title": "Per-Job-Type Finish Rate", @@ -189,7 +220,7 @@ "h": 8, "w": 24, "x": 0, - "y": 24 + "y": 22 }, "options": { "tooltip": { @@ -218,12 +249,13 @@ "custom": { "drawStyle": "line", "lineWidth": 1, - "fillOpacity": 5, + "fillOpacity": 0, "axisLabel": "Jobs / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -231,7 +263,20 @@ }, "overrides": [] }, - "id": 4 + "id": 6 + }, + { + "title": "Latency", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "collapsed": false, + "panels": [], + "id": 7 }, { "title": "Job Queue Wait Time", @@ -241,7 +286,7 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 31 }, "options": { "tooltip": { @@ -270,13 +315,14 @@ "unit": "\u00b5s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 5, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Duration (\u00b5s)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -284,7 +330,7 @@ }, "overrides": [] }, - "id": 5 + "id": 8 }, { "title": "Job Execution Time", @@ -294,7 +340,7 @@ "h": 8, "w": 12, "x": 12, - "y": 32 + "y": 31 }, "options": { "tooltip": { @@ -323,13 +369,14 @@ "unit": "\u00b5s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 5, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Duration (\u00b5s)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -337,7 +384,7 @@ }, "overrides": [] }, - "id": 6 + "id": 9 }, { "title": "Per-Job-Type Execution Time (p99)", @@ -347,7 +394,7 @@ "h": 8, "w": 24, "x": 0, - "y": 40 + "y": 39 }, "options": { "tooltip": { @@ -376,12 +423,13 @@ "custom": { "drawStyle": "line", "lineWidth": 1, - "fillOpacity": 5, + "fillOpacity": 0, "axisLabel": "Duration (\u00b5s)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -389,7 +437,20 @@ }, "overrides": [] }, - "id": 7 + "id": 10 + }, + { + "title": "Overflow", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 47 + }, + "collapsed": false, + "panels": [], + "id": 11 }, { "title": "Transaction Overflow Rate", @@ -397,7 +458,7 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, "y": 48 }, @@ -439,12 +500,13 @@ "custom": { "axisLabel": "Overflows / Min", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -452,7 +514,7 @@ }, "overrides": [] }, - "id": 8 + "id": 12 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/log-derived-insights.json b/docker/telemetry/grafana/dashboards/log-derived-insights.json index 8f7aedfbd5..2a10be92a0 100644 --- a/docker/telemetry/grafana/dashboards/log-derived-insights.json +++ b/docker/telemetry/grafana/dashboards/log-derived-insights.json @@ -220,7 +220,7 @@ "type": "text", "title": "Read This First — Debug Log Requirement", "gridPos": { - "h": 10, + "h": 4, "w": 24, "x": 0, "y": 0 @@ -244,24 +244,24 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 4 }, "panels": [], "id": 2 }, { "type": "stat", - "title": "Nodes By Error Volume", + "title": "Nodes By Error Volume [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes are logging the most errors.*\n\n###### How it's computed:\n*Count of ERR and FTL log lines per node over the dashboard window, ranked highest first.*\n\n###### Reading it:\n*The top entry is the node in the most trouble. Compare nodes rather than reading an absolute value.*\n\n###### Healthy range:\n*Zero, or a small flat count. ERR is not routine.*\n\n###### Watch for:\n*Any node pulling far ahead of its peers, which usually means a fault local to that node rather than a network condition.*\n\n###### Keywords:\n- **ERR / FTL** *(per line)* — the two most severe xrpld log levels; both survive the default Info threshold.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[Log.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/basics/Log.cpp)\n\n###### Function:\n`Logs::Sink::write`\n\n###### References:\n[Loki log queries](https://grafana.com/docs/loki/latest/query/log_queries/)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 13 + "y": 5 }, "fieldConfig": { "defaults": { @@ -318,21 +318,24 @@ "legendFormat": "Errors [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 3 + "id": 3, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "stat", - "title": "Nodes By Attack-Like Input", + "title": "Nodes By Attack-Like Input [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes are receiving the most malformed or unwanted peer payloads.*\n\n###### How it's computed:\n*Count of Resource fee charges whose reason is useless data, unwanted data, or init drop, per node.*\n\n###### Reading it:\n*These three reasons indicate a peer sent something the node could not use, so a high count is the closest log-derived proxy for abusive input.*\n\n###### Healthy range:\n*Low and flat. Some useless data is normal on a busy overlay.*\n\n###### Watch for:\n*A sharp rise on one node, especially paired with a single dominant IP in the charged-peers table below.*\n\n###### Keywords:\n- **Useless data** *(per charge)* — payload the node could not use, e.g. an empty or malformed message.\n- **Init drop** *(per charge)* — a connection dropped during initialisation.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[ResourceManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/resource/detail/ResourceManager.cpp)\n\n###### Function:\n`Logic::charge`\n\n###### Note:\n*Requires debug logs on the Resource partition.*\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 12, - "y": 13 + "y": 5 }, "fieldConfig": { "defaults": { @@ -389,21 +392,24 @@ "legendFormat": "Charges [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 4 + "id": 4, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "stat", - "title": "Nodes By Total Fee Charged", + "title": "Nodes By Total Fee Charged [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes are absorbing the most peer-imposed load, weighted by fee amount.*\n\n###### How it's computed:\n*Sum of every resource fee amount parsed from the Resource partition, per node.*\n\n###### Reading it:\n*Weighted by cost rather than event count, so one heavy request at 2000 outranks eight moderate ones at 250.*\n\n###### Healthy range:\n*Proportional to peer count and request volume; compare nodes of similar role.*\n\n###### Watch for:\n*One node far above peers of the same role, which means it is carrying disproportionate peer load.*\n\n###### Keywords:\n- **Fee weight** *(per charge)* — the credit amount: 2000 heavy, 250 moderate, 150 useless data.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[ResourceManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/resource/detail/ResourceManager.cpp)\n\n###### Function:\n`Logic::charge`\n\n###### Note:\n*Requires debug logs on the Resource partition.*\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 21 + "y": 9 }, "fieldConfig": { "defaults": { @@ -460,21 +466,24 @@ "legendFormat": "Fee Charged [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 5 + "id": 5, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "stat", - "title": "Nodes By Manifest Rejection", + "title": "Nodes By Manifest Rejection [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes are rejecting the most inbound manifests.*\n\n###### How it's computed:\n*Count of ManifestCache lines with a Stale, Invalid, Revoked, or UntrustedCapacity outcome, per node.*\n\n###### Reading it:\n*A manifest flood shows up here first. Stale dominates normally because peers re-gossip manifests the node already holds.*\n\n###### Healthy range:\n*Workload-dependent; nodes on the same network should sit close together.*\n\n###### Watch for:\n*A single node far ahead, or any Invalid at all, which means a signature failed verification.*\n\n###### Keywords:\n- **Rejection** *(per manifest)* — any non-accepted disposition.\n- **UntrustedCapacity** *(per manifest)* — rate-limit rejection for an unlisted key; rc5+ builds only.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[Manifest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/server/Manifest.cpp)\n\n###### Function:\n`ManifestCache::applyManifest`\n\n###### Note:\n*Requires debug logs on the ManifestCache partition.*\n\n###### References:\n[Validator keys](https://xrpl.org/docs/concepts/consensus-protocol/validator-keys)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 12, - "y": 21 + "y": 9 }, "fieldConfig": { "defaults": { @@ -531,21 +540,24 @@ "legendFormat": "Rejections [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 6 + "id": 6, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "stat", - "title": "Nodes By Consensus Problem", + "title": "Nodes By Consensus Problem [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes are logging the most consensus warnings and errors.*\n\n###### How it's computed:\n*Count of LedgerConsensus lines at WRN severity or above, per node.*\n\n###### Reading it:\n*Consensus is a network-wide process, so a single node standing out points at that node rather than the network.*\n\n###### Healthy range:\n*Low. Some warnings occur during normal round churn.*\n\n###### Watch for:\n*One node far above its peers, or a step change after a deploy.*\n\n###### Keywords:\n- **Consensus round** *(per ledger)* — one Open to Establish to Accepted cycle.\n- **Dispute** *(per transaction)* — a transaction peers disagree about including.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[Consensus.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/consensus/Consensus.h)\n\n###### Function:\n`Consensus::timerEntry / checkConsensus`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 29 + "y": 13 }, "fieldConfig": { "defaults": { @@ -602,21 +614,24 @@ "legendFormat": "Consensus Events [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 7 + "id": 7, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "stat", - "title": "Nodes By Job Latency Breach", + "title": "Nodes By Job Latency Breach [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes are missing job latency targets most often.*\n\n###### How it's computed:\n*Count of LoadMonitor slow-job lines per node; the emitter only fires above a 500ms threshold.*\n\n###### Reading it:\n*A direct read on which node is most overloaded. Works at the default log level.*\n\n###### Healthy range:\n*Low. A busy node breaches occasionally.*\n\n###### Watch for:\n*A node far ahead of its peers, which usually means disk or CPU pressure local to it.*\n\n###### Keywords:\n- **Breach** *(per job)* — one job execution over the 500ms LoadMonitor threshold.\n\n###### Computation boundary:\n*Result: Per node — a count of BREACHES, never of total job executions.*\n*Derived in the Grafana query. Use the native `job_*` metrics for totals.*\n\n###### Source:\n[LoadMonitor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/LoadMonitor.cpp)\n\n###### Function:\n`LoadMonitor::addLoadSample`\n\n###### Note:\n*Counts breaches above 500ms only, so it is an exception count and not a latency measure.*\n\n###### References:\n[Job queue](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 12, - "y": 29 + "y": 13 }, "fieldConfig": { "defaults": { @@ -673,21 +688,24 @@ "legendFormat": "Breaches [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 8 + "id": 8, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "stat", - "title": "Nodes By Sync Instability", + "title": "Nodes By Sync Instability [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes are spending the most time out of the full state.*\n\n###### How it's computed:\n*Count of STATE-> transitions into any non-full state, per node.*\n\n###### Reading it:\n*A stable node holds full and emits nothing here, so any non-zero value means it left full that many times.*\n\n###### Healthy range:\n*Zero on a synced node.*\n\n###### Watch for:\n*A repeating count, which means the node is flapping rather than having had one bad moment.*\n\n###### Keywords:\n- **Operating mode** *(per node)* — Disconnected, Connected, Syncing, Tracking, Full.\n- **Flapping** *(per node)* — repeated departures from full.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::setMode`\n\n###### References:\n[Server states](https://xrpl.org/docs/concepts/networks-and-servers/rippled-server-states)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 37 + "y": 17 }, "fieldConfig": { "defaults": { @@ -744,21 +762,24 @@ "legendFormat": "Transitions [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 9 + "id": 9, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "stat", - "title": "Nodes By Duplicate Ledger Fetches", + "title": "Nodes By Duplicate Ledger Fetches [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes wasted the most ledger-fetch bandwidth, counted as duplicate SHAMap nodes received.*\n\n###### How it's computed:\n*Sum of the `dupe` counter from `Ledger AS/TX node stats` log lines, per node, over the dashboard window. `dupe` is matched by its own regex because the emitter omits any counter that is zero.*\n\n###### Reading it:\n*Every duplicate is a ledger node the peer sent that this node already held: bytes crossed the network, the peer spent resource credit serving it, and it was discarded.*\n\n###### Healthy range:\n*Workload-dependent. Compare nodes of the same role rather than reading an absolute value; some duplication is unavoidable when fetching one ledger from several peers.*\n\n###### Watch for:\n*A node far ahead of its siblings, which points at that node's fetch logic or peer set rather than a network condition.*\n\n###### Keywords:\n- **Ledger node** *(per fetch)* — one SHAMap node fetched while acquiring a ledger.\n- **Duplicate** *(per fetch)* — a node already held locally; wasted bandwidth and peer credit.\n\n###### Computation boundary:\n*Result: Per node — a COUNT of duplicates, not a ratio.*\n*NOT recorded as a metric. Derived in the Grafana query via `unwrap` over raw log text.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::onTimer`\n\n###### Note:\n*Requires debug logs on the InboundLedger partition. For the duplicate RATIO see \"Ledger Node Fetch Duplicate Ratio\" in the Ledger Acquisition Efficiency row — a stat panel cannot render a division, because Loki does not name binary-operation frames and Grafana falls back to \"Value #A\".*\n\n###### References:\n[Ledgers](https://xrpl.org/docs/concepts/ledgers)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 12, - "y": 37 + "y": 17 }, "fieldConfig": { "defaults": { @@ -815,7 +836,10 @@ "legendFormat": "Duplicates [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 10 + "id": 10, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "row", @@ -825,7 +849,7 @@ "h": 1, "w": 24, "x": 0, - "y": 45 + "y": 21 }, "panels": [], "id": 11 @@ -839,10 +863,10 @@ "uid": "${DS_LOKI}" }, "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 46 + "y": 22 }, "fieldConfig": { "defaults": { @@ -851,13 +875,14 @@ "min": 0, "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Transitions / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -892,17 +917,17 @@ }, { "type": "state-timeline", - "title": "Node State Timeline", + "title": "Node State Timeline [$xrpl_network_type]", "description": "###### What this is:\n*The node's operating state over time, as a discrete timeline.*\n\n###### How it's computed:\n*The most recent `STATE->` transition in each interval, rendered as a state band.*\n\n###### Reading it:\n*Shows at a glance how long the node spent in each state and exactly when it left full.*\n\n###### Healthy range:\n*One unbroken full band across the window.*\n\n###### Watch for:\n*Any band that is not full, and repeated narrow bands, which indicate state flapping.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* — the node's sync level: Disconnected, Connected, Syncing, Tracking, Full.\n\n###### Computation boundary:\n*Result: Per node — one band per server per state.*\n*NOT recorded as a metric anywhere. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::setMode`\n\n###### Note:\n*Only transitions are logged, so a node that never changes state produces no data here. Read with the transition-rate panel beside it.*\n\n###### References:\n[Server states](https://xrpl.org/docs/concepts/networks-and-servers/rippled-server-states)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 46 + "y": 22 }, "fieldConfig": { "defaults": { @@ -945,7 +970,10 @@ "expr": "sum by (state, service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | severity =~ \"$severity\" | regexp `STATE->(?P\\w+)` | state != `` | label_format state=`{{if eq .state \"full\"}}Full{{else if eq .state \"syncing\"}}Syncing{{else if eq .state \"tracking\"}}Tracking{{else if eq .state \"connected\"}}Connected{{else if eq .state \"disconnected\"}}Disconnected{{else}}{{.state}}{{end}}` [$__auto]))" } ], - "id": 13 + "id": 13, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "row", @@ -955,7 +983,7 @@ "h": 1, "w": 24, "x": 0, - "y": 56 + "y": 30 }, "panels": [], "id": 14 @@ -965,10 +993,10 @@ "title": "Log Line Rate By Severity", "description": "###### What this is:\n*Rate of log lines emitted by xrpld, split by severity.*\n\n###### How it's computed:\n*Per-second count of matching log lines grouped by the severity field parsed out of each line.*\n\n###### Reading it:\n*Use this to confirm the log pipeline is alive, and to see at a glance whether DBG lines are being collected at all.*\n\n###### Healthy range:\n*Workload-dependent. If the DBG series is absent, every panel in a [DBG] row on this dashboard will be empty.*\n\n###### Watch for:\n*A sudden collapse to only WRN and ERR, which means debug logging was turned off and the [DBG] rows have gone blind rather than quiet.*\n\n###### Keywords:\n- **Severity** *(per line)* — xrpld log level: DBG, NFO, WRN, ERR, FTL.\n- **Structured metadata** *(per line)* — Loki fields parsed from the line, filtered with `|` rather than in the stream selector.\n\n###### Computation boundary:\n*Result: Per node per severity — a count of log lines, not of events in the node.*\n*Derived in the Grafana query; the collector's filelog receiver parses severity, xrpld itself exports no such metric.*\n\n###### Source:\n[Log.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/basics/Log.cpp)\n\n###### Function:\n`Logs::Sink::write`\n\n###### References:\n[Loki structured metadata](https://grafana.com/docs/loki/latest/get-started/labels/structured-metadata/)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 57 + "y": 31 }, "fieldConfig": { "defaults": { @@ -976,13 +1004,14 @@ "unit": "suffix: lines/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Log Lines / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1024,10 +1053,10 @@ "title": "Log Line Rate By Partition (Top $topn)", "description": "###### What this is:\n*The busiest xrpld log partitions by line rate.*\n\n###### How it's computed:\n*Per-second count of log lines grouped by the partition field, limited to the top N series.*\n\n###### Reading it:\n*Shows which subsystem dominates log volume, which is the main cost driver for Loki ingest.*\n\n###### Healthy range:\n*Workload-dependent. Resource, JobQueue, and LedgerConsensus are normally the loudest.*\n\n###### Watch for:\n*A partition suddenly dominating, which usually means a subsystem entered a retry or error loop.*\n\n###### Keywords:\n- **Partition** *(per line)* — the xrpld subsystem that emitted the line, e.g. ManifestCache, Resource, LedgerConsensus.\n\n###### Computation boundary:\n*Result: Per node per partition — a count of log lines.*\n*Derived in the Grafana query; truncated to the top N by Loki's series limit.*\n\n###### Source:\n[Log.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/basics/Log.cpp)\n\n###### Function:\n`Logs::Sink::write`\n\n###### References:\n[Loki log queries](https://grafana.com/docs/loki/latest/query/log_queries/)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 57 + "y": 31 }, "fieldConfig": { "defaults": { @@ -1035,13 +1064,14 @@ "unit": "suffix: lines/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Log Lines / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1086,7 +1116,7 @@ "h": 1, "w": 24, "x": 0, - "y": 67 + "y": 39 }, "panels": [], "id": 17 @@ -1096,10 +1126,10 @@ "title": "Manifest Disposition Rate", "description": "###### What this is:\n*Rate of manifest apply outcomes: accepted, stale, revoked, invalid, or rate-limited.*\n\n###### How it's computed:\n*Per-second count of ManifestCache log lines, with the action parsed out of the `Manifest: ;Pk: ...` text.*\n\n###### Reading it:\n*AcceptedNew and AcceptedUpdate are useful work; Stale and UntrustedCapacity are rejections and normally dominate.*\n\n###### Healthy range:\n*Accepted rates are low and bursty. A high sustained Stale rate is normal — peers gossip manifests this node already holds.*\n\n###### Watch for:\n*Any Invalid, which means a signature failed verification, and a sustained UntrustedCapacity climb, which indicates a manifest flood from unlisted keys.*\n\n###### Keywords:\n- **Manifest** *(per validator)* — a signed record binding a validator's master key to its current signing key.\n- **Disposition** *(per manifest)* — the apply outcome: AcceptedNew, AcceptedUpdate, Stale, Revoked, Invalid, UntrustedCapacity.\n- **Stale** *(per manifest)* — sequence number not greater than the one already held; the common benign rejection.\n\n###### Computation boundary:\n*Result: Per node per action — counts log EVENTS, not distinct manifests.*\n*NOT recorded as a metric. `applyManifest` has no instrumentation; this is regex over `logMftAct` output in the Grafana query.*\n\n###### Source:\n[Manifest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/server/Manifest.cpp)\n\n###### Function:\n`ManifestCache::applyManifest / logMftAct`\n\n###### References:\n[Validator keys](https://xrpl.org/docs/concepts/consensus-protocol/validator-keys)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 68 + "y": 40 }, "fieldConfig": { "defaults": { @@ -1107,13 +1137,14 @@ "unit": "suffix: manifests/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Manifests / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1155,23 +1186,24 @@ "title": "Manifest Accept Vs Reject Rate", "description": "###### What this is:\n*Manifest outcomes collapsed into two series: accepted versus rejected.*\n\n###### How it's computed:\n*Per-second counts of ManifestCache lines, with AcceptedNew and AcceptedUpdate summed as accepted and all other actions summed as rejected.*\n\n###### Reading it:\n*Gives the useful-work fraction of inbound manifest processing without per-action detail.*\n\n###### Healthy range:\n*Rejected normally far exceeds accepted; peers routinely re-gossip known manifests.*\n\n###### Watch for:\n*A rejected rate in the hundreds per second, which indicates a manifest flood consuming the JtManifest job queue.*\n\n###### Keywords:\n- **Accepted** *(per manifest)* — AcceptedNew or AcceptedUpdate; the manifest changed cache state.\n- **Rejected** *(per manifest)* — Stale, Invalid, Revoked, or UntrustedCapacity.\n\n###### Computation boundary:\n*Result: Per node — two summed series over the same log lines.*\n*Derived in the Grafana query. Accepted may legitimately be absent: measured 1,360 AcceptedNew per 7d, so short windows often contain none. Both series carry `or vector(0)` so a zero-accept window renders a flat zero line rather than disappearing.*\n\n###### Source:\n[Manifest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/server/Manifest.cpp)\n\n###### Function:\n`ManifestCache::applyManifest / logMftAct`\n\n###### References:\n[Validator keys](https://xrpl.org/docs/concepts/consensus-protocol/validator-keys)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 68 + "y": 40 }, "fieldConfig": { "defaults": { "unit": "suffix: manifests/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Manifests / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1220,13 +1252,13 @@ }, { "type": "table", - "title": "Top $topn Manifest Producers By Master Key", + "title": "Top $topn Manifest Producers By Master Key [$xrpl_network_type]", "description": "###### What this is:\n*The master keys responsible for the most manifest log events in the selected window.*\n\n###### How it's computed:\n*Count of ManifestCache lines grouped by the base58 master key parsed from the log text, limited to the top N.\nCounts are per log EVENT, not per distinct manifest.*\n\n###### Reading it:\n*Identifies which validator keys generate the most manifest churn on this node.*\n\n###### Healthy range:\n*A small number of keys with modest repeat counts. Live observation shows roughly 19 to 24 events per key per hour.*\n\n###### Watch for:\n*A single unlisted key with a very high count, which is the signature of a targeted manifest flood.*\n\n###### Keywords:\n- **Master key** *(per validator)* — the long-lived base58 key a manifest is signed under.\n- **Log event** *(per line)* — one manifest apply attempt; the same key recurs as peers re-gossip.\n\n###### Computation boundary:\n*Result: Per master key per node — counts log EVENTS, NOT distinct manifests and NOT distinct keys.*\n*Derived in the Grafana query. Truncated to top N: Loki rejects queries returning over 2000 series, so a true distinct-key count is impossible here.*\n\n###### Source:\n[Manifest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/server/Manifest.cpp)\n\n###### Function:\n`logMftAct`\n\n###### Note:\n*Truncated to the top N. Loki rejects a query returning over 2000 series, so a full distinct-key count is not possible here.*\n\n###### References:\n[Validator keys](https://xrpl.org/docs/concepts/consensus-protocol/validator-keys)", "gridPos": { - "h": 14, + "h": 12, "w": 24, "x": 0, - "y": 78 + "y": 48 }, "fieldConfig": { "defaults": { @@ -1353,7 +1385,10 @@ "datasource": { "type": "loki", "uid": "${DS_LOKI}" - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "row", @@ -1363,7 +1398,7 @@ "h": 1, "w": 24, "x": 0, - "y": 92 + "y": 60 }, "panels": [], "id": 21 @@ -1373,10 +1408,10 @@ "title": "Fee Charge Rate By Reason", "description": "###### What this is:\n*Rate at which this node charges peers a resource fee, split by the reason for the charge.*\n\n###### How it's computed:\n*Per-second count of Resource log lines, with the reason parsed from the `for ($)` text.*\n\n###### Reading it:\n*Shows what kind of peer behaviour is costing this node the most. Moderate peer request normally dominates.*\n\n###### Healthy range:\n*Workload-dependent and roughly proportional to peer count and request volume.*\n\n###### Watch for:\n*A rising useless data rate, which means peers are sending malformed or unwanted payloads.*\n\n###### Keywords:\n- **Resource fee** *(per peer)* — an internal load credit charged against a peer, unrelated to XRP transaction fees.\n- **Reason** *(per charge)* — why the charge was applied, e.g. moderate peer request, useless data.\n\n###### Computation boundary:\n*Result: Per node per reason — a count of charge events.*\n*NOT recorded as a metric. Derived in the Grafana query from `Logic::charge` log output.*\n\n###### Source:\n[ResourceManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/resource/detail/ResourceManager.cpp)\n\n###### Function:\n`Logic::charge`\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 93 + "y": 61 }, "fieldConfig": { "defaults": { @@ -1384,13 +1419,14 @@ "unit": "suffix: charges/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Charges / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1432,10 +1468,10 @@ "title": "Fee-Weighted Charge Load", "description": "###### What this is:\n*Resource fee charges weighted by the fee amount, rather than counted equally.*\n\n###### How it's computed:\n*Per-second sum of the fee value parsed from each Resource charge line, using LogQL unwrap.*\n\n###### Reading it:\n*A single heavy peer request at 2000 outweighs eight moderate ones at 250, so this ranks real cost rather than event count.*\n\n###### Healthy range:\n*Workload-dependent; should track peer request volume smoothly.*\n\n###### Watch for:\n*Spikes that do not appear in the charge-count panel, which mean a shift toward expensive request types.*\n\n###### Keywords:\n- **Fee weight** *(per charge)* — the credit amount, e.g. 250 moderate, 2000 heavy; higher means costlier.\n\n###### Computation boundary:\n*Result: Per node per reason — a SUM of fee amounts, not a count of events.*\n*Derived in the Grafana query via LogQL `unwrap` over the parsed fee value.*\n\n###### Source:\n[ResourceManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/resource/detail/ResourceManager.cpp)\n\n###### Function:\n`Logic::charge`\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 93 + "y": 61 }, "fieldConfig": { "defaults": { @@ -1443,13 +1479,14 @@ "unit": "suffix: fee/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Fee Units / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1488,13 +1525,13 @@ }, { "type": "table", - "title": "Top $topn Charged Peers By IP And Public Key", + "title": "Top $topn Charged Peers By IP And Public Key [$xrpl_network_type]", "description": "###### What this is:\n*The peers accumulating the most resource fee charges, identified by IP address and node public key.*\n\n###### How it's computed:\n*Sum of fee amounts parsed from Resource charge lines, grouped by IP and public key, limited to the top N.*\n\n###### Reading it:\n*This is the direct answer to which peer is costing this node the most, and is the primary abuse-attribution view.*\n\n###### Healthy range:\n*Charges spread across many peers with no single dominant entry.*\n\n###### Watch for:\n*One IP far above the rest, especially paired with the useless data reason, which indicates an abusive or misbehaving peer.*\n\n###### Keywords:\n- **Node public key** *(per peer)* — the peer's base58 identity, stable across reconnects.\n- **IP address** *(per peer)* — source address; the optional `:port` suffix is stripped so one peer is one row.\n\n###### Computation boundary:\n*Result: Per peer per node — a SUM of fee amounts over the dashboard window.*\n*Derived in the Grafana query. Truncated to top N by Loki's 2000-series limit, so this is the head of the distribution, never a total.*\n\n###### Source:\n[ResourceManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/resource/detail/ResourceManager.cpp)\n\n###### Function:\n`Logic::charge`\n\n###### Note:\n*Truncated to the top N because of Loki's 2000-series query limit.*\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", "gridPos": { - "h": 14, + "h": 12, "w": 24, "x": 0, - "y": 103 + "y": 69 }, "fieldConfig": { "defaults": { @@ -1633,7 +1670,10 @@ "datasource": { "type": "loki", "uid": "${DS_LOKI}" - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "row", @@ -1643,7 +1683,7 @@ "h": 1, "w": 24, "x": 0, - "y": 117 + "y": 81 }, "panels": [], "id": 25 @@ -1653,10 +1693,10 @@ "title": "Ledger Node Fetch Duplicate Ratio", "description": "###### What this is:\n*The fraction of fetched ledger nodes that were duplicates this node already held.*\n\n###### How it's computed:\n*Sum of the dupe counter divided by the sum of good plus dupe, parsed from the `Ledger AS/TX node stats` log lines. Each field is matched independently because the emitter omits any counter that is zero.*\n\n###### Reading it:\n*This is wasted ledger-fetch bandwidth. A ratio of 0.5 means half of everything fetched was already present.*\n\n###### Healthy range:\n*Below roughly 0.3. Some duplication is unavoidable when fetching from several peers at once.*\n\n###### Watch for:\n*Sustained values above 0.8. Live observation has recorded good:142 dupe:891, an 86 percent duplicate rate, which wastes both bandwidth and peer resource credit.*\n\n###### Keywords:\n- **Ledger node** *(per fetch)* — one SHAMap node fetched while acquiring a ledger.\n- **Duplicate** *(per fetch)* — a node already held locally; wasted bandwidth and peer credit.\n\n###### Computation boundary:\n*Result: Per node — a ratio of two summed counters, dupe / (good + dupe).*\n*NOT recorded as a metric. Derived in the Grafana query via `unwrap` over the counters in the InboundLedger stats line.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::onTimer / takeAsRootNode`\n\n###### References:\n[Ledgers](https://xrpl.org/docs/concepts/ledgers)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 118 + "y": 82 }, "fieldConfig": { "defaults": { @@ -1665,13 +1705,14 @@ "max": 1, "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Duplicate Ratio", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1731,23 +1772,24 @@ "title": "Ledger Node Fetch Rate — Good Vs Duplicate Vs Timeout", "description": "###### What this is:\n*Absolute rate of useful ledger nodes fetched, duplicates received, and acquire timeouts.*\n\n###### How it's computed:\n*Per-second sums of the good, dupe, and timeouts counters parsed from InboundLedger acquire log lines.*\n\n###### Reading it:\n*Gives the absolute volumes behind the duplicate-ratio panel, so a high ratio at trivial volume can be told apart from a high ratio at scale.*\n\n###### Healthy range:\n*Good should exceed duplicate during active sync. Timeouts should stay near zero.*\n\n###### Watch for:\n*A rising timeout series, which means peers are not answering ledger requests and sync will stall.*\n\n###### Keywords:\n- **Good** *(per fetch)* — a useful, previously unheld ledger node.\n- **Timeout** *(per acquire)* — a ledger request a peer never answered.\n\n###### Computation boundary:\n*Result: Per node — SUMS of the parsed counters, giving absolute volumes behind the ratio panel.*\n*Derived in the Grafana query via `unwrap`.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::onTimer`\n\n###### References:\n[Ledgers](https://xrpl.org/docs/concepts/ledgers)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 118 + "y": 82 }, "fieldConfig": { "defaults": { "unit": "suffix: nodes/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Ledger Nodes / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1811,7 +1853,7 @@ "h": 1, "w": 24, "x": 0, - "y": 128 + "y": 90 }, "panels": [], "id": 28 @@ -1821,10 +1863,10 @@ "title": "Peer Disconnect Rate By Reason", "description": "###### What this is:\n*Rate of peer connection endings, split by the reason recorded in the log.*\n\n###### How it's computed:\n*Per-second count of Peer log lines matching Timeout, Closed, or a refused connection attempt.*\n\n###### Reading it:\n*Distinguishes clean teardown from failure. Closed is a normal ending; Timeout and Connection refused are not.*\n\n###### Healthy range:\n*Closed dominant with a low, steady background of the others.*\n\n###### Watch for:\n*A Timeout rate approaching the Closed rate, which points at network trouble or unresponsive peers.*\n\n###### Keywords:\n- **Closed** *(per peer)* — a clean connection teardown; the normal ending.\n- **Timeout** *(per peer)* — the peer stopped responding.\n- **Connection refused** *(per attempt)* — an outbound attempt the remote rejected.\n\n###### Computation boundary:\n*Result: Per node per outcome — a count of peer lifecycle events.*\n*Derived in the Grafana query. Note `overlay_peer_disconnects` exists as a metric but carries no reason breakdown, which is what this panel adds.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::close / onTimer`\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 129 + "y": 91 }, "fieldConfig": { "defaults": { @@ -1832,13 +1874,14 @@ "unit": "suffix: events/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Peer Events / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1880,23 +1923,24 @@ "title": "Peer Handshake And Accept Rate", "description": "###### What this is:\n*Rate of completed peer handshakes and accepted inbound connections.*\n\n###### How it's computed:\n*Per-second count of PeerFinder log lines matching handshake and accept events.*\n\n###### Reading it:\n*Read together with the disconnect panel: healthy churn has handshakes roughly balancing disconnects.*\n\n###### Healthy range:\n*Low and steady once the node has a full peer slate.*\n\n###### Watch for:\n*A high handshake rate alongside a high disconnect rate, which means peers connect and immediately drop.*\n\n###### Keywords:\n- **Handshake** *(per peer)* — protocol negotiation completed with a peer.\n- **Accept** *(per peer)* — an inbound connection admitted to a peer slot.\n\n###### Computation boundary:\n*Result: Per node — counts of PeerFinder events.*\n*Derived in the Grafana query.*\n\n###### Source:\n[Logic.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/peerfinder/detail/Logic.h)\n\n###### Function:\n`Logic::on_handshake / on_accept`\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 129 + "y": 91 }, "fieldConfig": { "defaults": { "unit": "suffix: events/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Peer Events / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1951,7 +1995,7 @@ "h": 1, "w": 24, "x": 0, - "y": 139 + "y": 99 }, "panels": [], "id": 31 @@ -1961,10 +2005,10 @@ "title": "Consensus Phase Transition Rate", "description": "###### What this is:\n*Rate of consensus phase transitions into Open, Establish, and Accepted.*\n\n###### How it's computed:\n*Per-second count of log lines announcing a ConsensusPhase transition, grouped by the target phase.*\n\n###### Reading it:\n*A healthy node cycles Open to Establish to Accepted once per ledger, so all three series should track together.*\n\n###### Healthy range:\n*Roughly one transition per phase per ledger interval, near 0.25 per second on a 4-second close.*\n\n###### Watch for:\n*Establish transitions outnumbering Accepted, which means rounds start but fail to converge.*\n\n###### Keywords:\n- **Consensus phase** *(per round)* — Open collects transactions, Establish converges on a set, Accepted applies it.\n- **Transition** *(per round)* — one phase change; a healthy node cycles all three once per ledger.\n\n###### Computation boundary:\n*Result: Per node per phase — a count of transitions.*\n*Derived in the Grafana query by regex over the phase-change log line.*\n\n###### Source:\n[Consensus.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/consensus/Consensus.h)\n\n###### Function:\n`Consensus::phase transition logging`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 140 + "y": 100 }, "fieldConfig": { "defaults": { @@ -1972,13 +2016,14 @@ "unit": "suffix: transitions/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Transitions / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -2020,10 +2065,10 @@ "title": "Consensus Operating Mode Rate", "description": "###### What this is:\n*Rate of consensus heartbeat log lines, split by the operating mode reported in each.*\n\n###### How it's computed:\n*Per-second count of ConsensusLogger heartbeat lines, with the mode parsed from the `mode: ` text.*\n\n###### Reading it:\n*This acts as a mode-occupancy proxy: the heartbeat fires about once per second, so the dominant series is the node's current mode.*\n\n###### Healthy range:\n*Almost entirely the full series on a synced node.*\n\n###### Watch for:\n*Any sustained syncing, observing, or tracking share, which means the node is not participating normally.*\n\n###### Keywords:\n- **Consensus mode** *(per node)* — the node's participation level: full proposing, observing, syncing, tracking.\n- **Heartbeat** *(per second)* — the ConsensusLogger timer line, emitted roughly once per second.\n\n###### Computation boundary:\n*Result: Per node per mode — a count of heartbeat lines, used as a mode-occupancy PROXY rather than a true duration.*\n*Derived in the Grafana query. For exact durations use the state_accounting metrics on the Node Health dashboard.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`ConsensusLogger heartbeat`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 140 + "y": 100 }, "fieldConfig": { "defaults": { @@ -2031,13 +2076,14 @@ "unit": "suffix: beats/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Heartbeats / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -2079,10 +2125,10 @@ "title": "Validator List Quorum And Trusted Set Size", "description": "###### What this is:\n*The quorum threshold and trusted validator count this node computed, as recorded in the log.*\n\n###### How it's computed:\n*Quorum and total are parsed from the `Using quorum of N for new set of M trusted validators` line and plotted as values.*\n\n###### Reading it:\n*Confirms the node agrees with the network on how many validators it trusts and how many must agree.*\n\n###### Healthy range:\n*Stable, with quorum at roughly 80 percent of the trusted total.*\n\n###### Watch for:\n*A drop in the trusted total, which shrinks the quorum and weakens the node's safety margin.*\n\n###### Keywords:\n- **Quorum** *(per node)* — how many trusted validators must agree; normally about 80 percent of the trusted set.\n- **Trusted set** *(per node)* — validators from the UNL this node currently trusts.\n\n###### Computation boundary:\n*Result: Per node — the last VALUE parsed from the log line, not a rate or count.*\n*Derived in the Grafana query via `unwrap`. Overlaps the `unl_quorum` metric; prefer that metric for the value and this panel for churn events.*\n\n###### Source:\n[ValidatorList.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/ValidatorList.cpp)\n\n###### Function:\n`ValidatorList::updateTrusted`\n\n###### References:\n[Unique Node List](https://xrpl.org/docs/concepts/consensus-protocol/unique-node-list)", "gridPos": { - "h": 10, - "w": 24, + "h": 8, + "w": 12, "x": 0, - "y": 150 + "y": 108 }, "fieldConfig": { "defaults": { @@ -2090,17 +2136,19 @@ "min": 0, "custom": { "drawStyle": "line", - "lineWidth": 2, + "lineWidth": 1, "fillOpacity": 0, "axisLabel": "Validators", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" - } + }, + "decimals": 0 }, "overrides": [] }, @@ -2151,7 +2199,7 @@ "h": 1, "w": 24, "x": 0, - "y": 160 + "y": 116 }, "panels": [], "id": 35 @@ -2161,10 +2209,10 @@ "title": "Slow Job Run Time (p99)", "description": "###### What this is:\n*The 99th percentile run time of jobs that breached their latency target.*\n\n###### How it's computed:\n*Quantile over the run millisecond value parsed from LoadMonitor job lines, grouped by job name.*\n\n###### Reading it:\n*Identifies which job types are the slowest when they do run long.*\n\n###### Healthy range:\n*Only a handful of job names should appear at all. Values in the tens of seconds indicate real stalls.*\n\n###### Watch for:\n*ProcessLData or InboundLedger dominating, which is the signature of the cold-read sync bottleneck.*\n\n###### Keywords:\n- **Job** *(per task)* — a unit of work on xrpld's job queue, e.g. ProcessLData, InboundLedger.\n- **Run time** *(per job)* — time spent executing, excluding queue wait.\n- **Breach** *(per job)* — an execution exceeding the 500ms LoadMonitor threshold; only these are logged.\n\n###### Computation boundary:\n*Result: Per node per job type — a p99 over BREACHES ONLY, never over all executions.*\n*Derived in the Grafana query via `unwrap`. For total job counts and latencies use the native `job_*` metrics on the Job Queue Analysis dashboard.*\n\n###### Source:\n[LoadMonitor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/LoadMonitor.cpp)\n\n###### Function:\n`LoadMonitor::addLoadSample`\n\n###### Note:\n*These lines are only emitted above a 500ms latency threshold, so this panel counts BREACHES, never total job executions. Use the Job Queue Analysis dashboard for totals.*\n\n###### References:\n[Job queue](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 161 + "y": 117 }, "fieldConfig": { "defaults": { @@ -2173,13 +2221,14 @@ "min": 0, "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Run Time (ms)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -2221,10 +2270,10 @@ "title": "Slow Job Breach Rate By Job Name (Top $topn)", "description": "###### What this is:\n*How often each job type breaches its latency target.*\n\n###### How it's computed:\n*Per-second count of LoadMonitor job lines grouped by job name, limited to the top N.*\n\n###### Reading it:\n*Read with the p99 panel: a job can breach rarely but severely, or often but mildly.*\n\n###### Healthy range:\n*Near zero. Any sustained non-zero rate means a job type is routinely missing its target.*\n\n###### Watch for:\n*A step change after a deploy, which points at a regression in that job's path.*\n\n###### Keywords:\n- **Breach** *(per job)* — an execution over the 500ms threshold.\n- **Job type** *(per task)* — the named job class, e.g. sweep, ProcessLData.\n\n###### Computation boundary:\n*Result: Per node per job type — a count of BREACHES, not of executions.*\n*Derived in the Grafana query; truncated to top N.*\n\n###### Source:\n[LoadMonitor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/LoadMonitor.cpp)\n\n###### Function:\n`LoadMonitor::addLoadSample`\n\n###### Note:\n*Counts BREACHES above the 500ms threshold, not total job executions.*\n\n###### References:\n[Job queue](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 161 + "y": 117 }, "fieldConfig": { "defaults": { @@ -2232,13 +2281,14 @@ "unit": "suffix: breaches/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Breaches / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -2280,10 +2330,10 @@ "title": "Slow Job Queue Wait Time (p99)", "description": "###### What this is:\n*The 99th percentile queue wait time of jobs that breached their latency target.*\n\n###### How it's computed:\n*Quantile over the wait millisecond value parsed from LoadMonitor job lines, grouped by job name.*\n\n###### Reading it:\n*Separates queueing delay from execution cost. High wait with low run means the job queue is saturated rather than the work being slow.*\n\n###### Healthy range:\n*Near zero. Live observation shows wait at 0ms for most breaches, meaning run time is the cause.*\n\n###### Watch for:\n*Wait time rising above run time, which shifts the diagnosis from slow work to a saturated job queue.*\n\n###### Keywords:\n- **Wait time** *(per job)* — time queued before execution began.\n- **Run versus wait** *(per job)* — high wait means a saturated queue; high run means slow work.\n\n###### Computation boundary:\n*Result: Per node per job type — a p99 over BREACHES ONLY.*\n*Derived in the Grafana query via `unwrap`.*\n\n###### Source:\n[LoadMonitor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/LoadMonitor.cpp)\n\n###### Function:\n`LoadMonitor::addLoadSample`\n\n###### Note:\n*Counts BREACHES above the 500ms threshold, not total job executions.*\n\n###### References:\n[Job queue](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)", "gridPos": { - "h": 10, - "w": 24, + "h": 8, + "w": 12, "x": 0, - "y": 171 + "y": 125 }, "fieldConfig": { "defaults": { @@ -2292,13 +2342,14 @@ "min": 0, "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Queue Wait Time (ms)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -2343,7 +2394,7 @@ "h": 1, "w": 24, "x": 0, - "y": 181 + "y": 133 }, "panels": [], "id": 39 @@ -2353,10 +2404,10 @@ "title": "Warning And Error Rate By Partition (Top $topn)", "description": "###### What this is:\n*Rate of WRN, ERR, and FTL log lines by partition.*\n\n###### How it's computed:\n*Per-second count of log lines at warning severity or above, grouped by partition and limited to the top N.*\n\n###### Reading it:\n*This is the one row that works fully at default log level, so it is the first place to look on an unmodified node.*\n\n###### Healthy range:\n*Low and flat. LoadMonitor warnings are expected on a busy node.*\n\n###### Watch for:\n*Any ERR or FTL series appearing, and step changes in a partition that is normally quiet.*\n\n###### Keywords:\n- **Severity** *(per line)* — WRN, ERR, or FTL; the levels a default-configured node still writes.\n- **Partition** *(per line)* — the emitting xrpld subsystem.\n\n###### Computation boundary:\n*Result: Per node per partition per severity — a count of log lines.*\n*Derived in the Grafana query. This row is the only one that works fully at default log level.*\n\n###### Source:\n[Log.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/basics/Log.cpp)\n\n###### Function:\n`Logs::Sink::write`\n\n###### References:\n[Loki log queries](https://grafana.com/docs/loki/latest/query/log_queries/)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 182 + "y": 134 }, "fieldConfig": { "defaults": { @@ -2364,13 +2415,14 @@ "unit": "suffix: lines/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Log Lines / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -2431,10 +2483,10 @@ "title": "Recent Warnings And Errors", "description": "###### What this is:\n*The most recent log lines at warning severity or above.*\n\n###### How it's computed:\n*Raw log stream filtered to WRN, ERR, and FTL, newest first.*\n\n###### Reading it:\n*Use this to read the actual text behind a spike in the rate panel beside it.*\n\n###### Healthy range:\n*Sparse. LoadMonitor slow-job warnings are the common benign entry.*\n\n###### Watch for:\n*Repeating identical errors, which indicate a stuck retry loop rather than a transient fault.*\n\n###### Keywords:\n- **Log stream** *(per node)* — the raw lines behind the aggregate rates, newest first.\n\n###### Computation boundary:\n*Result: Raw log lines, no aggregation.*\n*Rendered directly from Loki; no computation applied.*\n\n###### Source:\n[Log.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/basics/Log.cpp)\n\n###### Function:\n`Logs::Sink::write`\n\n###### References:\n[Loki log queries](https://grafana.com/docs/loki/latest/query/log_queries/)", "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 182 + "h": 12, + "w": 24, + "x": 0, + "y": 142 }, "fieldConfig": { "defaults": {}, diff --git a/docker/telemetry/grafana/dashboards/peer-quality.json b/docker/telemetry/grafana/dashboards/peer-quality.json index e17bb2dda4..2da2cfbd12 100644 --- a/docker/telemetry/grafana/dashboards/peer-quality.json +++ b/docker/telemetry/grafana/dashboards/peer-quality.json @@ -39,15 +39,28 @@ "id": null, "links": [], "panels": [ + { + "title": "Peer Latency", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [], + "id": 1 + }, { "title": "P90 Peer Latency", "description": "###### What this is:\n*90th-percentile round-trip latency to connected peers, in milliseconds.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the p90 peer latency.*\n\n###### Reading it:\n*Lower is better; green under 200ms, yellow to 500ms, red above.*\n\n###### Healthy range:\n*Under 200ms.*\n\n###### Watch for:\n*Rising latency, which points to network congestion or geographically distant, poorly performing peers.*\n\n###### Keywords:\n- **Peer** *(per node)* \u2014 another server this node holds a protocol connection to.\n- **Overlay** *(network-wide)* \u2014 the peer-to-peer mesh xrpld nodes form to gossip transactions, proposals and validations.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#peer)", "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 0 + "y": 1 }, "options": { "tooltip": { @@ -72,12 +85,13 @@ "custom": { "axisLabel": "Latency (ms)", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -100,17 +114,31 @@ } }, "overrides": [] - } + }, + "id": 2 }, { - "title": "Insane/Diverged Peers", + "title": "Peer Health", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "collapsed": false, + "panels": [], + "id": 3 + }, + { + "title": "Insane/Diverged Peers [$xrpl_network_type]", "description": "###### What this is:\n*Count of connected peers whose ledger state has diverged from the network.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the diverged-peer count.*\n\n###### Reading it:\n*Zero is healthy; any count means those peers disagree on ledger state.*\n\n###### Healthy range:\n*0 diverged peers.*\n\n###### Watch for:\n*A persistent non-zero count, which can indicate peers on a fork or misbehaving peers.*\n\n###### Keywords:\n- **Insane / diverged peers** *(per node)* \u2014 connected peers whose ledger state disagrees with the network \u2014 possibly on a fork or misbehaving.\n- **Fork** *(network-wide)* \u2014 when nodes validate divergent ledger chains instead of a single agreed history.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`\n\n###### References:\n[Fork](https://xrpl.org/docs/concepts/consensus-protocol) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#insane-diverged-peers)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 8 + "y": 10 }, "options": { "tooltip": { @@ -148,20 +176,25 @@ } ] }, - "custom": {} + "custom": {}, + "decimals": 0 }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 4 }, { - "title": "Higher Version Peers %", + "title": "Higher Version Peers % [$xrpl_network_type]", "description": "###### What this is:\n*Percentage of connected peers running a newer rippled version than this node.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the higher-version peer percentage.*\n\n###### Reading it:\n*A high percentage suggests this node is behind and should be upgraded.*\n\n###### Healthy range:\n*Under 30%.*\n\n###### Watch for:\n*A majority of peers on a newer version, a strong upgrade signal.*\n\n###### Keywords:\n- **Peer** *(per node)* \u2014 another server this node holds a protocol connection to.\n- **Overlay** *(network-wide)* \u2014 the peer-to-peer mesh xrpld nodes form to gossip transactions, proposals and validations.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#peer)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 16 + "h": 4, + "w": 12, + "x": 12, + "y": 10 }, "options": { "tooltip": { @@ -204,17 +237,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 5 }, { - "title": "Upgrade Recommended", + "title": "Upgrade Recommended [$xrpl_network_type]", "description": "###### What this is:\n*A flag indicating whether an upgrade is advised based on peer version analysis (Yes/No).*\n\n###### How it's computed:\n*Instantaneous gauge reading of the upgrade-recommended flag.*\n\n###### Reading it:\n*No is healthy; Yes means most peers run a newer version.*\n\n###### Healthy range:\n*No.*\n\n###### Watch for:\n*A Yes state, indicating the node risks falling out of step with the network.*\n\n###### Keywords:\n- **Peer** *(per node)* \u2014 another server this node holds a protocol connection to.\n- **Overlay** *(network-wide)* \u2014 the peer-to-peer mesh xrpld nodes form to gossip transactions, proposals and validations.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#peer)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 24 + "y": 14 }, "options": { "tooltip": { @@ -271,7 +308,24 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 6 + }, + { + "title": "Disconnects & Connection Mix", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "collapsed": false, + "panels": [], + "id": 7 }, { "title": "Resource Disconnects", @@ -279,9 +333,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 32 + "y": 19 }, "options": { "tooltip": { @@ -306,29 +360,32 @@ "custom": { "axisLabel": "Disconnects", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" - } + }, + "decimals": 0 }, "overrides": [] - } + }, + "id": 8 }, { - "title": "Inbound vs Outbound Peers", + "title": "Inbound vs Outbound Peers [$xrpl_network_type]", "description": "###### What this is:\n*Active inbound versus outbound peer connection counts.*\n\n###### How it's computed:\n*Instantaneous gauge readings of active inbound and outbound peer counts.*\n\n###### Reading it:\n*A balanced mix is healthy for connectivity and resilience.*\n\n###### Healthy range:\n*A mix of both; some outbound connections present.*\n\n###### Watch for:\n*All-inbound with no outbound, which usually means NAT or firewall issues block outbound peering.*\n\n###### Keywords:\n- **Peer** *(per node)* \u2014 another server this node holds a protocol connection to.\n- **Overlay** *(network-wide)* \u2014 the peer-to-peer mesh xrpld nodes form to gossip transactions, proposals and validations.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PeerfinderManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/peerfinder/detail/PeerfinderManager.cpp)\n\n###### Function:\n`Logic Stats ctor`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#peer)", "type": "bargauge", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 40 + "y": 27 }, "options": { "orientation": "horizontal", @@ -367,7 +424,8 @@ } ] }, - "custom": {} + "custom": {}, + "decimals": 0 }, "overrides": [ { @@ -401,7 +459,11 @@ ] } ] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 9 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/validator-health.json b/docker/telemetry/grafana/dashboards/validator-health.json index 4e32e7e606..3a0cde97bc 100644 --- a/docker/telemetry/grafana/dashboards/validator-health.json +++ b/docker/telemetry/grafana/dashboards/validator-health.json @@ -49,15 +49,16 @@ "y": 0 }, "collapsed": false, - "panels": [] + "panels": [], + "id": 1 }, { - "title": "Agreement % (1h)", + "title": "Agreement % (1h) [$xrpl_network_type]", "description": "###### What this is:\n*Share of ledgers over the last hour where this validator agreed with the network consensus.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the 1-hour agreement percentage.*\n\n###### Reading it:\n*Higher is better; green at 95%+, yellow from 80%, red below.*\n\n###### Healthy range:\n*95-100%.*\n\n###### Watch for:\n*Values below 80%, meaning the validator frequently disagrees with consensus.*\n\n###### Keywords:\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n- **Validation agreement** *(per node)* \u2014 the share of ledgers where this validator's validation matched network consensus versus was missed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Validation agreement](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, "y": 1 }, @@ -102,17 +103,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 2 }, { - "title": "Agreement % (24h)", + "title": "Agreement % (24h) [$xrpl_network_type]", "description": "###### What this is:\n*Share of ledgers over the last 24 hours where this validator agreed with consensus.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the 24-hour agreement percentage.*\n\n###### Reading it:\n*Higher is better; a smoother, longer-term view than the 1h stat.*\n\n###### Healthy range:\n*95-100%.*\n\n###### Watch for:\n*A sustained dip below 90%, which can indicate configuration drift or a network partition.*\n\n###### Keywords:\n- **Validation agreement** *(per node)* \u2014 the share of ledgers where this validator's validation matched network consensus versus was missed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`\n\n###### References:\n[Validation agreement](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-agreement)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 9 + "h": 4, + "w": 12, + "x": 12, + "y": 1 }, "options": { "tooltip": { @@ -155,17 +160,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 3 }, { - "title": "Agreements vs Missed (1h)", + "title": "Agreements vs Missed (1h) [$xrpl_network_type]", "description": "###### What this is:\n*Counts of agreed versus missed validations over the last hour.*\n\n###### How it's computed:\n*Instantaneous gauge readings of the 1-hour agreed and missed counts.*\n\n###### Reading it:\n*Agreements should dominate; missed should be small.*\n\n###### Healthy range:\n*Missed near 0.*\n\n###### Watch for:\n*A high missed count, meaning the validator is skipping consensus rounds.*\n\n###### Keywords:\n- **Validation agreement** *(per node)* \u2014 the share of ledgers where this validator's validation matched network consensus versus was missed.\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`\n\n###### References:\n[Validation agreement](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-agreement)", "type": "bargauge", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 17 + "y": 5 }, "options": { "orientation": "horizontal", @@ -223,17 +232,21 @@ ] } ] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 4 }, { - "title": "Agreements vs Missed (24h)", + "title": "Agreements vs Missed (24h) [$xrpl_network_type]", "description": "###### What this is:\n*Counts of agreed versus missed validations over the last 24 hours.*\n\n###### How it's computed:\n*Instantaneous gauge readings of the 24-hour agreed and missed counts.*\n\n###### Reading it:\n*Agreements should dominate over the full day.*\n\n###### Healthy range:\n*Missed a small fraction of agreements.*\n\n###### Watch for:\n*A growing missed share, signalling longer-term reliability problems.*\n\n###### Keywords:\n- **Validation agreement** *(per node)* \u2014 the share of ledgers where this validator's validation matched network consensus versus was missed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`\n\n###### References:\n[Validation agreement](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-agreement)", "type": "bargauge", "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 25 + "h": 4, + "w": 12, + "x": 12, + "y": 5 }, "options": { "orientation": "horizontal", @@ -291,7 +304,11 @@ ] } ] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 5 }, { "title": "Validation Rates", @@ -300,20 +317,21 @@ "h": 1, "w": 24, "x": 0, - "y": 33 + "y": 9 }, "collapsed": false, - "panels": [] + "panels": [], + "id": 6 }, { - "title": "Validation Rate", + "title": "Validation Rate [$xrpl_network_type]", "description": "###### What this is:\n*Validations this node sends per minute.*\n\n###### How it's computed:\n*Per-second rate of the sent-validations counter over 5 minutes, scaled to per minute.*\n\n###### Reading it:\n*Should track the ledger close cadence; roughly one validation per closed ledger.*\n\n###### Healthy range:\n*About 12-20 per minute (one per closed ledger, ~3-5s close).*\n\n###### Watch for:\n*A drop toward zero, meaning the validator has stopped participating.*\n\n###### Keywords:\n- **Validations checked vs sent** *(per node)* \u2014 validations this node received from peers and checked, versus validations it issued itself.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementValidationsSent (caller RCLConsensus.cpp)`\n\n###### References:\n[Validations checked vs sent](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validations-checked-vs-sent)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 34 + "y": 10 }, "options": { "tooltip": { @@ -354,17 +372,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 7 }, { - "title": "Validations Checked Rate", + "title": "Validations Checked Rate [$xrpl_network_type]", "description": "###### What this is:\n*Validations received from peers and checked per minute.*\n\n###### How it's computed:\n*Per-second rate of the checked-validations counter over 5 minutes, scaled to per minute.*\n\n###### Reading it:\n*Reflects how much validation traffic the network is delivering to this node.*\n\n###### Healthy range:\n*Workload-dependent; scales with trusted validator count.*\n\n###### Watch for:\n*A sudden collapse, which suggests peer connectivity loss or network isolation.*\n\n###### Keywords:\n- **Validations checked vs sent** *(per node)* \u2014 validations this node received from peers and checked, versus validations it issued itself.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementValidationsChecked (caller NetworkOPs.cpp)`\n\n###### References:\n[Validations checked vs sent](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validations-checked-vs-sent)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 42 + "h": 4, + "w": 12, + "x": 12, + "y": 10 }, "options": { "tooltip": { @@ -389,17 +411,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 8 }, { - "title": "Amendment Blocked", + "title": "Amendment Blocked [$xrpl_network_type]", "description": "###### What this is:\n*Whether the node is amendment-blocked (OK/BLOCKED).*\n\n###### How it's computed:\n*Instantaneous gauge reading of the amendment-blocked flag.*\n\n###### Reading it:\n*OK is healthy; BLOCKED means an enabled amendment is unsupported by this build.*\n\n###### Healthy range:\n*OK.*\n\n###### Watch for:\n*A BLOCKED state, which halts validation and requires a software upgrade.*\n\n###### Keywords:\n- **Amendment blocked** *(per node)* \u2014 the node has halted because the network enabled an amendment its software version does not support.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidatorHealthGauge`\n\n###### References:\n[Amendment blocked](https://xrpl.org/docs/concepts/networks-and-servers/amendments) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#amendment-blocked)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 50 + "y": 14 }, "options": { "tooltip": { @@ -456,17 +482,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 9 }, { - "title": "UNL Expiry (days)", + "title": "UNL Expiry (days) [$xrpl_network_type]", "description": "###### What this is:\n*Days remaining until the current UNL (trusted validator list) expires.*\n\n###### How it's computed:\n*Instantaneous gauge reading of days-to-expiry.*\n\n###### Reading it:\n*Higher is safer; green at 30+, yellow under 7, red at expiry.*\n\n###### Healthy range:\n*30+ days.*\n\n###### Watch for:\n*Fewer than 7 days, after which the node loses its trusted validator set if not renewed.*\n\n###### Keywords:\n- **UNL (Unique Node List)** *(per node)* \u2014 the list of validators a node trusts not to collude; the basis for its consensus and quorum.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidatorHealthGauge`\n\n###### References:\n[UNL (Unique Node List)](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-unique-node-list)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 58 + "h": 4, + "w": 12, + "x": 12, + "y": 14 }, "options": { "tooltip": { @@ -507,17 +537,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 10 }, { - "title": "UNL Blocked", + "title": "UNL Blocked [$xrpl_network_type]", "description": "###### What this is:\n*Whether the node's UNL is blocked (OK/BLOCKED).*\n\n###### How it's computed:\n*Instantaneous gauge reading of the UNL-blocked flag.*\n\n###### Reading it:\n*OK is healthy; BLOCKED means validator trust cannot be established.*\n\n###### Healthy range:\n*OK.*\n\n###### Watch for:\n*A BLOCKED state, which can stop the node participating in consensus.*\n\n###### Keywords:\n- **UNL (Unique Node List)** *(per node)* \u2014 the list of validators a node trusts not to collude; the basis for its consensus and quorum.\n- **UNL blocked** *(per node)* \u2014 the node cannot establish a usable trusted validator list, so it cannot safely validate.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidatorHealthGauge`\n\n###### References:\n[UNL (Unique Node List)](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [UNL blocked](https://xrpl.org/docs/concepts/consensus-protocol) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-unique-node-list)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 66 + "y": 18 }, "options": { "tooltip": { @@ -574,7 +608,11 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 11 }, { "title": "Agreement/Missed Counters (Rate)", @@ -582,9 +620,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 74 + "y": 22 }, "options": { "tooltip": { @@ -616,12 +654,13 @@ "custom": { "axisLabel": "Validations / Min", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -644,7 +683,8 @@ ] } ] - } + }, + "id": 12 }, { "title": "Server State & Consensus", @@ -653,20 +693,21 @@ "h": 1, "w": 24, "x": 0, - "y": 82 + "y": 30 }, "collapsed": false, - "panels": [] + "panels": [], + "id": 13 }, { - "title": "Validation Quorum", + "title": "Validation Quorum [$xrpl_network_type]", "description": "###### What this is:\n*Minimum number of trusted validations required to declare a ledger fully validated.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the current quorum requirement.*\n\n###### Reading it:\n*Tracks the quorum derived from the active validator list; changes when the list changes.*\n\n###### Healthy range:\n*Stable at the network-appropriate value.*\n\n###### Watch for:\n*An unexpected drop, which can weaken consensus safety guarantees.*\n\n###### Keywords:\n- **Validation quorum** *(network-wide)* \u2014 the minimum number of agreeing trusted validations needed to declare a ledger fully validated.\n- **Validator list** *(network-wide)* \u2014 signed lists of recommended validators (UNLs) that peers distribute to each other.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidatorHealthGauge`\n\n###### References:\n[Validation quorum](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) \u00b7 [Validator list](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-quorum)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 83 + "y": 31 }, "options": { "tooltip": { @@ -688,10 +729,15 @@ "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", "unit": "none", - "custom": {} + "custom": {}, + "decimals": 0 }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 14 }, { "title": "State Value Timeline", @@ -699,9 +745,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 91 + "y": 35 }, "options": { "tooltip": { @@ -726,29 +772,31 @@ "custom": { "axisLabel": "State", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" } }, "overrides": [] - } + }, + "id": 15 }, { - "title": "Time in Current State", + "title": "Time in Current State [$xrpl_network_type]", "description": "###### What this is:\n*How long the server has held its current operating state, in seconds.*\n\n###### How it's computed:\n*Current value of the time-in-state gauge.*\n\n###### Reading it:\n*Not yet wired in the code; the value currently always reads 0.*\n\n###### Healthy range:\n*Not applicable; the value is always 0 today.*\n\n###### Watch for:\n*n/a until implemented.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerStateTrackingGauge`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 99 + "y": 43 }, "options": { "tooltip": { @@ -773,17 +821,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 16 }, { - "title": "State Changes Rate", + "title": "State Changes Rate [$xrpl_network_type]", "description": "###### What this is:\n*Rate of server operating-state changes per hour.*\n\n###### How it's computed:\n*Per-hour rate of the state-change counter, averaged over a 1-hour window.*\n\n###### Reading it:\n*Near zero is healthy; each increment is one state transition.*\n\n###### Healthy range:\n*Near 0 changes per hour.*\n\n###### Watch for:\n*Frequent transitions, which point to network instability or configuration problems.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementStateChanges (caller NetworkOPs.cpp)`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 107 + "h": 4, + "w": 12, + "x": 12, + "y": 43 }, "options": { "tooltip": { @@ -824,17 +876,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 17 }, { - "title": "Ledgers Closed Rate", + "title": "Ledgers Closed Rate [$xrpl_network_type]", "description": "###### What this is:\n*Ledgers closed per minute by this node.*\n\n###### How it's computed:\n*Per-second rate of the ledgers-closed counter over 5 minutes, scaled to per minute.*\n\n###### Reading it:\n*Should match the network's steady close cadence.*\n\n###### Healthy range:\n*About 12-20 per minute (one per closed ledger, ~3-5s close).*\n\n###### Watch for:\n*Deviation from the expected cadence, which indicates consensus timing trouble or the node falling behind.*\n\n###### Keywords:\n- **Ledgers closed rate** *(per node)* \u2014 how many ledgers this node closed per minute; should match the network close cadence.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementLedgersClosed (caller RCLConsensus.cpp)`\n\n###### References:\n[Ledgers closed rate](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledgers-closed-rate)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 115 + "y": 47 }, "options": { "tooltip": { @@ -875,17 +931,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 18 }, { - "title": "Agreement % (7d)", + "title": "Agreement % (7d) [$xrpl_network_type]", "description": "###### What this is:\n*Share of ledgers over the trailing 7 days where this validator agreed with consensus.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the 7-day agreement percentage.*\n\n###### Reading it:\n*The long-term reliability window; higher is better.*\n\n###### Healthy range:\n*95-100%.*\n\n###### Watch for:\n*A gradual decline, which reflects chronic rather than transient disagreement.*\n\n###### Keywords:\n- **Validation agreement** *(per node)* \u2014 the share of ledgers where this validator's validation matched network consensus versus was missed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`\n\n###### References:\n[Validation agreement](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-agreement)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 123 + "h": 4, + "w": 12, + "x": 12, + "y": 47 }, "options": { "tooltip": { @@ -928,7 +988,11 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 19 }, { "title": "Agreements vs Missed (7d)", @@ -936,9 +1000,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 131 + "y": 51 }, "options": { "tooltip": { @@ -972,11 +1036,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 20 } ], "schemaVersion": 39, From d5763420f95106641dddc3d61c0345e05cc65cd4 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:54:48 +0100 Subject: [PATCH 3/4] style(telemetry): lead each dashboard row with its single-value panels Guideline 8 asks for gauges and stats at the top. Seven dashboards had them scattered below charts, so the reader met a wall of time series before the at-a-glance numbers that give those series context. Stats, gauges and bar gauges now come first within each row section. The move is deliberately scoped to inside a section: shifting a panel across a row boundary would change which category it belongs to. Panels keep their relative order otherwise, so the reading sequence within each group is unchanged. Whole panel objects are cut and re-spliced as raw text, so their contents stay byte-identical and only gridPos and id are recomputed. Verified per dashboard: panel count unchanged, no panel lost, every targets block byte-identical, ids still 1..N, and no row section left with a stat below a chart. --- .../grafana/dashboards/consensus-health.json | 80 ++-- .../grafana/dashboards/ledger-data-sync.json | 178 ++++---- .../grafana/dashboards/ledger-operations.json | 90 ++-- .../grafana/dashboards/node-health.json | 414 +++++++++--------- .../grafana/dashboards/rpc-pathfinding.json | 218 ++++----- .../grafana/dashboards/rpc-performance.json | 340 +++++++------- .../dashboards/transaction-overview.json | 130 +++--- 7 files changed, 725 insertions(+), 725 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json index 0c127ddcb3..8bda530684 100644 --- a/docker/telemetry/grafana/dashboards/consensus-health.json +++ b/docker/telemetry/grafana/dashboards/consensus-health.json @@ -87,6 +87,43 @@ "panels": [], "id": 3 }, + { + "title": "Validation Send Rate [$xrpl_network_type]", + "description": "**What:** How often this node issues a validation confirming it has fully validated a ledger.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should closely track the ledger close rate on a healthy validator.\n**Healthy range:** About one per ledger (~0.25/s on mainnet).\n**Watch for:** Validations lagging closes suggests the node is falling behind on validation.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::createValidationSpan", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 10 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", consensus_mode=~\"$consensus_mode\", span_name=\"consensus.validation.send\"}[$__rate_interval]))", + "interval": "15s", + "legendFormat": "Validations / Sec [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: validations/s" + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 4 + }, { "title": "Consensus Proposals Sent Rate", "description": "**What:** How often this node broadcasts its own consensus proposal (its candidate transaction set) to the network.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** A proposing validator emits a steady stream, roughly one per consensus round; observing-only nodes emit none.\n**Healthy range:** About one proposal per round (~0.2-0.3/s on mainnet) for a proposing node.\n**Watch for:** A validator that drops to zero has stopped proposing and may be unhealthy or misconfigured.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::propose", @@ -95,7 +132,7 @@ "h": 8, "w": 12, "x": 0, - "y": 10 + "y": 14 }, "options": { "tooltip": { @@ -130,7 +167,7 @@ }, "overrides": [] }, - "id": 4 + "id": 5 }, { "title": "Ledger Close Duration", @@ -140,7 +177,7 @@ "h": 8, "w": 12, "x": 12, - "y": 10 + "y": 14 }, "options": { "tooltip": { @@ -175,43 +212,6 @@ }, "overrides": [] }, - "id": 5 - }, - { - "title": "Validation Send Rate [$xrpl_network_type]", - "description": "**What:** How often this node issues a validation confirming it has fully validated a ledger.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should closely track the ledger close rate on a healthy validator.\n**Healthy range:** About one per ledger (~0.25/s on mainnet).\n**Watch for:** Validations lagging closes suggests the node is falling behind on validation.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::createValidationSpan", - "type": "stat", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 18 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", consensus_mode=~\"$consensus_mode\", span_name=\"consensus.validation.send\"}[$__rate_interval]))", - "interval": "15s", - "legendFormat": "Validations / Sec [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "suffix: validations/s" - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, "id": 6 }, { diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index 3532137aef..7de2692b78 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -9,6 +9,67 @@ "id": null, "links": [], "panels": [ + { + "title": "Overlay Traffic Heatmap (All Categories, Bytes In) [$xrpl_network_type]", + "description": "###### What this is:\n*All overlay traffic categories ranked by inbound bytes, giving an at-a-glance view of which message types consume the most receive bandwidth.*\n\n###### How it's computed:\n*Top categories by latest inbound byte value across all traffic categories. Each bar is labelled with its traffic category followed by the node identity; the shared `_bytes_in` suffix is dropped from the category name because the panel already reports inbound bytes.*\n\n###### Reading it:\n*The longest bars are the biggest bandwidth consumers; on a synced node transactions, proposals, and validations usually lead.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*A single ledger-data or fetch category dominating (ongoing sync) or an unexpected category topping the list.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "bargauge", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(topk(20, {service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", __name__=~\".*_bytes_in\", __name__!~\"total_.*\"}), \"series\", \"$1\", \"__name__\", \"(.*)_bytes_in\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "decbytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1048576 + }, + { + "color": "red", + "value": 104857600 + } + ] + } + }, + "overrides": [] + }, + "id": 1, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, { "title": "Ledger Data \u2014 Ledger", "description": "###### What this is:\n*Inbound bytes for ledger-data message categories, split into aggregate get/share plus the transaction-set, transaction-node, and account-state-node sub-types the node receives from peers.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Normally low and flat once synced. Account-state-node traffic dominates during state sync; transaction-set-candidate traffic dominates during consensus catch-up.*\n\n###### Healthy range:\n*workload-dependent; low and steady on a synced node.*\n\n###### Watch for:\n*Sustained high account-state or tx-node inbound bytes on a node that should be caught up (repeated re-sync, missing history), or a single peer driving all traffic.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", @@ -17,7 +78,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 4 }, "options": { "tooltip": { @@ -57,7 +118,7 @@ }, "overrides": [] }, - "id": 1 + "id": 2 }, { "title": "Ledger Data \u2014 Transaction", @@ -67,7 +128,7 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 4 }, "options": { "tooltip": { @@ -119,7 +180,7 @@ }, "overrides": [] }, - "id": 2 + "id": 3 }, { "title": "Ledger Data \u2014 Account State", @@ -129,7 +190,7 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 12 }, "options": { "tooltip": { @@ -169,7 +230,7 @@ }, "overrides": [] }, - "id": 3 + "id": 4 }, { "title": "Ledger Traffic \u2014 Ledger", @@ -179,7 +240,7 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 12 }, "options": { "tooltip": { @@ -219,7 +280,7 @@ }, "overrides": [] }, - "id": 4 + "id": 5 }, { "title": "Ledger Traffic \u2014 Transaction", @@ -229,7 +290,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 20 }, "options": { "tooltip": { @@ -281,7 +342,7 @@ }, "overrides": [] }, - "id": 5 + "id": 6 }, { "title": "Ledger Traffic \u2014 Account State", @@ -291,7 +352,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 20 }, "options": { "tooltip": { @@ -331,7 +392,7 @@ }, "overrides": [] }, - "id": 6 + "id": 7 }, { "title": "GetObject \u2014 Ledger", @@ -341,7 +402,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 28 }, "options": { "tooltip": { @@ -381,7 +442,7 @@ }, "overrides": [] }, - "id": 7 + "id": 8 }, { "title": "GetObject \u2014 Transaction", @@ -391,7 +452,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 28 }, "options": { "tooltip": { @@ -443,7 +504,7 @@ }, "overrides": [] }, - "id": 8 + "id": 9 }, { "title": "GetObject \u2014 Account State", @@ -453,7 +514,7 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 36 }, "options": { "tooltip": { @@ -493,7 +554,7 @@ }, "overrides": [] }, - "id": 9 + "id": 10 }, { "title": "GetObject Messages \u2014 Ledger", @@ -503,7 +564,7 @@ "h": 8, "w": 12, "x": 12, - "y": 32 + "y": 36 }, "options": { "tooltip": { @@ -537,7 +598,7 @@ }, "overrides": [] }, - "id": 10 + "id": 11 }, { "title": "GetObject Messages \u2014 Transaction", @@ -547,7 +608,7 @@ "h": 8, "w": 12, "x": 0, - "y": 40 + "y": 44 }, "options": { "tooltip": { @@ -587,7 +648,7 @@ }, "overrides": [] }, - "id": 11 + "id": 12 }, { "title": "GetObject Messages \u2014 Account State", @@ -597,7 +658,7 @@ "h": 8, "w": 12, "x": 12, - "y": 40 + "y": 44 }, "options": { "tooltip": { @@ -631,7 +692,7 @@ }, "overrides": [] }, - "id": 12 + "id": 13 }, { "title": "GetObject Messages \u2014 Specials", @@ -641,7 +702,7 @@ "h": 8, "w": 12, "x": 0, - "y": 48 + "y": 52 }, "options": { "tooltip": { @@ -687,7 +748,7 @@ }, "overrides": [] }, - "id": 13 + "id": 14 }, { "title": "GetObject \u2014 Specials", @@ -697,7 +758,7 @@ "h": 8, "w": 12, "x": 12, - "y": 48 + "y": 52 }, "options": { "tooltip": { @@ -767,68 +828,7 @@ }, "overrides": [] }, - "id": 14 - }, - { - "title": "Overlay Traffic Heatmap (All Categories, Bytes In) [$xrpl_network_type]", - "description": "###### What this is:\n*All overlay traffic categories ranked by inbound bytes, giving an at-a-glance view of which message types consume the most receive bandwidth.*\n\n###### How it's computed:\n*Top categories by latest inbound byte value across all traffic categories. Each bar is labelled with its traffic category followed by the node identity; the shared `_bytes_in` suffix is dropped from the category name because the panel already reports inbound bytes.*\n\n###### Reading it:\n*The longest bars are the biggest bandwidth consumers; on a synced node transactions, proposals, and validations usually lead.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*A single ledger-data or fetch category dominating (ongoing sync) or an unexpected category topping the list.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", - "type": "bargauge", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 56 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - }, - "displayMode": "gradient", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "label_replace(topk(20, {service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", __name__=~\".*_bytes_in\", __name__!~\"total_.*\"}), \"series\", \"$1\", \"__name__\", \"(.*)_bytes_in\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "decbytes", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1048576 - }, - { - "color": "red", - "value": 104857600 - } - ] - } - }, - "overrides": [] - }, - "id": 15, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2 + "id": 15 }, { "title": "Sync Diagnostics", diff --git a/docker/telemetry/grafana/dashboards/ledger-operations.json b/docker/telemetry/grafana/dashboards/ledger-operations.json index 4c04a97c66..e4b8f02b3d 100644 --- a/docker/telemetry/grafana/dashboards/ledger-operations.json +++ b/docker/telemetry/grafana/dashboards/ledger-operations.json @@ -58,6 +58,43 @@ "maxPerRow": 2, "id": 2 }, + { + "title": "Ledger Validation Rate [$xrpl_network_type]", + "description": "**What:** How often ledgers reach the trusted-validation quorum and become fully validated.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should match the build and close rates on a healthy node.\n**Healthy range:** About 0.25/s on mainnet.\n**Watch for:** A shortfall means ledgers are not accumulating enough trusted validations to be accepted.\n**Source:** src/xrpld/app/ledger/detail/LedgerMaster.cpp:LedgerMaster::checkAccept", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 1 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"ledger.validate\"}[$__rate_interval]))", + "interval": "15s", + "legendFormat": "Validations / Sec [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: ledgers/s" + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 3 + }, { "title": "Ledger Build Duration", "description": "**What:** Time to build a full ledger, including applying transactions, writing state, and finalising acceptance.\n**How it's computed:** 95th and 50th percentile of per-build durations over a 5-minute window, per node.\n**Reading it:** Scales with transaction volume and disk I/O.\n**Healthy range:** Tens to a few hundred milliseconds under typical load.\n**Watch for:** Long build times indicate expensive transaction sets or I/O pressure while flushing state.\n**Source:** src/xrpld/app/ledger/detail/BuildLedger.cpp:buildLedgerImpl", @@ -109,43 +146,6 @@ }, "overrides": [] }, - "id": 3 - }, - { - "title": "Ledger Validation Rate [$xrpl_network_type]", - "description": "**What:** How often ledgers reach the trusted-validation quorum and become fully validated.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should match the build and close rates on a healthy node.\n**Healthy range:** About 0.25/s on mainnet.\n**Watch for:** A shortfall means ledgers are not accumulating enough trusted validations to be accepted.\n**Source:** src/xrpld/app/ledger/detail/LedgerMaster.cpp:LedgerMaster::checkAccept", - "type": "stat", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 13 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"ledger.validate\"}[$__rate_interval]))", - "interval": "15s", - "legendFormat": "Validations / Sec [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "suffix: ledgers/s" - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, "id": 4 }, { @@ -155,8 +155,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 17 + "x": 12, + "y": 5 }, "options": { "tooltip": { @@ -197,7 +197,7 @@ "h": 1, "w": 24, "x": 0, - "y": 25 + "y": 13 }, "collapsed": false, "panels": [], @@ -211,7 +211,7 @@ "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 14 }, "options": { "tooltip": { @@ -264,7 +264,7 @@ "h": 8, "w": 12, "x": 12, - "y": 26 + "y": 14 }, "options": { "tooltip": { @@ -308,7 +308,7 @@ "h": 1, "w": 24, "x": 0, - "y": 34 + "y": 22 }, "collapsed": false, "panels": [], @@ -322,7 +322,7 @@ "h": 4, "w": 12, "x": 0, - "y": 35 + "y": 23 }, "options": { "tooltip": { @@ -359,7 +359,7 @@ "h": 8, "w": 12, "x": 0, - "y": 39 + "y": 27 }, "options": { "tooltip": { diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index f0a560c73a..0e11e71d9d 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -113,6 +113,90 @@ "maxPerRow": 2, "id": 2 }, + { + "title": "Ledger Fetch Rate [$xrpl_network_type]", + "description": "**What:** Rate at which the node requests ledgers from peers.\n**How it's computed:** Per-second rate of the ledger-fetches counter over 5 minutes.\n**Reading it:** Near zero when synced; rises when catching up or backfilling history.\n**Healthy range:** workload-dependent; low on a synced node.\n**Watch for:** Sustained high fetch rate on a node that should be current (repeatedly missing ledgers).\n**Source:** src/xrpld/app/ledger/detail/InboundLedgers.cpp InboundLedgersImp ctor", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 4 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "rate(ledger_fetches_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", + "legendFormat": "Fetches / Sec [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: fetches/s" + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 3 + }, + { + "title": "Ledger History Mismatches [$xrpl_network_type]", + "description": "**What:** Rate of ledger-history hash mismatches: built ledgers whose hash disagrees with the validated hash.\n**How it's computed:** Per-second rate of the history-mismatch counter over 5 minutes.\n**Reading it:** Should be flat at zero.\n**Healthy range:** zero.\n**Watch for:** Any non-zero value (consensus divergence, corrupted history, or database issues).\n**Source:** src/xrpld/telemetry/MetricsRegistry.cpp incrementLedgerHistoryMismatch (caller LedgerHistory.cpp)", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 4 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "rate(ledger_history_mismatch_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", + "legendFormat": "Mismatches / Sec [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: mismatches/s", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.01 + } + ] + } + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 4 + }, { "title": "Operating Mode (Time Share)", "description": "**What:** Fraction of time in each operating mode (per-interval): Disconnected, Connected, Syncing, Tracking, Full.\n**How it's computed:** Per-mode duration rate divided by the sum of all five mode rates over the interval; the five shares stack to 100%.\n**Reading it:** Full should fill almost the entire band on a healthy node.\n**Healthy range:** Full near 100%; other modes near 0.\n**Watch for:** A rising share of Syncing, Connected, or Disconnected (instability or repeated resync).\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp NetworkOPsImp::Stats ctor", @@ -121,7 +205,7 @@ "h": 8, "w": 12, "x": 0, - "y": 4 + "y": 8 }, "options": { "tooltip": { @@ -186,7 +270,7 @@ }, "overrides": [] }, - "id": 3 + "id": 5 }, { "title": "Operating Mode Transitions", @@ -196,7 +280,7 @@ "h": 8, "w": 12, "x": 12, - "y": 4 + "y": 8 }, "options": { "tooltip": { @@ -258,7 +342,7 @@ }, "overrides": [] }, - "id": 4 + "id": 6 }, { "title": "I/O Latency", @@ -268,7 +352,7 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 16 }, "options": { "tooltip": { @@ -309,7 +393,7 @@ }, "overrides": [] }, - "id": 5 + "id": 7 }, { "title": "Job Queue Depth", @@ -319,7 +403,7 @@ "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 16 }, "options": { "tooltip": { @@ -354,90 +438,6 @@ }, "overrides": [] }, - "id": 6 - }, - { - "title": "Ledger Fetch Rate [$xrpl_network_type]", - "description": "**What:** Rate at which the node requests ledgers from peers.\n**How it's computed:** Per-second rate of the ledger-fetches counter over 5 minutes.\n**Reading it:** Near zero when synced; rises when catching up or backfilling history.\n**Healthy range:** workload-dependent; low on a synced node.\n**Watch for:** Sustained high fetch rate on a node that should be current (repeatedly missing ledgers).\n**Source:** src/xrpld/app/ledger/detail/InboundLedgers.cpp InboundLedgersImp ctor", - "type": "stat", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 20 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "rate(ledger_fetches_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", - "legendFormat": "Fetches / Sec [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "suffix: fetches/s" - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, - "id": 7 - }, - { - "title": "Ledger History Mismatches [$xrpl_network_type]", - "description": "**What:** Rate of ledger-history hash mismatches: built ledgers whose hash disagrees with the validated hash.\n**How it's computed:** Per-second rate of the history-mismatch counter over 5 minutes.\n**Reading it:** Should be flat at zero.\n**Healthy range:** zero.\n**Watch for:** Any non-zero value (consensus divergence, corrupted history, or database issues).\n**Source:** src/xrpld/telemetry/MetricsRegistry.cpp incrementLedgerHistoryMismatch (caller LedgerHistory.cpp)", - "type": "stat", - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 20 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "rate(ledger_history_mismatch_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", - "legendFormat": "Mismatches / Sec [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "suffix: mismatches/s", - "thresholds": { - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.01 - } - ] - } - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, "id": 8 }, { @@ -453,6 +453,112 @@ "panels": [], "id": 9 }, + { + "title": "FullBelowCache Hit Rate [$xrpl_network_type]", + "description": "**What:** Hit-rate percentage of the FullBelowCache.\n**How it's computed:** Current value of the cache hit-rate gauge (0-100%).\n**Reading it:** Higher is better; means the node reuses knowledge of complete subtrees.\n**Healthy range:** above ~50% in steady state.\n**Watch for:** Low hit rate during steady state (cache too small or constant re-acquisition).\n**Source:** include/xrpl/basics/TaggedCache.h TaggedCache::Stats ctor", + "type": "gauge", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 25 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "node_family_full_below_cache_hit_rate{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}", + "legendFormat": "Hit Rate" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 25 + }, + { + "color": "green", + "value": 50 + } + ] + } + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 10 + }, + { + "title": "Ledger Publish Gap [$xrpl_network_type]", + "description": "**What:** Difference between published and validated ledger ages: how far the publish pipeline trails validation.\n**How it's computed:** Published-ledger age minus validated-ledger age, in seconds.\n**Reading it:** Near zero means publishing keeps up with validation.\n**Healthy range:** within a few seconds of zero.\n**Watch for:** A growing gap (publish backlog, stale data for subscribers).\n**Source:** src/xrpld/app/ledger/LedgerMaster.h LedgerMaster::Stats ctor", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 25 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "ledgermaster_published_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"} - ledgermaster_validated_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}", + "legendFormat": "Publish Gap" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 11 + }, { "title": "Key Jobs Execution Time", "description": "**What:** Execution time of the most important job types (accept/advance ledger, transaction, write objects, heartbeat, sweep, trusted validation/proposal, publish, client RPC, ledger data) at the selected quantile.\n**How it's computed:** Selected quantile of each job's execution-time histogram over 5 minutes.\n**Reading it:** Lower and stable is better; shows where consensus, transaction, and maintenance time goes.\n**Healthy range:** workload-dependent; most jobs in low tens of milliseconds.\n**Watch for:** One job type spiking (a specific bottleneck) or broad increases (overload).\n**Source:** include/xrpl/core/JobTypeData.h JobTypeData ctor", @@ -461,7 +567,7 @@ "h": 8, "w": 12, "x": 0, - "y": 25 + "y": 29 }, "options": { "tooltip": { @@ -565,7 +671,7 @@ }, "overrides": [] }, - "id": 10 + "id": 12 }, { "title": "Key Jobs Dequeue Wait Time", @@ -575,7 +681,7 @@ "h": 8, "w": 12, "x": 12, - "y": 25 + "y": 29 }, "options": { "tooltip": { @@ -679,7 +785,7 @@ }, "overrides": [] }, - "id": 11 + "id": 13 }, { "title": "FullBelowCache Size", @@ -689,7 +795,7 @@ "h": 8, "w": 12, "x": 0, - "y": 33 + "y": 37 }, "options": { "tooltip": { @@ -723,112 +829,6 @@ }, "overrides": [] }, - "id": 12 - }, - { - "title": "FullBelowCache Hit Rate [$xrpl_network_type]", - "description": "**What:** Hit-rate percentage of the FullBelowCache.\n**How it's computed:** Current value of the cache hit-rate gauge (0-100%).\n**Reading it:** Higher is better; means the node reuses knowledge of complete subtrees.\n**Healthy range:** above ~50% in steady state.\n**Watch for:** Low hit rate during steady state (cache too small or constant re-acquisition).\n**Source:** include/xrpl/basics/TaggedCache.h TaggedCache::Stats ctor", - "type": "gauge", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 41 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "node_family_full_below_cache_hit_rate{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}", - "legendFormat": "Hit Rate" - } - ], - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 25 - }, - { - "color": "green", - "value": 50 - } - ] - } - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, - "id": 13 - }, - { - "title": "Ledger Publish Gap [$xrpl_network_type]", - "description": "**What:** Difference between published and validated ledger ages: how far the publish pipeline trails validation.\n**How it's computed:** Published-ledger age minus validated-ledger age, in seconds.\n**Reading it:** Near zero means publishing keeps up with validation.\n**Healthy range:** within a few seconds of zero.\n**Watch for:** A growing gap (publish backlog, stale data for subscribers).\n**Source:** src/xrpld/app/ledger/LedgerMaster.h LedgerMaster::Stats ctor", - "type": "stat", - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 41 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "ledgermaster_published_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"} - ledgermaster_validated_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}", - "legendFormat": "Publish Gap" - } - ], - "fieldConfig": { - "defaults": { - "unit": "s", - "thresholds": { - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "red", - "value": 10 - } - ] - } - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, "id": 14 }, { @@ -838,8 +838,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 45 + "x": 12, + "y": 37 }, "options": { "tooltip": { @@ -889,7 +889,7 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, + "x": 0, "y": 45 }, "options": { @@ -933,8 +933,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 53 + "x": 12, + "y": 45 }, "options": { "tooltip": { diff --git a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json index 094f864531..32c7868e93 100644 --- a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json +++ b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json @@ -247,112 +247,6 @@ "panels": [], "id": 6 }, - { - "title": "Pathfinding Fast Duration", - "description": "**What:** P95 and P50 execution time of the fast pathfinding search (simplified, favoring speed over completeness).\n**How it's computed:** 95th and 50th percentiles over a 5-minute window.\n**Reading it:** Lower is better; fast pathfinding should stay quick.\n**Healthy range:** workload-dependent; typically well below full pathfinding.\n**Watch for:** Rising fast-path latency (pathfinding load or complex order books).\n**Source:** src/xrpld/rpc/detail/PathRequestManager.h ctor", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 22 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "histogram_quantile(0.95, sum by (le, service_instance_id) (rate(pathfind_fast_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", - "interval": "15s", - "legendFormat": "P95 Fast Pathfind [{{service_instance_id}}]" - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "histogram_quantile(0.5, sum by (le, service_instance_id) (rate(pathfind_fast_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", - "interval": "15s", - "legendFormat": "P50 Fast Pathfind [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "ms", - "custom": { - "axisLabel": "Duration (ms)", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 5, - "lineWidth": 1, - "fillOpacity": 0, - "gradientMode": "none" - } - }, - "overrides": [] - }, - "id": 7 - }, - { - "title": "Pathfinding Full Duration", - "description": "**What:** P95 and P50 execution time of the full, exhaustive pathfinding search.\n**How it's computed:** 95th and 50th percentiles over a 5-minute window.\n**Reading it:** Full pathfinding is heavier and slower than fast mode.\n**Healthy range:** workload-dependent.\n**Watch for:** High or rising full-path latency (expensive path computation under subscription load).\n**Source:** src/xrpld/rpc/detail/PathRequestManager.h ctor", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 22 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "histogram_quantile(0.95, sum by (le, service_instance_id) (rate(pathfind_full_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", - "interval": "15s", - "legendFormat": "P95 Full Pathfind [{{service_instance_id}}]" - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "histogram_quantile(0.5, sum by (le, service_instance_id) (rate(pathfind_full_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", - "interval": "15s", - "legendFormat": "P50 Full Pathfind [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "ms", - "custom": { - "axisLabel": "Duration (ms)", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 5, - "lineWidth": 1, - "fillOpacity": 0, - "gradientMode": "none" - } - }, - "overrides": [] - }, - "id": 8 - }, { "title": "Resource Warnings Rate [$xrpl_network_type]", "description": "**What:** Rate of resource-limit warnings raised when a peer or client exceeds its usage warning threshold.\n**How it's computed:** Per-second rate of the warn counter over 5 minutes.\n**Reading it:** Occasional warnings are normal under load; a rising rate flags aggressive clients.\n**Healthy range:** workload-dependent; low.\n**Watch for:** A climbing warning rate (clients approaching limits, a precursor to drops).\n**Source:** include/xrpl/resource/detail/Logic.h Logic::Stats ctor", @@ -361,7 +255,7 @@ "h": 4, "w": 12, "x": 0, - "y": 30 + "y": 22 }, "options": { "tooltip": { @@ -404,7 +298,7 @@ "repeat": "xrpl_network_type", "repeatDirection": "h", "maxPerRow": 2, - "id": 9 + "id": 7 }, { "title": "Resource Drops Rate [$xrpl_network_type]", @@ -414,7 +308,7 @@ "h": 4, "w": 12, "x": 12, - "y": 30 + "y": 22 }, "options": { "tooltip": { @@ -457,6 +351,112 @@ "repeat": "xrpl_network_type", "repeatDirection": "h", "maxPerRow": 2, + "id": 8 + }, + { + "title": "Pathfinding Fast Duration", + "description": "**What:** P95 and P50 execution time of the fast pathfinding search (simplified, favoring speed over completeness).\n**How it's computed:** 95th and 50th percentiles over a 5-minute window.\n**Reading it:** Lower is better; fast pathfinding should stay quick.\n**Healthy range:** workload-dependent; typically well below full pathfinding.\n**Watch for:** Rising fast-path latency (pathfinding load or complex order books).\n**Source:** src/xrpld/rpc/detail/PathRequestManager.h ctor", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le, service_instance_id) (rate(pathfind_fast_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "interval": "15s", + "legendFormat": "P95 Fast Pathfind [{{service_instance_id}}]" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le, service_instance_id) (rate(pathfind_fast_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "interval": "15s", + "legendFormat": "P50 Fast Pathfind [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + } + }, + "overrides": [] + }, + "id": 9 + }, + { + "title": "Pathfinding Full Duration", + "description": "**What:** P95 and P50 execution time of the full, exhaustive pathfinding search.\n**How it's computed:** 95th and 50th percentiles over a 5-minute window.\n**Reading it:** Full pathfinding is heavier and slower than fast mode.\n**Healthy range:** workload-dependent.\n**Watch for:** High or rising full-path latency (expensive path computation under subscription load).\n**Source:** src/xrpld/rpc/detail/PathRequestManager.h ctor", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le, service_instance_id) (rate(pathfind_full_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "interval": "15s", + "legendFormat": "P95 Full Pathfind [{{service_instance_id}}]" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le, service_instance_id) (rate(pathfind_full_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "interval": "15s", + "legendFormat": "P50 Full Pathfind [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + } + }, + "overrides": [] + }, "id": 10 }, { diff --git a/docker/telemetry/grafana/dashboards/rpc-performance.json b/docker/telemetry/grafana/dashboards/rpc-performance.json index f2f19ce578..aa8bac5c83 100644 --- a/docker/telemetry/grafana/dashboards/rpc-performance.json +++ b/docker/telemetry/grafana/dashboards/rpc-performance.json @@ -8,96 +8,6 @@ "id": null, "links": [], "panels": [ - { - "title": "RPC Request Rate by Command", - "description": "**What:** Per-command throughput of RPC calls the node serves (e.g. server_info, submit, account_info).\n**How it's computed:** Per-second rate split by command name over a 5-minute window, per node.\n**Reading it:** Shows which API endpoints drive load and how demand shifts over time.\n**Healthy range:** Workload-dependent; varies with client mix.\n**Watch for:** A sudden spike on one command can indicate a runaway client or scripted abuse.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc", - "maxHeight": 600 - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "sum by (command, service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval]))", - "interval": "15s", - "legendFormat": "{{command}} [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "reqps", - "custom": { - "axisLabel": "Requests / Sec", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 5, - "lineWidth": 1, - "fillOpacity": 0, - "gradientMode": "none" - } - }, - "overrides": [] - }, - "id": 1 - }, - { - "title": "RPC Latency P95 by Command", - "description": "**What:** 95th-percentile response time for each RPC command.\n**How it's computed:** 95th percentile of per-command durations over a 5-minute window, per node.\n**Reading it:** Read-only lookups should be fast; heavy commands like path finding are naturally slower.\n**Healthy range:** Sub-second for most commands; command-dependent.\n**Watch for:** Rising p95 on a normally fast command signals contention or an expensive query pattern.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc", - "maxHeight": 600 - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "histogram_quantile(0.95, sum by (le, command, service_instance_id) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\"}[5m])))", - "interval": "15s", - "legendFormat": "P95 {{command}} [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "ms", - "custom": { - "axisLabel": "Latency (ms)", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 5, - "lineWidth": 1, - "fillOpacity": 0, - "gradientMode": "none" - } - }, - "overrides": [] - }, - "id": 2 - }, { "title": "RPC Error Rate [$xrpl_network_type]", "description": "**What:** Percentage of each RPC command's calls that ended in an error.\n**How it's computed:** Error calls divided by total calls per command over a 5-minute window, per node.\n**Reading it:** Green below 1%, yellow 1-5%, red above 5%.\n**Healthy range:** Below 1% for healthy commands.\n**Watch for:** A sustained error spike on one command is consistent with a client probing or misusing that endpoint.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", @@ -106,7 +16,7 @@ "h": 4, "w": 12, "x": 0, - "y": 8 + "y": 0 }, "options": { "tooltip": { @@ -149,8 +59,172 @@ "repeat": "xrpl_network_type", "repeatDirection": "h", "maxPerRow": 2, + "id": 1 + }, + { + "title": "Top Commands by Volume [$xrpl_network_type]", + "description": "**What:** The ten most-called RPC commands by total invocations recently.\n**How it's computed:** Ranked total counts per command over a 5-minute window, per node.\n**Reading it:** Identifies the hottest API endpoints driving node load.\n**Healthy range:** Workload-dependent.\n**Watch for:** An unexpected command dominating the ranking can indicate scripted abuse or a misbehaving client.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", + "type": "bargauge", + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 0 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "topk(10, sum by (command, service_instance_id) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval])))", + "interval": "15s", + "legendFormat": "{{command}} [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 2 + }, + { + "title": "WebSocket Message Rate [$xrpl_network_type]", + "description": "**What:** Rate of incoming WebSocket RPC messages the server processes.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Non-zero only when clients connect over WebSocket rather than HTTP.\n**Healthy range:** Zero is normal for HTTP-only deployments; otherwise workload-dependent.\n**Watch for:** An unexpected surge points to a chatty or abusive WebSocket client.\n**Source:** src/xrpld/rpc/detail/ServerHandler.cpp:ServerHandler::processSession", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 4 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=\"rpc.ws_message\"}[$__rate_interval]))", + "interval": "15s", + "legendFormat": "WS Messages / Sec [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: messages/s" + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, "id": 3 }, + { + "title": "RPC Request Rate by Command", + "description": "**What:** Per-command throughput of RPC calls the node serves (e.g. server_info, submit, account_info).\n**How it's computed:** Per-second rate split by command name over a 5-minute window, per node.\n**Reading it:** Shows which API endpoints drive load and how demand shifts over time.\n**Healthy range:** Workload-dependent; varies with client mix.\n**Watch for:** A sudden spike on one command can indicate a runaway client or scripted abuse.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "sum by (command, service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval]))", + "interval": "15s", + "legendFormat": "{{command}} [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "axisLabel": "Requests / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + } + }, + "overrides": [] + }, + "id": 4 + }, + { + "title": "RPC Latency P95 by Command", + "description": "**What:** 95th-percentile response time for each RPC command.\n**How it's computed:** 95th percentile of per-command durations over a 5-minute window, per node.\n**Reading it:** Read-only lookups should be fast; heavy commands like path finding are naturally slower.\n**Healthy range:** Sub-second for most commands; command-dependent.\n**Watch for:** Rising p95 on a normally fast command signals contention or an expensive query pattern.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le, command, service_instance_id) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\"}[5m])))", + "interval": "15s", + "legendFormat": "P95 {{command}} [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "axisLabel": "Latency (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + } + }, + "overrides": [] + }, + "id": 5 + }, { "title": "RPC Latency Heatmap", "description": "**What:** Distribution of RPC response times over time across all commands.\n**How it's computed:** Counts of requests by duration band in each 5-minute window, per node.\n**Reading it:** A single tight band is healthy; brighter cells mark the common latency.\n**Healthy range:** Concentrated at low latency; command-dependent.\n**Watch for:** Two separate bands (bimodal latency) reveal a slow path affecting some requests.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", @@ -159,7 +233,7 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 16 }, "options": { "tooltip": { @@ -191,7 +265,7 @@ } } }, - "id": 4 + "id": 6 }, { "title": "Overall RPC Throughput", @@ -201,7 +275,7 @@ "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 16 }, "options": { "tooltip": { @@ -244,7 +318,7 @@ }, "overrides": [] }, - "id": 5 + "id": 7 }, { "title": "RPC Success vs Error", @@ -254,7 +328,7 @@ "h": 8, "w": 12, "x": 0, - "y": 20 + "y": 24 }, "options": { "tooltip": { @@ -297,80 +371,6 @@ }, "overrides": [] }, - "id": 6 - }, - { - "title": "Top Commands by Volume [$xrpl_network_type]", - "description": "**What:** The ten most-called RPC commands by total invocations recently.\n**How it's computed:** Ranked total counts per command over a 5-minute window, per node.\n**Reading it:** Identifies the hottest API endpoints driving node load.\n**Healthy range:** Workload-dependent.\n**Watch for:** An unexpected command dominating the ranking can indicate scripted abuse or a misbehaving client.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", - "type": "bargauge", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 28 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "topk(10, sum by (command, service_instance_id) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval])))", - "interval": "15s", - "legendFormat": "{{command}} [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "none" - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, - "id": 7 - }, - { - "title": "WebSocket Message Rate [$xrpl_network_type]", - "description": "**What:** Rate of incoming WebSocket RPC messages the server processes.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Non-zero only when clients connect over WebSocket rather than HTTP.\n**Healthy range:** Zero is normal for HTTP-only deployments; otherwise workload-dependent.\n**Watch for:** An unexpected surge points to a chatty or abusive WebSocket client.\n**Source:** src/xrpld/rpc/detail/ServerHandler.cpp:ServerHandler::processSession", - "type": "stat", - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 28 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=\"rpc.ws_message\"}[$__rate_interval]))", - "interval": "15s", - "legendFormat": "WS Messages / Sec [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "suffix: messages/s" - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, "id": 8 }, { diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json index 27e7254194..565edda59a 100644 --- a/docker/telemetry/grafana/dashboards/transaction-overview.json +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -8,6 +8,60 @@ "id": null, "links": [], "panels": [ + { + "title": "Transaction Apply Failed Rate [$xrpl_network_type]", + "description": "**What:** Rate of transactor-stage applies that returned a non-success result while building a ledger.\n**How it's computed:** Per-second rate of tx.transactor spans at stage=apply whose ter_result is not tesSUCCESS, per node.\n**Reading it:** Green below 1/s, yellow 1-5/s, red above 5/s.\n**Healthy range:** A steady low non-zero rate is normal (e.g. tefPAST_SEQ, tecUNFUNDED and other benign conflicts).\n**Watch for:** A sustained jump above the normal baseline can indicate malformed transaction floods or a systemic issue.\n**Source:** src/libxrpl/tx/Transactor.cpp:Transactor::operator()", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"tx.transactor\", stage=\"apply\", ter_result!~\"tesSUCCESS|\"}[$__rate_interval]))", + "interval": "15s", + "legendFormat": "Failed / Sec [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: transactions/s", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + } + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 1 + }, { "title": "Transaction Processing Rate", "description": "**What:** Rate of transactions entering the processing pipeline (submitted locally or relayed by peers) versus raw transaction messages arriving from peers before deduplication.\n**How it's computed:** Per-second rate of each over a 5-minute window, per node.\n**Reading it:** The received line sits above the processed line by the volume of duplicates filtered out.\n**Healthy range:** Workload-dependent; scales with network transaction volume.\n**Watch for:** A large and growing gap means heavy duplicate traffic; a processed-rate collapse means submissions are not being handled.\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp:NetworkOPsImp::processTransaction, src/xrpld/overlay/detail/PeerImp.cpp:PeerImp::handleTransaction", @@ -16,7 +70,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 4 }, "options": { "tooltip": { @@ -59,7 +113,7 @@ }, "overrides": [] }, - "id": 1 + "id": 2 }, { "title": "Transaction Processing Latency by Type", @@ -69,7 +123,7 @@ "h": 8, "w": 24, "x": 0, - "y": 8 + "y": 12 }, "options": { "tooltip": { @@ -117,7 +171,7 @@ }, "overrides": [] }, - "id": 2 + "id": 3 }, { "title": "Transaction Path Distribution", @@ -127,7 +181,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 20 }, "options": { "tooltip": { @@ -151,7 +205,7 @@ }, "overrides": [] }, - "id": 3 + "id": 4 }, { "title": "Transaction Receive vs Suppressed", @@ -161,7 +215,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 20 }, "options": { "tooltip": { @@ -196,7 +250,7 @@ }, "overrides": [] }, - "id": 4 + "id": 5 }, { "title": "Transaction Processing Duration Heatmap", @@ -206,7 +260,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 28 }, "options": { "tooltip": { @@ -238,7 +292,7 @@ } } }, - "id": 5 + "id": 6 }, { "title": "Transaction Apply Duration per Ledger", @@ -248,7 +302,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 28 }, "options": { "tooltip": { @@ -291,60 +345,6 @@ }, "overrides": [] }, - "id": 6 - }, - { - "title": "Transaction Apply Failed Rate [$xrpl_network_type]", - "description": "**What:** Rate of transactor-stage applies that returned a non-success result while building a ledger.\n**How it's computed:** Per-second rate of tx.transactor spans at stage=apply whose ter_result is not tesSUCCESS, per node.\n**Reading it:** Green below 1/s, yellow 1-5/s, red above 5/s.\n**Healthy range:** A steady low non-zero rate is normal (e.g. tefPAST_SEQ, tecUNFUNDED and other benign conflicts).\n**Watch for:** A sustained jump above the normal baseline can indicate malformed transaction floods or a systemic issue.\n**Source:** src/libxrpl/tx/Transactor.cpp:Transactor::operator()", - "type": "stat", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 32 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc", - "maxHeight": 600 - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"tx.transactor\", stage=\"apply\", ter_result!~\"tesSUCCESS|\"}[$__rate_interval]))", - "interval": "15s", - "legendFormat": "Failed / Sec [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "suffix: transactions/s", - "thresholds": { - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 5 - } - ] - } - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, "id": 7 }, { From 0accb580f8c2ee2d7b36ec967d3abd45bcdf09ef Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:57:09 +0100 Subject: [PATCH 4/4] style(telemetry): lead phase-9 rows with single-value panels, fix two labels Applies the guideline-8 ordering to the two dashboards introduced on this branch, and clears the last two labelling gaps found while auditing the set: - peer-quality, validator-health: stats and bar gauges now lead their row section instead of trailing the charts, matching the other dashboards. - node-health "Peer Disconnects (Resources)": had an empty axisLabel, the only chart in the set without one. Now reads "Disconnects". - rpc-performance row "Aggregate RPC (all commands)" -> "(All Commands)", the one title that was not in Title Case. Reordering stays inside each row section so no panel changes category, and the panel objects are cut and re-spliced as raw text, so their contents are byte-identical. Verified against origin/phase9: no panel lost, every targets block unchanged, ids 1..N, and every row section now leads with its single-value panels. --- .../grafana/dashboards/node-health.json | 2 +- .../grafana/dashboards/peer-quality.json | 102 +++++++------- .../grafana/dashboards/rpc-performance.json | 2 +- .../grafana/dashboards/validator-health.json | 124 +++++++++--------- 4 files changed, 115 insertions(+), 115 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index edb1cad3ac..1a744439ae 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -4481,7 +4481,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "Disconnects", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, diff --git a/docker/telemetry/grafana/dashboards/peer-quality.json b/docker/telemetry/grafana/dashboards/peer-quality.json index 2da2cfbd12..ad4bc0c526 100644 --- a/docker/telemetry/grafana/dashboards/peer-quality.json +++ b/docker/telemetry/grafana/dashboards/peer-quality.json @@ -327,56 +327,6 @@ "panels": [], "id": 7 }, - { - "title": "Resource Disconnects", - "description": "###### What this is:\n*Cumulative count of peers dropped for exceeding resource (load) limits.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the resource-disconnect total over time.*\n\n###### Reading it:\n*A flat line is healthy; a rising line means peers are being dropped for overuse.*\n\n###### Healthy range:\n*Flat / near constant.*\n\n###### Watch for:\n*A steep climb, which flags aggressive or misbehaving peers being shed as backpressure.*\n\n###### Keywords:\n- **Resource disconnect** *(per node)* \u2014 a peer dropped for exceeding resource/load limits \u2014 the node shedding abusive or overactive peers.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#resource-disconnect)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 19 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(server_info{metric=\"peer_disconnects_resources\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Disconnects\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "short", - "custom": { - "axisLabel": "Disconnects", - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 0, - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 5, - "gradientMode": "none" - }, - "color": { - "mode": "palette-classic" - }, - "decimals": 0 - }, - "overrides": [] - }, - "id": 8 - }, { "title": "Inbound vs Outbound Peers [$xrpl_network_type]", "description": "###### What this is:\n*Active inbound versus outbound peer connection counts.*\n\n###### How it's computed:\n*Instantaneous gauge readings of active inbound and outbound peer counts.*\n\n###### Reading it:\n*A balanced mix is healthy for connectivity and resilience.*\n\n###### Healthy range:\n*A mix of both; some outbound connections present.*\n\n###### Watch for:\n*All-inbound with no outbound, which usually means NAT or firewall issues block outbound peering.*\n\n###### Keywords:\n- **Peer** *(per node)* \u2014 another server this node holds a protocol connection to.\n- **Overlay** *(network-wide)* \u2014 the peer-to-peer mesh xrpld nodes form to gossip transactions, proposals and validations.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PeerfinderManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/peerfinder/detail/PeerfinderManager.cpp)\n\n###### Function:\n`Logic Stats ctor`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#peer)", @@ -385,7 +335,7 @@ "h": 4, "w": 12, "x": 0, - "y": 27 + "y": 19 }, "options": { "orientation": "horizontal", @@ -463,6 +413,56 @@ "repeat": "xrpl_network_type", "repeatDirection": "h", "maxPerRow": 2, + "id": 8 + }, + { + "title": "Resource Disconnects", + "description": "###### What this is:\n*Cumulative count of peers dropped for exceeding resource (load) limits.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the resource-disconnect total over time.*\n\n###### Reading it:\n*A flat line is healthy; a rising line means peers are being dropped for overuse.*\n\n###### Healthy range:\n*Flat / near constant.*\n\n###### Watch for:\n*A steep climb, which flags aggressive or misbehaving peers being shed as backpressure.*\n\n###### Keywords:\n- **Resource disconnect** *(per node)* \u2014 a peer dropped for exceeding resource/load limits \u2014 the node shedding abusive or overactive peers.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#resource-disconnect)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(server_info{metric=\"peer_disconnects_resources\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Disconnects\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Disconnects", + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "gradientMode": "none" + }, + "color": { + "mode": "palette-classic" + }, + "decimals": 0 + }, + "overrides": [] + }, "id": 9 } ], diff --git a/docker/telemetry/grafana/dashboards/rpc-performance.json b/docker/telemetry/grafana/dashboards/rpc-performance.json index c85d252d1d..3fed35f9dc 100644 --- a/docker/telemetry/grafana/dashboards/rpc-performance.json +++ b/docker/telemetry/grafana/dashboards/rpc-performance.json @@ -41,7 +41,7 @@ { "id": 1, "type": "row", - "title": "Aggregate RPC (all commands)", + "title": "Aggregate RPC (All Commands)", "collapsed": false, "gridPos": { "h": 1, diff --git a/docker/telemetry/grafana/dashboards/validator-health.json b/docker/telemetry/grafana/dashboards/validator-health.json index 3a0cde97bc..870eb6c4d9 100644 --- a/docker/telemetry/grafana/dashboards/validator-health.json +++ b/docker/telemetry/grafana/dashboards/validator-health.json @@ -739,55 +739,6 @@ "maxPerRow": 2, "id": 14 }, - { - "title": "State Value Timeline", - "description": "###### What this is:\n*Numeric encoding of the server operating state (disconnected, connected, syncing, tracking, full, validating, proposing) over time.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the encoded state value.*\n\n###### Reading it:\n*A flat line at the full-operation state is healthy; steps show transitions.*\n\n###### Healthy range:\n*Steady at the highest (full) state.*\n\n###### Watch for:\n*Frequent transitions, useful for correlating state flapping with other metrics.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n- **Consensus mode** *(per node)* \u2014 the node's role/health in the current round: Proposing, Observing, Wrong Ledger, or Switched Ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerStateTrackingGauge`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Consensus mode](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 35 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(state_tracking{metric=\"state_value\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"State\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "short", - "custom": { - "axisLabel": "State", - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 0, - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 5, - "gradientMode": "none" - }, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "id": 15 - }, { "title": "Time in Current State [$xrpl_network_type]", "description": "###### What this is:\n*How long the server has held its current operating state, in seconds.*\n\n###### How it's computed:\n*Current value of the time-in-state gauge.*\n\n###### Reading it:\n*Not yet wired in the code; the value currently always reads 0.*\n\n###### Healthy range:\n*Not applicable; the value is always 0 today.*\n\n###### Watch for:\n*n/a until implemented.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerStateTrackingGauge`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", @@ -795,8 +746,8 @@ "gridPos": { "h": 4, "w": 12, - "x": 0, - "y": 43 + "x": 12, + "y": 31 }, "options": { "tooltip": { @@ -825,7 +776,7 @@ "repeat": "xrpl_network_type", "repeatDirection": "h", "maxPerRow": 2, - "id": 16 + "id": 15 }, { "title": "State Changes Rate [$xrpl_network_type]", @@ -834,8 +785,8 @@ "gridPos": { "h": 4, "w": 12, - "x": 12, - "y": 43 + "x": 0, + "y": 35 }, "options": { "tooltip": { @@ -880,7 +831,7 @@ "repeat": "xrpl_network_type", "repeatDirection": "h", "maxPerRow": 2, - "id": 17 + "id": 16 }, { "title": "Ledgers Closed Rate [$xrpl_network_type]", @@ -889,8 +840,8 @@ "gridPos": { "h": 4, "w": 12, - "x": 0, - "y": 47 + "x": 12, + "y": 35 }, "options": { "tooltip": { @@ -935,7 +886,7 @@ "repeat": "xrpl_network_type", "repeatDirection": "h", "maxPerRow": 2, - "id": 18 + "id": 17 }, { "title": "Agreement % (7d) [$xrpl_network_type]", @@ -944,8 +895,8 @@ "gridPos": { "h": 4, "w": 12, - "x": 12, - "y": 47 + "x": 0, + "y": 39 }, "options": { "tooltip": { @@ -992,6 +943,55 @@ "repeat": "xrpl_network_type", "repeatDirection": "h", "maxPerRow": 2, + "id": 18 + }, + { + "title": "State Value Timeline", + "description": "###### What this is:\n*Numeric encoding of the server operating state (disconnected, connected, syncing, tracking, full, validating, proposing) over time.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the encoded state value.*\n\n###### Reading it:\n*A flat line at the full-operation state is healthy; steps show transitions.*\n\n###### Healthy range:\n*Steady at the highest (full) state.*\n\n###### Watch for:\n*Frequent transitions, useful for correlating state flapping with other metrics.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n- **Consensus mode** *(per node)* \u2014 the node's role/health in the current round: Proposing, Observing, Wrong Ledger, or Switched Ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerStateTrackingGauge`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Consensus mode](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 43 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(state_tracking{metric=\"state_value\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"State\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "State", + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "gradientMode": "none" + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, "id": 19 }, { @@ -1001,8 +1001,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 51 + "x": 12, + "y": 43 }, "options": { "tooltip": {