From 96ccbad5af26a742dbcfbd0126392c9072f36429 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:57:53 +0100 Subject: [PATCH] fix(telemetry): finish the count conversion and split panel 66's two axes Two panels were left out of the previous sweep because a blanket rate-to-count rewrite would have been wrong for them. Handled per target here. Panel 65 (Sweep Heap-Trim) carries two cumulative counters, minor faults and reclaimed KB. Both are count-shaped, so it becomes a bargauge over round(increase(...[$__range])) like the rest, and the legends drop their "/ Sec" suffix now that the values are totals rather than rates. Panel 66 (Online-Delete Rotation) stays a timeseries. Its two targets are not the same kind of thing: target A reads rotation_state{in_flight}, a 0/1 flag whose whole value is seeing when it is high and for how long, and target B rates rotation_state{copy_forward}, a cumulative write total. A count bargauge would destroy the flag's time dimension. Instead the shared "cps" unit -- wrong for a flag -- is replaced by per-target overrides: the flag pinned to a 0..1 left axis, the write rate on a right axis in cps. This matches how the metric is documented to be read (MetricNames.h:648): copy_forward climbing while in_flight is 1 is expected, climbing while it is 0 means the flag leaked. Verified against Grafana Cloud Prometheus: 0 parse errors on both panels. Co-Authored-By: Claude Opus 5 (1M context) --- .../dashboards/ledger-sync-health.json | 132 ++++++++++-------- 1 file changed, 75 insertions(+), 57 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/ledger-sync-health.json b/docker/telemetry/grafana/dashboards/ledger-sync-health.json index 75fa4e55d4..71f244a992 100644 --- a/docker/telemetry/grafana/dashboards/ledger-sync-health.json +++ b/docker/telemetry/grafana/dashboards/ledger-sync-health.json @@ -3976,40 +3976,6 @@ "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Faults & KB / Sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", "thresholds": { "mode": "absolute", @@ -4020,8 +3986,10 @@ } ] }, - "unit": "cps" - } + "unit": "short", + "decimals": 0 + }, + "overrides": [] }, "gridPos": { "h": 12, @@ -4031,24 +3999,26 @@ }, "id": 65, "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, + "displayMode": "gradient", "legend": { "calcs": [], "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", "placement": "bottom", - "showLegend": true + "showLegend": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" }, "pluginVersion": "13.2.0-28926505616", "targets": [ @@ -4057,20 +4027,22 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(sweep_malloc_trim_minor_faults_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Minor Faults / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(sweep_malloc_trim_minor_faults_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range]))), \"series\", \"Minor Faults\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(sweep_malloc_trim_reclaimed_kb_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Reclaimed KB / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "B" + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(sweep_malloc_trim_reclaimed_kb_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range]))), \"series\", \"Reclaimed KB\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B", + "instant": true } ], - "title": "Sweep Heap-Trim Faults & Reclaim Rate", - "type": "timeseries" + "title": "Sweep Heap-Trim Faults & Reclaim (Count)", + "type": "bargauge" }, { "datasource": { @@ -4127,8 +4099,54 @@ } ] }, - "unit": "cps" - } + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*Rotation In Flight.*" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "left" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + }, + { + "id": "custom.axisLabel", + "value": "In Flight (0/1)" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Copy-Forward.*" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "cps" + }, + { + "id": "custom.axisLabel", + "value": "Writes / Sec" + } + ] + } + ] }, "gridPos": { "h": 12,