From 62ef753804512b7ae649ee5bc56a727b01b9627e Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Thu, 20 Aug 2026 18:33:19 +0100 Subject: [PATCH] feat(telemetry): split perf-run annotations by load driver A single "Annotate perf-iac runs" layer matched only `perf-iac`, so a Locust load window was indistinguishable from a JMeter one. perf-iac now tags every region with its load driver, so each driver can have its own layer and colour. - Replace that layer with `Perf Runs (JMeter)` and `Perf Runs (Locust)`, each matching ["perf-iac", ""] with matchAny:false, on 12 dashboards. - job-queue, ledger-data-sync and log-derived-insights had an empty annotations list and drew no perf regions at all; they now carry the builtIn layer plus both driver layers. - Grafana tag matching is a superset AND with no negation, so a generic `perf-iac` layer also matches every driver region. Keeping one alongside the driver layers would draw each load window twice, so it is replaced, not kept. - Document the layers in the telemetry runbook, including two rendering limits: annotations draw only on timeseries, state-timeline and candlestick panels, and the shaded fill is 10% opacity so the region edges carry the colour. - Add `jmeter` to the cspell dictionary; the hook rejects the bare word. --- .cspell.config.yaml | 1 + .../grafana/dashboards/consensus-health.json | 23 +++++++-- .../grafana/dashboards/fee-market.json | 23 +++++++-- .../grafana/dashboards/job-queue.json | 49 ++++++++++++++++++- .../grafana/dashboards/ledger-data-sync.json | 49 ++++++++++++++++++- .../grafana/dashboards/ledger-operations.json | 23 +++++++-- .../dashboards/log-derived-insights.json | 49 ++++++++++++++++++- .../grafana/dashboards/network-traffic.json | 23 +++++++-- .../grafana/dashboards/node-health.json | 23 +++++++-- .../dashboards/overlay-traffic-detail.json | 23 +++++++-- .../grafana/dashboards/peer-network.json | 23 +++++++-- .../grafana/dashboards/peer-quality.json | 23 +++++++-- .../grafana/dashboards/rpc-pathfinding.json | 23 +++++++-- .../grafana/dashboards/rpc-performance.json | 23 +++++++-- .../dashboards/transaction-overview.json | 23 +++++++-- .../grafana/dashboards/validator-health.json | 23 +++++++-- docs/telemetry-runbook.md | 45 +++++++++++++++++ 17 files changed, 430 insertions(+), 39 deletions(-) diff --git a/.cspell.config.yaml b/.cspell.config.yaml index b8c5ea1d29..5d09c900f0 100644 --- a/.cspell.config.yaml +++ b/.cspell.config.yaml @@ -163,6 +163,7 @@ words: - itype - jemalloc - jlog + - jmeter - jtnofill - keylet - keylets diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json index 2dc19084e3..2871889a0d 100644 --- a/docker/telemetry/grafana/dashboards/consensus-health.json +++ b/docker/telemetry/grafana/dashboards/consensus-health.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/fee-market.json b/docker/telemetry/grafana/dashboards/fee-market.json index 71dd0ba6ed..3e5493e829 100644 --- a/docker/telemetry/grafana/dashboards/fee-market.json +++ b/docker/telemetry/grafana/dashboards/fee-market.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/job-queue.json b/docker/telemetry/grafana/dashboards/job-queue.json index 52319e1df1..286b732de0 100644 --- a/docker/telemetry/grafana/dashboards/job-queue.json +++ b/docker/telemetry/grafana/dashboards/job-queue.json @@ -1,6 +1,53 @@ { "annotations": { - "list": [] + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], + "type": "tags" + }, + "type": "tags" + } + ] }, "description": "What this shows: Per-job-type throughput, queue wait times, and execution times for the node's internal worker job queue.\nUse it to: Find job types that are backing up or running slowly and causing internal processing delays.", "editable": true, diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index 5b2d4ed048..97d1209aae 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -1,6 +1,53 @@ { "annotations": { - "list": [] + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], + "type": "tags" + }, + "type": "tags" + } + ] }, "description": "What this shows: Ledger data exchange and object-fetch traffic between this node and its peers: ledger sync, tree-node retrieval, and transaction-set exchange.\nUse it to: See how much ledger data the node is pulling or serving and spot catch-up activity.", "editable": true, diff --git a/docker/telemetry/grafana/dashboards/ledger-operations.json b/docker/telemetry/grafana/dashboards/ledger-operations.json index 9cc7726cb0..84f7d8292e 100644 --- a/docker/telemetry/grafana/dashboards/ledger-operations.json +++ b/docker/telemetry/grafana/dashboards/ledger-operations.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/log-derived-insights.json b/docker/telemetry/grafana/dashboards/log-derived-insights.json index 6acda66c69..f4c085ba3d 100644 --- a/docker/telemetry/grafana/dashboards/log-derived-insights.json +++ b/docker/telemetry/grafana/dashboards/log-derived-insights.json @@ -14,7 +14,54 @@ "uid": "log-derived-insights", "refresh": "60s", "annotations": { - "list": [] + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], + "type": "tags" + }, + "type": "tags" + } + ] }, "templating": { "list": [ diff --git a/docker/telemetry/grafana/dashboards/network-traffic.json b/docker/telemetry/grafana/dashboards/network-traffic.json index f258ff8db1..8fc1e9618e 100644 --- a/docker/telemetry/grafana/dashboards/network-traffic.json +++ b/docker/telemetry/grafana/dashboards/network-traffic.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index 2c50b0869d..5b10fa83a2 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json index df48858eab..62ffd6d7e5 100644 --- a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json +++ b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/peer-network.json b/docker/telemetry/grafana/dashboards/peer-network.json index ca4d55239a..94d4a3d3a6 100644 --- a/docker/telemetry/grafana/dashboards/peer-network.json +++ b/docker/telemetry/grafana/dashboards/peer-network.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/peer-quality.json b/docker/telemetry/grafana/dashboards/peer-quality.json index 7dbf311418..bae6b2ed21 100644 --- a/docker/telemetry/grafana/dashboards/peer-quality.json +++ b/docker/telemetry/grafana/dashboards/peer-quality.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json index b751c16959..f1ebe0d30a 100644 --- a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json +++ b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/rpc-performance.json b/docker/telemetry/grafana/dashboards/rpc-performance.json index 8228c9283b..5f2ff427f2 100644 --- a/docker/telemetry/grafana/dashboards/rpc-performance.json +++ b/docker/telemetry/grafana/dashboards/rpc-performance.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json index f5e6820a81..12e6c32fc5 100644 --- a/docker/telemetry/grafana/dashboards/transaction-overview.json +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/validator-health.json b/docker/telemetry/grafana/dashboards/validator-health.json index 15254cb9ca..f08abd0932 100644 --- a/docker/telemetry/grafana/dashboards/validator-health.json +++ b/docker/telemetry/grafana/dashboards/validator-health.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 5fe5ef5433..c88c8dd1c8 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -1985,6 +1985,51 @@ attributes from their own alloy pipeline. Outside those runs the labels are absent; leaving the filters on **All** keeps every dashboard rendering normally. +### Perf Run Annotations + +Perf load windows are drawn on the dashboards as shaded region annotations +rather than single markers. perf-iac's +`.github/scripts/post_grafana_annotation.sh` opens an annotation when a load +phase starts and closes it with an end time when that phase finishes, so the +shaded band covers exactly the interval over which the load was applied. + +Every dashboard carries two tag-matched annotation layers, one per load driver: + +| Layer | Tags | Color | +| -------------------- | --------------------- | ------ | +| `Perf Runs (JMeter)` | `perf-iac` + `jmeter` | violet | +| `Perf Runs (Locust)` | `perf-iac` + `locust` | aqua | + +Both layers set `matchAny: false`, so a region is drawn only if it carries +**both** of the layer's tags — the tag list is an AND, not an OR. Grafana tag +matching is a superset AND-match with no negation, so a layer listing only +`perf-iac` would also match every driver region, and "`perf-iac` but neither +driver" cannot be expressed at all. That is why there is no catch-all layer +beside these two: a generic layer could only ever redraw the same regions the +driver layers already show, giving two overlapping bands and two tooltips for +one load window. + +JMeter posts **two** regions per load job, one around the warm-up phase and one +around the measured phase. Locust posts **one**, for the measured phase only, +because it has no warm-up step — so a Locust leg shows a single band where a +JMeter leg shows two. + +The driver tag is not something a run supplies. Each load workflow hardcodes it +as a `LOAD_DRIVER` environment value (`reusable-jmeter-test.yml` sets `jmeter`, +`reusable-locust-test.yml` sets `locust`), so it is never a dispatch input and no +current workflow can omit it; the script warns in CI if one ever does. Alongside +the driver, each region also carries the ticket (work item), the side (`test` or +`baseline`), the ref, the commit, and the phase; blank values are dropped. The +tooltip lists those, which is how one band is told from another when several runs +overlap. The driver is carried only as a tag, not in the tooltip — which layer +drew the band is what identifies it. + +Two rendering limits are worth knowing. Grafana draws annotations only on time +series, state timeline and candlestick panels, so on a board of mostly stats and +gauges most panels show no band. And the shaded fill is rendered at 10% opacity, +so the two drivers' colours are near-identical inside the band; the region's two +full-colour dashed edges and the toolbar toggles are what tell them apart. + ### Who owns which attribute - **Node and service** come from xrpld config (`service_instance_id`,