diff --git a/.cspell.config.yaml b/.cspell.config.yaml index d20b1ae7b3..01a51ea0e6 100644 --- a/.cspell.config.yaml +++ b/.cspell.config.yaml @@ -164,6 +164,7 @@ words: - itype - jemalloc - jlog + - jmeter - jtnofill - keylet - keylets diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json index 2dc19084e3..2871889a0d 100644 --- a/docker/telemetry/grafana/dashboards/consensus-health.json +++ b/docker/telemetry/grafana/dashboards/consensus-health.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/fee-market.json b/docker/telemetry/grafana/dashboards/fee-market.json index 71dd0ba6ed..3e5493e829 100644 --- a/docker/telemetry/grafana/dashboards/fee-market.json +++ b/docker/telemetry/grafana/dashboards/fee-market.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/job-queue.json b/docker/telemetry/grafana/dashboards/job-queue.json index 52319e1df1..286b732de0 100644 --- a/docker/telemetry/grafana/dashboards/job-queue.json +++ b/docker/telemetry/grafana/dashboards/job-queue.json @@ -1,6 +1,53 @@ { "annotations": { - "list": [] + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], + "type": "tags" + }, + "type": "tags" + } + ] }, "description": "What this shows: Per-job-type throughput, queue wait times, and execution times for the node's internal worker job queue.\nUse it to: Find job types that are backing up or running slowly and causing internal processing delays.", "editable": true, diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index 5b2d4ed048..97d1209aae 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -1,6 +1,53 @@ { "annotations": { - "list": [] + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], + "type": "tags" + }, + "type": "tags" + } + ] }, "description": "What this shows: Ledger data exchange and object-fetch traffic between this node and its peers: ledger sync, tree-node retrieval, and transaction-set exchange.\nUse it to: See how much ledger data the node is pulling or serving and spot catch-up activity.", "editable": true, diff --git a/docker/telemetry/grafana/dashboards/ledger-operations.json b/docker/telemetry/grafana/dashboards/ledger-operations.json index 9cc7726cb0..84f7d8292e 100644 --- a/docker/telemetry/grafana/dashboards/ledger-operations.json +++ b/docker/telemetry/grafana/dashboards/ledger-operations.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/log-derived-insights.json b/docker/telemetry/grafana/dashboards/log-derived-insights.json index 6acda66c69..f4c085ba3d 100644 --- a/docker/telemetry/grafana/dashboards/log-derived-insights.json +++ b/docker/telemetry/grafana/dashboards/log-derived-insights.json @@ -14,7 +14,54 @@ "uid": "log-derived-insights", "refresh": "60s", "annotations": { - "list": [] + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], + "type": "tags" + }, + "type": "tags" + } + ] }, "templating": { "list": [ diff --git a/docker/telemetry/grafana/dashboards/network-traffic.json b/docker/telemetry/grafana/dashboards/network-traffic.json index f258ff8db1..8fc1e9618e 100644 --- a/docker/telemetry/grafana/dashboards/network-traffic.json +++ b/docker/telemetry/grafana/dashboards/network-traffic.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index 8502de0fe7..48bc83c0ea 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json index df48858eab..62ffd6d7e5 100644 --- a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json +++ b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/peer-network.json b/docker/telemetry/grafana/dashboards/peer-network.json index ca4d55239a..94d4a3d3a6 100644 --- a/docker/telemetry/grafana/dashboards/peer-network.json +++ b/docker/telemetry/grafana/dashboards/peer-network.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/peer-quality.json b/docker/telemetry/grafana/dashboards/peer-quality.json index d45f516796..bf47b869fe 100644 --- a/docker/telemetry/grafana/dashboards/peer-quality.json +++ b/docker/telemetry/grafana/dashboards/peer-quality.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json index b751c16959..f1ebe0d30a 100644 --- a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json +++ b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/rpc-performance.json b/docker/telemetry/grafana/dashboards/rpc-performance.json index 8228c9283b..5f2ff427f2 100644 --- a/docker/telemetry/grafana/dashboards/rpc-performance.json +++ b/docker/telemetry/grafana/dashboards/rpc-performance.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json index f5e6820a81..12e6c32fc5 100644 --- a/docker/telemetry/grafana/dashboards/transaction-overview.json +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/validator-health.json b/docker/telemetry/grafana/dashboards/validator-health.json index 15254cb9ca..f08abd0932 100644 --- a/docker/telemetry/grafana/dashboards/validator-health.json +++ b/docker/telemetry/grafana/dashboards/validator-health.json @@ -20,12 +20,29 @@ }, "enable": true, "hide": false, - "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "iconColor": "rgb(144, 133, 233)", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(25, 158, 112)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index d63b10c74c..e9b1a12239 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -2012,6 +2012,51 @@ attributes from their own alloy pipeline. Outside those runs the labels are absent; leaving the filters on **All** keeps every dashboard rendering normally. +### Perf Run Annotations + +Perf load windows are drawn on the dashboards as shaded region annotations +rather than single markers. perf-iac's +`.github/scripts/post_grafana_annotation.sh` opens an annotation when a load +phase starts and closes it with an end time when that phase finishes, so the +shaded band covers exactly the interval over which the load was applied. + +Every dashboard carries two tag-matched annotation layers, one per load driver: + +| Layer | Tags | Color | +| -------------------- | --------------------- | ------ | +| `Perf Runs (JMeter)` | `perf-iac` + `jmeter` | violet | +| `Perf Runs (Locust)` | `perf-iac` + `locust` | aqua | + +Both layers set `matchAny: false`, so a region is drawn only if it carries +**both** of the layer's tags — the tag list is an AND, not an OR. Grafana tag +matching is a superset AND-match with no negation, so a layer listing only +`perf-iac` would also match every driver region, and "`perf-iac` but neither +driver" cannot be expressed at all. That is why there is no catch-all layer +beside these two: a generic layer could only ever redraw the same regions the +driver layers already show, giving two overlapping bands and two tooltips for +one load window. + +JMeter posts **two** regions per load job, one around the warm-up phase and one +around the measured phase. Locust posts **one**, for the measured phase only, +because it has no warm-up step — so a Locust leg shows a single band where a +JMeter leg shows two. + +The driver tag is not something a run supplies. Each load workflow hardcodes it +as a `LOAD_DRIVER` environment value (`reusable-jmeter-test.yml` sets `jmeter`, +`reusable-locust-test.yml` sets `locust`), so it is never a dispatch input and no +current workflow can omit it; the script warns in CI if one ever does. Alongside +the driver, each region also carries the ticket (work item), the side (`test` or +`baseline`), the ref, the commit, and the phase; blank values are dropped. The +tooltip lists those, which is how one band is told from another when several runs +overlap. The driver is carried only as a tag, not in the tooltip — which layer +drew the band is what identifies it. + +Two rendering limits are worth knowing. Grafana draws annotations only on time +series, state timeline and candlestick panels, so on a board of mostly stats and +gauges most panels show no band. And the shaded fill is rendered at 10% opacity, +so the two drivers' colours are near-identical inside the band; the region's two +full-colour dashed edges and the toolbar toggles are what tell them apart. + ### Who owns which attribute - **Node and service** come from xrpld config (`service_instance_id`,