From fc98dac0f1af79017bb4ec970fe5a3081ac9601a Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Wed, 23 Sep 2026 17:40:10 +0100 Subject: [PATCH] fix(telemetry): name each rule's query steps, and let email keep its own layout The alert notification ended with `Values: A=0 B=0 C=1`, the internal ids of the three query steps, which mean nothing to a reader. The previous attempt removed that line by replacing the whole email body with a custom template. That worked, but it threw away everything else Grafana's default email gives: bold section headings, a styled label table, annotations rendered as clickable links, and the View and Silence buttons. The result read as a flat block of text and raw URLs. Fix the line instead of the body. Rename all 13 rules' steps from A/B/C to names that say what they hold -- close_rate_5m, close_rate_now, is_stalled -- so the default body's value line reads close_rate_5m=0 close_rate_now=0 is_stalled=1. The threshold step is named is_* so a 0/1 verdict is obviously a verdict and not a rate. Drop the email subject/message override so the default layout returns, and drop the now-unused xrpld.email.body template. Slack keeps its own body: Grafana escapes HTML in an email message, so a custom email body can only ever be plain text, while Slack needs mrkdwn and links. One body cannot serve both, and only Slack benefits from having one at all. Verified by provisioning the real files into Grafana: all 13 rules load with the new step names and evaluate health=ok, and the Slack template renders with no parse error. --- .../provisioning/alerting/contactpoints.yaml | 18 +- .../grafana/provisioning/alerting/rules.yaml | 260 +++++++++--------- .../provisioning/alerting/templates.yaml | 45 +-- 3 files changed, 151 insertions(+), 172 deletions(-) diff --git a/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml b/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml index 2fd6ce6a52..b905c889f4 100644 --- a/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml +++ b/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml @@ -96,16 +96,14 @@ contactPoints: addresses: alerts-disabled@xrpld.invalid # One message listing all recipients, rather than one message each. singleEmail: true - # Setting `message` replaces Grafana's default email body, which is - # what removes the raw value dump and the label list. Grafana's own - # header ("N firing alert instances", "Grouped by") and footer are - # fixed chrome and stay. - # - # This uses the EMAIL body, not the Slack one. Email escapes HTML and - # cannot render Slack markup, so a shared body arrives as literal - # asterisks and raw URLs. See templates.yaml. - subject: '{{ template "xrpld.title" . }}' - message: '{{ template "xrpld.email.body" . }}' + # DELIBERATELY no `subject`/`message` override here, unlike the Slack + # receivers above. Grafana's default email body is the good one: bold + # section headings, a styled label table, annotations rendered as real + # clickable links, and the View/Silence buttons. Overriding `message` + # replaces all of that with plain text, because email escapes HTML, so + # a custom body cannot reproduce any of it. The value line the default + # body carries is made readable by NAMING each rule's query steps (see + # rules.yaml), not by replacing the body. disableResolveMessage: false # To retire a receiver that a running Grafana has already stored, uncomment diff --git a/docker/telemetry/grafana/provisioning/alerting/rules.yaml b/docker/telemetry/grafana/provisioning/alerting/rules.yaml index e6adaa7ee9..1d88132564 100644 --- a/docker/telemetry/grafana/provisioning/alerting/rules.yaml +++ b/docker/telemetry/grafana/provisioning/alerting/rules.yaml @@ -56,7 +56,7 @@ groups: # hence increase() over a wide window rather than a decaying rate(). - uid: xrpld-ledger-history-mismatch title: LedgerHistoryMismatch - condition: C + condition: is_mismatching for: 2m isPaused: true noDataState: NoData @@ -68,45 +68,45 @@ groups: summary: "Ledger history mismatch on {{ $labels.service_instance_id }}" description: >- Ledger history mismatches on {{ $labels.service_instance_id }} in the last - 15m: {{ printf "%.0f" $values.B.Value }} (threshold: more than 0). + 15m: {{ printf "%.0f" $values.mismatches_now.Value }} (threshold: more than 0). action: >- A built ledger diverged from the validated network chain, which can mean corrupt local history. Check byzantine ledger jumps and the node-store. data: - - refId: A + - refId: mismatches_15m relativeTimeRange: from: 1200 to: 0 datasourceUid: prometheus model: - refId: A + refId: mismatches_15m expr: sum by (service_instance_id) (increase(ledger_history_mismatch_total{service_name="xrpld"}[15m])) instant: true range: false intervalMs: 1000 maxDataPoints: 43200 - - refId: B + - refId: mismatches_now relativeTimeRange: from: 1200 to: 0 datasourceUid: __expr__ model: - refId: B + refId: mismatches_now type: reduce reducer: last - expression: A + expression: mismatches_15m datasource: type: __expr__ uid: __expr__ - - refId: C + - refId: is_mismatching relativeTimeRange: from: 1200 to: 0 datasourceUid: __expr__ model: - refId: C + refId: is_mismatching type: threshold - expression: B + expression: mismatches_now conditions: - evaluator: type: gt @@ -122,7 +122,7 @@ groups: # dead node trips the threshold instead of vanishing from the result. - uid: xrpld-ledger-close-stalled title: LedgerCloseStalled - condition: C + condition: is_stalled for: 3m isPaused: true noDataState: Alerting @@ -134,18 +134,18 @@ groups: summary: "Ledger closing stalled on {{ $labels.service_instance_id }}" description: >- Ledger close rate on {{ $labels.service_instance_id }} over 5m: {{ printf - "%.4f" $values.B.Value }} ledgers/s (threshold: below 0.001). + "%.4f" $values.close_rate_now.Value }} ledgers/s (threshold: below 0.001). action: >- Consensus or ledger advancement is stuck, or the process is gone. Check consensus round duration, worker pool saturation and peer supply. data: - - refId: A + - refId: close_rate_5m relativeTimeRange: from: 600 to: 0 datasourceUid: prometheus model: - refId: A + refId: close_rate_5m expr: |- sum by (service_instance_id) (rate(ledgers_closed_total{service_name="xrpld"}[5m])) or (0 * max by (service_instance_id) (max_over_time(ledgers_closed_total{service_name="xrpld"}[1h]))) @@ -153,28 +153,28 @@ groups: range: false intervalMs: 1000 maxDataPoints: 43200 - - refId: B + - refId: close_rate_now relativeTimeRange: from: 600 to: 0 datasourceUid: __expr__ model: - refId: B + refId: close_rate_now type: reduce reducer: last - expression: A + expression: close_rate_5m datasource: type: __expr__ uid: __expr__ - - refId: C + - refId: is_stalled relativeTimeRange: from: 600 to: 0 datasourceUid: __expr__ model: - refId: C + refId: is_stalled type: threshold - expression: B + expression: close_rate_now conditions: - evaluator: type: lt @@ -199,7 +199,7 @@ groups: # validated ledger is caught by LedgerCloseStalled and NodeNotFull. - uid: xrpld-validated-ledger-stale title: ValidatedLedgerStale - condition: C + condition: is_stale for: 5m isPaused: true noDataState: NoData @@ -211,45 +211,45 @@ groups: summary: "Validated ledger stale on {{ $labels.service_instance_id }}" description: >- Validated ledger age on {{ $labels.service_instance_id }}: {{ printf "%.0f" - $values.B.Value }}s (threshold: over 60s). + $values.ledger_age_now.Value }}s (threshold: over 60s). action: >- The node is no longer tracking the network. Check peer connectivity, node- store IO latency and consensus rounds. data: - - refId: A + - refId: ledger_age_s relativeTimeRange: from: 600 to: 0 datasourceUid: prometheus model: - refId: A + refId: ledger_age_s expr: max by (service_instance_id) (ledgermaster_validated_ledger_age{service_name="xrpld"} < 1209600) instant: true range: false intervalMs: 1000 maxDataPoints: 43200 - - refId: B + - refId: ledger_age_now relativeTimeRange: from: 600 to: 0 datasourceUid: __expr__ model: - refId: B + refId: ledger_age_now type: reduce reducer: last - expression: A + expression: ledger_age_s datasource: type: __expr__ uid: __expr__ - - refId: C + - refId: is_stale relativeTimeRange: from: 600 to: 0 datasourceUid: __expr__ model: - refId: C + refId: is_stale type: threshold - expression: B + expression: ledger_age_now conditions: - evaluator: type: gt @@ -280,7 +280,7 @@ groups: # disagreement among nodes that do validate. - uid: xrpld-validations-missed title: ValidationsMissed - condition: C + condition: is_missing_validations for: 15m isPaused: true noDataState: NoData @@ -292,18 +292,18 @@ groups: summary: "Validations missed on {{ $labels.service_instance_id }}" description: >- Missed-validation fraction on {{ $labels.service_instance_id }} over 15m: {{ - printf "%.3f" $values.B.Value }} (threshold: over 0.1). + printf "%.3f" $values.missed_ratio_now.Value }} (threshold: over 0.1). action: >- Its validations are not agreeing with the validated ledger, which risks removal from UNLs. data: - - refId: A + - refId: missed_ratio_15m relativeTimeRange: from: 1200 to: 0 datasourceUid: prometheus model: - refId: A + refId: missed_ratio_15m expr: |- ( sum by (service_instance_id) (rate(validation_missed_total{service_name="xrpld"}[15m])) @@ -318,28 +318,28 @@ groups: range: false intervalMs: 1000 maxDataPoints: 43200 - - refId: B + - refId: missed_ratio_now relativeTimeRange: from: 1200 to: 0 datasourceUid: __expr__ model: - refId: B + refId: missed_ratio_now type: reduce reducer: last - expression: A + expression: missed_ratio_15m datasource: type: __expr__ uid: __expr__ - - refId: C + - refId: is_missing_validations relativeTimeRange: from: 1200 to: 0 datasourceUid: __expr__ model: - refId: C + refId: is_missing_validations type: threshold - expression: B + expression: missed_ratio_now conditions: - evaluator: type: gt @@ -354,7 +354,7 @@ groups: # (see the LedgerCloseStalled comment). - uid: xrpld-validations-not-checked title: ValidationsNotChecked - condition: C + condition: is_not_checking for: 5m isPaused: true noDataState: Alerting @@ -366,18 +366,18 @@ groups: summary: "No validations checked on {{ $labels.service_instance_id }}" description: >- Validations-checked rate on {{ $labels.service_instance_id }} over 5m: {{ - printf "%.4f" $values.B.Value }} per s (threshold: below 0.001). + printf "%.4f" $values.check_rate_now.Value }} per s (threshold: below 0.001). action: >- The validation stream from peers may have stopped. Check peer count and the overlay. data: - - refId: A + - refId: check_rate_5m relativeTimeRange: from: 600 to: 0 datasourceUid: prometheus model: - refId: A + refId: check_rate_5m expr: |- sum by (service_instance_id) (rate(validations_checked_total{service_name="xrpld"}[5m])) or (0 * max by (service_instance_id) (max_over_time(validations_checked_total{service_name="xrpld"}[1h]))) @@ -385,28 +385,28 @@ groups: range: false intervalMs: 1000 maxDataPoints: 43200 - - refId: B + - refId: check_rate_now relativeTimeRange: from: 600 to: 0 datasourceUid: __expr__ model: - refId: B + refId: check_rate_now type: reduce reducer: last - expression: A + expression: check_rate_5m datasource: type: __expr__ uid: __expr__ - - refId: C + - refId: is_not_checking relativeTimeRange: from: 600 to: 0 datasourceUid: __expr__ model: - refId: C + refId: is_not_checking type: threshold - expression: B + expression: check_rate_now conditions: - evaluator: type: lt @@ -429,7 +429,7 @@ groups: # sparse counters). - uid: xrpld-jobqueue-tx-overflow title: JobQueueTxOverflow - condition: C + condition: is_overflowing for: 2m isPaused: true noDataState: NoData @@ -441,45 +441,45 @@ groups: summary: "Job queue transaction overflow on {{ $labels.service_instance_id }}" description: >- Transaction job-queue overflows on {{ $labels.service_instance_id }} in the - last 15m: {{ printf "%.0f" $values.B.Value }} (threshold: more than 0). + last 15m: {{ printf "%.0f" $values.overflows_now.Value }} (threshold: more than 0). action: >- Transactions are being dropped under load. Check job-queue depth and worker saturation. data: - - refId: A + - refId: overflows_15m relativeTimeRange: from: 1200 to: 0 datasourceUid: prometheus model: - refId: A + refId: overflows_15m expr: sum by (service_instance_id) (increase(jq_trans_overflow_total{service_name="xrpld"}[15m])) instant: true range: false intervalMs: 1000 maxDataPoints: 43200 - - refId: B + - refId: overflows_now relativeTimeRange: from: 1200 to: 0 datasourceUid: __expr__ model: - refId: B + refId: overflows_now type: reduce reducer: last - expression: A + expression: overflows_15m datasource: type: __expr__ uid: __expr__ - - refId: C + - refId: is_overflowing relativeTimeRange: from: 1200 to: 0 datasourceUid: __expr__ model: - refId: C + refId: is_overflowing type: threshold - expression: B + expression: overflows_now conditions: - evaluator: type: gt @@ -502,7 +502,7 @@ groups: # healthy p99 and fires only on genuine saturation, which is the intent. - uid: xrpld-jobqueue-latency-high title: JobQueueLatencyHigh - condition: C + condition: is_queue_slow for: 5m isPaused: true noDataState: NoData @@ -514,44 +514,44 @@ groups: summary: "Job queue latency high on {{ $labels.service_instance_id }}" description: >- p99 job-queue wait on {{ $labels.service_instance_id }} over 5m: {{ printf - "%.0f" $values.B.Value }}us (threshold: over 1000000us, i.e. 1s). + "%.0f" $values.queue_p99_us_now.Value }}us (threshold: over 1000000us, i.e. 1s). action: >- The node is saturated and jobs are backing up. Check worker pool saturation. data: - - refId: A + - refId: queue_p99_us_5m relativeTimeRange: from: 600 to: 0 datasourceUid: prometheus model: - refId: A + refId: queue_p99_us_5m expr: histogram_quantile(0.99, sum by (le, service_instance_id) (rate(job_queued_us_bucket{service_name="xrpld"}[5m]))) instant: true range: false intervalMs: 1000 maxDataPoints: 43200 - - refId: B + - refId: queue_p99_us_now relativeTimeRange: from: 600 to: 0 datasourceUid: __expr__ model: - refId: B + refId: queue_p99_us_now type: reduce reducer: last - expression: A + expression: queue_p99_us_5m datasource: type: __expr__ uid: __expr__ - - refId: C + - refId: is_queue_slow relativeTimeRange: from: 600 to: 0 datasourceUid: __expr__ model: - refId: C + refId: is_queue_slow type: threshold - expression: B + expression: queue_p99_us_now conditions: - evaluator: type: gt @@ -573,7 +573,7 @@ groups: # is 12.9ms, far below the threshold. - uid: xrpld-nodestore-io-latency-high title: NodeStoreIOLatencyHigh - condition: C + condition: is_io_slow for: 10m isPaused: true noDataState: NoData @@ -585,44 +585,44 @@ groups: summary: "Node store IO latency high on {{ $labels.service_instance_id }}" description: >- p95 node-store IO latency on {{ $labels.service_instance_id }} over 10m: {{ - printf "%.0f" $values.B.Value }}ms (threshold: over 1000ms). + printf "%.0f" $values.io_p95_ms_now.Value }}ms (threshold: over 1000ms). action: >- Check disk utilisation and whether the store is on a slow volume. data: - - refId: A + - refId: io_p95_ms_10m relativeTimeRange: from: 900 to: 0 datasourceUid: prometheus model: - refId: A + refId: io_p95_ms_10m expr: histogram_quantile(0.95, sum by (le, service_instance_id) (rate(ios_latency_milliseconds_bucket{service_name="xrpld"}[10m]))) instant: true range: false intervalMs: 1000 maxDataPoints: 43200 - - refId: B + - refId: io_p95_ms_now relativeTimeRange: from: 900 to: 0 datasourceUid: __expr__ model: - refId: B + refId: io_p95_ms_now type: reduce reducer: last - expression: A + expression: io_p95_ms_10m datasource: type: __expr__ uid: __expr__ - - refId: C + - refId: is_io_slow relativeTimeRange: from: 900 to: 0 datasourceUid: __expr__ model: - refId: C + refId: is_io_slow type: threshold - expression: B + expression: io_p95_ms_now conditions: - evaluator: type: gt @@ -657,7 +657,7 @@ groups: # the gate suppresses that first hour so a restart does not page. - uid: xrpld-node-state-flapping title: NodeStateFlapping - condition: C + condition: is_flapping for: 15m isPaused: true noDataState: OK @@ -669,20 +669,20 @@ groups: summary: "Node state flapping on {{ $labels.service_instance_id }}" description: >- FULL-state re-entries on {{ $labels.service_instance_id }} in the last hour: - {{ printf "%.0f" $values.B.Value }} (threshold: more than 0, past the first + {{ printf "%.0f" $values.full_transitions_now.Value }} (threshold: more than 0, past the first hour of uptime). action: >- The node is flapping out of sync rather than holding FULL. A likely cause is the online-delete rotation cache-freshen; check the rotation spans and the cache lock-hold peak. data: - - refId: A + - refId: full_transitions_1h relativeTimeRange: from: 3900 to: 0 datasourceUid: prometheus model: - refId: A + refId: full_transitions_1h expr: |- sum by (service_instance_id) (increase(state_accounting_full_transitions{service_name="xrpld"}[1h])) and on (service_instance_id) @@ -691,28 +691,28 @@ groups: range: false intervalMs: 1000 maxDataPoints: 43200 - - refId: B + - refId: full_transitions_now relativeTimeRange: from: 3900 to: 0 datasourceUid: __expr__ model: - refId: B + refId: full_transitions_now type: reduce reducer: last - expression: A + expression: full_transitions_1h datasource: type: __expr__ uid: __expr__ - - refId: C + - refId: is_flapping relativeTimeRange: from: 3900 to: 0 datasourceUid: __expr__ model: - refId: C + refId: is_flapping type: threshold - expression: B + expression: full_transitions_now conditions: - evaluator: type: gt @@ -728,7 +728,7 @@ groups: # SYNCING=2, TRACKING=3, FULL=4. - uid: xrpld-node-not-full title: NodeNotFull - condition: C + condition: is_not_full for: 15m isPaused: true noDataState: NoData @@ -740,19 +740,19 @@ groups: summary: "Node not in FULL state on {{ $labels.service_instance_id }}" description: >- Server state on {{ $labels.service_instance_id }} for the last 15m: {{ - printf "%.0f" $values.B.Value }} (threshold: below 4; 0=disconnected + printf "%.0f" $values.server_state_now.Value }} (threshold: below 4; 0=disconnected 1=connected 2=syncing 3=tracking 4=full). action: >- The node is not fully synced with the network. Check peer supply and sync progress. data: - - refId: A + - refId: server_state_code relativeTimeRange: from: 1200 to: 0 datasourceUid: prometheus model: - refId: A + refId: server_state_code expr: |- max by (service_instance_id) (server_info{service_name="xrpld", metric="server_state"}) and on (service_instance_id) @@ -761,28 +761,28 @@ groups: range: false intervalMs: 1000 maxDataPoints: 43200 - - refId: B + - refId: server_state_now relativeTimeRange: from: 1200 to: 0 datasourceUid: __expr__ model: - refId: B + refId: server_state_now type: reduce reducer: last - expression: A + expression: server_state_code datasource: type: __expr__ uid: __expr__ - - refId: C + - refId: is_not_full relativeTimeRange: from: 1200 to: 0 datasourceUid: __expr__ model: - refId: C + refId: is_not_full type: threshold - expression: B + expression: server_state_now conditions: - evaluator: type: lt @@ -814,7 +814,7 @@ groups: # burst (peaks of 5 and 11, lasting well under 10m) from paging. - uid: xrpld-manifest-job-convoy title: ManifestJobQueueConvoy - condition: C + condition: is_convoying for: 10m isPaused: true noDataState: NoData @@ -826,45 +826,45 @@ groups: summary: "Manifest job convoy on {{ $labels.service_instance_id }}" description: >- Manifest jobs waiting on {{ $labels.service_instance_id }} over 10m: {{ - printf "%.0f" $values.B.Value }} (threshold: more than 3). + printf "%.0f" $values.manifest_waiting_now.Value }} (threshold: more than 3). action: >- Peer manifest dumps are saturating the job pool and convoying on the manifest cache lock. data: - - refId: A + - refId: manifest_waiting_10m relativeTimeRange: from: 900 to: 0 datasourceUid: prometheus model: - refId: A + refId: manifest_waiting_10m expr: sum by (service_instance_id) (jobq_manifest_waiting{service_name="xrpld"}) instant: true range: false intervalMs: 1000 maxDataPoints: 43200 - - refId: B + - refId: manifest_waiting_now relativeTimeRange: from: 900 to: 0 datasourceUid: __expr__ model: - refId: B + refId: manifest_waiting_now type: reduce reducer: last - expression: A + expression: manifest_waiting_10m datasource: type: __expr__ uid: __expr__ - - refId: C + - refId: is_convoying relativeTimeRange: from: 900 to: 0 datasourceUid: __expr__ model: - refId: C + refId: is_convoying type: threshold - expression: B + expression: manifest_waiting_now conditions: - evaluator: type: gt @@ -893,7 +893,7 @@ groups: # alerted. ManifestJobQueueConvoy covers that window via the job pool. - uid: xrpld-manifest-flood-inbound title: ManifestFloodInbound - condition: C + condition: is_flooding for: 10m isPaused: true noDataState: NoData @@ -905,18 +905,18 @@ groups: summary: "Inbound manifest flood on {{ $labels.service_instance_id }}" description: >- Inbound manifest traffic on {{ $labels.service_instance_id }} over 10m: {{ - printf "%.0f" $values.B.Value }} B/s (threshold: over 524288 B/s, i.e. 512 + printf "%.0f" $values.manifest_bytes_s_now.Value }} B/s (threshold: over 524288 B/s, i.e. 512 KiB/s). action: >- A peer is flooding oversized TMManifests dumps. data: - - refId: A + - refId: manifest_bytes_s_10m relativeTimeRange: from: 900 to: 0 datasourceUid: prometheus model: - refId: A + refId: manifest_bytes_s_10m expr: |- sum by (service_instance_id) (rate(overhead_manifest_bytes_in{service_name="xrpld"}[10m])) and on (service_instance_id) @@ -925,28 +925,28 @@ groups: range: false intervalMs: 1000 maxDataPoints: 43200 - - refId: B + - refId: manifest_bytes_s_now relativeTimeRange: from: 900 to: 0 datasourceUid: __expr__ model: - refId: B + refId: manifest_bytes_s_now type: reduce reducer: last - expression: A + expression: manifest_bytes_s_10m datasource: type: __expr__ uid: __expr__ - - refId: C + - refId: is_flooding relativeTimeRange: from: 900 to: 0 datasourceUid: __expr__ model: - refId: C + refId: is_flooding type: threshold - expression: B + expression: manifest_bytes_s_now conditions: - evaluator: type: gt @@ -965,7 +965,7 @@ groups: # trips on a node already in trouble. - uid: xrpld-peer-resource-disconnects title: PeerResourceDisconnects - condition: C + condition: is_disconnecting for: 5m isPaused: true noDataState: NoData @@ -977,44 +977,44 @@ groups: summary: "Resource-driven peer disconnects on {{ $labels.service_instance_id }}" description: >- Resource-budget peer disconnects on {{ $labels.service_instance_id }} in the - last 30m: {{ printf "%.0f" $values.B.Value }} (threshold: more than 5). + last 30m: {{ printf "%.0f" $values.disconnects_now.Value }} (threshold: more than 5). action: >- Sustained disconnects can starve the node of peers. data: - - refId: A + - refId: disconnects_30m relativeTimeRange: from: 2100 to: 0 datasourceUid: prometheus model: - refId: A + refId: disconnects_30m expr: sum by (service_instance_id) (increase(server_info{service_name="xrpld", metric="peer_disconnects_resources"}[30m])) instant: true range: false intervalMs: 1000 maxDataPoints: 43200 - - refId: B + - refId: disconnects_now relativeTimeRange: from: 2100 to: 0 datasourceUid: __expr__ model: - refId: B + refId: disconnects_now type: reduce reducer: last - expression: A + expression: disconnects_30m datasource: type: __expr__ uid: __expr__ - - refId: C + - refId: is_disconnecting relativeTimeRange: from: 2100 to: 0 datasourceUid: __expr__ model: - refId: C + refId: is_disconnecting type: threshold - expression: B + expression: disconnects_now conditions: - evaluator: type: gt diff --git a/docker/telemetry/grafana/provisioning/alerting/templates.yaml b/docker/telemetry/grafana/provisioning/alerting/templates.yaml index 071fd7499a..8d158318f3 100644 --- a/docker/telemetry/grafana/provisioning/alerting/templates.yaml +++ b/docker/telemetry/grafana/provisioning/alerting/templates.yaml @@ -7,23 +7,24 @@ # says. These templates replace that body with the rule's own prose. # # --------------------------------------------------------------------------- -# Slack and email need SEPARATE bodies. Do not share one. +# SLACK ONLY. Email deliberately has no template here. # --------------------------------------------------------------------------- -# Measured against a real email and a real Slack payload: +# Measured, and the reason this file covers one channel rather than two: # -# * Email ESCAPES any HTML in the message: a literal `
` arrives as -# `<br>`. So no tags, and no clickable anchors are possible here. -# * Email DOES turn a newline in the template into a real `
`, so line -# breaks are the only layout tool available. -# * Slack markup is meaningless in email. Bold asterisks, backticks and -# `:rotating_light:` all arrive as literal characters, and worst of all a -# `` link cannot become a link, so it dumps the whole URL -# inline. Four panel links then bury the prose in a wall of text. +# * Email ESCAPES any HTML in the message body, so a literal `
` arrives +# as `<br>` and no tag or anchor is possible. A custom email body is +# therefore plain text and cannot reproduce Grafana's default layout, which +# already gives section headings, a label table, clickable annotation links +# and the View/Silence buttons. So email keeps the default body. +# * Slack markup is meaningless in email anyway: bold asterisks, backticks +# and `:rotating_light:` arrive as literal characters, and a `` +# link cannot become a link, so it dumps the whole URL inline. Four panel +# links bury the prose in a wall of text. That is what a shared body did. # * Real emoji CHARACTERS render in both channels; the `:shortcode:` form # only works in Slack. # -# So the email body is plain text with one fact per line and each link on its -# own labelled line, and the Slack body keeps mrkdwn. +# The value line Grafana's default email body ends with is made readable by +# NAMING each rule's query steps (see rules.yaml), not by replacing the body. # # --------------------------------------------------------------------------- # What a template may and may not use @@ -93,23 +94,3 @@ templates: {{ .Annotations.description }} {{ end }}{{ end }} {{- end }} - - {{- /* EMAIL body: plain text only. Newlines are the sole layout tool, - and each link sits on its own labelled line. */ -}} - {{ define "xrpld.email.body" -}} - {{ with .Alerts.Firing }}{{ range . }} - FIRING - {{ .Labels.alertname }} on {{ template "xrpld.node" . }} - {{ .Annotations.description }} - {{- with .Annotations.action }} - Why it matters: {{ . }} - {{- end }} - {{- range $key, $url := .Annotations }} - {{- if match "^panel_" $key }} - {{ title (reReplaceAll "_" " " (reReplaceAll "^panel_" "" $key)) }}: {{ $url }} - {{- end }}{{ end }} - {{ end }}{{ end }} - {{- with .Alerts.Resolved }}{{ range . }} - RESOLVED - {{ .Labels.alertname }} on {{ template "xrpld.node" . }} - {{ .Annotations.description }} - {{ end }}{{ end }} - {{- end }}