diff --git a/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml b/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml index 0083b584d4..2fd6ce6a52 100644 --- a/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml +++ b/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml @@ -41,11 +41,14 @@ # GF_SMTP_ENABLED=true and the relay settings point somewhere real. # # Grafana Cloud does NOT use this file — Cloud has no provisioning filesystem. -# Cloud delivery is created over the REST API instead: a single email-only -# contact point, attached to each rule via per-rule notification_settings. +# Cloud delivery is created over the REST API instead: one contact point named +# `xrpld-alerts` holding a Slack receiver AND an email receiver, attached to +# each rule via per-rule notification_settings. Keep this file's templates and +# the Cloud receivers' templates in step by hand; nothing syncs them. # The notification policy tree must never be pushed to a shared Cloud # instance -- there is exactly one tree per org and the PUT endpoint -# replaces it wholesale. +# replaces it wholesale. That Cloud instance is shared with other teams, so a +# change there must be scoped to the `xrpld-` uids and the xrpld folder. apiVersion: 1 @@ -66,7 +69,7 @@ contactPoints: # label and is absent on NoData/Error evaluations, so the templates # guard it rather than referencing it bare. title: '{{ template "xrpld.title" . }}' - text: '{{ template "xrpld.body" . }}' + text: '{{ template "xrpld.slack.body" . }}' disableResolveMessage: false # --- Critical tier: Slack + email --- @@ -81,7 +84,7 @@ contactPoints: # The title template already carries the severity label, so this tier # needs no separate "[CRITICAL]" prefix. title: '{{ template "xrpld.title" . }}' - text: '{{ template "xrpld.body" . }}' + text: '{{ template "xrpld.slack.body" . }}' disableResolveMessage: false - uid: xrpld-email-critical type: email @@ -97,8 +100,12 @@ contactPoints: # what removes the raw value dump and the label list. Grafana's own # header ("N firing alert instances", "Grouped by") and footer are # fixed chrome and stay. + # + # This uses the EMAIL body, not the Slack one. Email escapes HTML and + # cannot render Slack markup, so a shared body arrives as literal + # asterisks and raw URLs. See templates.yaml. subject: '{{ template "xrpld.title" . }}' - message: '{{ template "xrpld.body" . }}' + message: '{{ template "xrpld.email.body" . }}' disableResolveMessage: false # To retire a receiver that a running Grafana has already stored, uncomment diff --git a/docker/telemetry/grafana/provisioning/alerting/templates.yaml b/docker/telemetry/grafana/provisioning/alerting/templates.yaml index 0e533aee9b..071fd7499a 100644 --- a/docker/telemetry/grafana/provisioning/alerting/templates.yaml +++ b/docker/telemetry/grafana/provisioning/alerting/templates.yaml @@ -7,19 +7,39 @@ # says. These templates replace that body with the rule's own prose. # # --------------------------------------------------------------------------- +# Slack and email need SEPARATE bodies. Do not share one. +# --------------------------------------------------------------------------- +# Measured against a real email and a real Slack payload: +# +# * Email ESCAPES any HTML in the message: a literal `
` arrives as +# `<br>`. So no tags, and no clickable anchors are possible here. +# * Email DOES turn a newline in the template into a real `
`, so line +# breaks are the only layout tool available. +# * Slack markup is meaningless in email. Bold asterisks, backticks and +# `:rotating_light:` all arrive as literal characters, and worst of all a +# `` link cannot become a link, so it dumps the whole URL +# inline. Four panel links then bury the prose in a wall of text. +# * Real emoji CHARACTERS render in both channels; the `:shortcode:` form +# only works in Slack. +# +# So the email body is plain text with one fact per line and each link on its +# own labelled line, and the Slack body keeps mrkdwn. +# +# --------------------------------------------------------------------------- # What a template may and may not use # --------------------------------------------------------------------------- -# The function set here is much smaller than Go's text/template plus sprig. +# The function set is much smaller than Go's text/template plus sprig. # Available and used below: `match` (regexp), `reReplaceAll`, `title`, `printf`, # `len`, `index`, `eq`. NOT available — each fails the whole template with -# `function "X" not defined`, which silently ships the raw template text in the -# notification instead: `hasPrefix`, `hasSuffix`, `contains`, `humanize`, +# `function "X" not defined`: `hasPrefix`, `hasSuffix`, `contains`, `humanize`, # `humanizeDuration`, `humanizePercentage`. # -# A template error does NOT mark the rule unhealthy. Rules keep reporting -# `health=ok` and the broken text is only visible in the delivered message, so -# any change here must be checked against a real notification, not against rule -# state. +# A template failure is nearly silent, which is the trap. Measured: a missing +# define logs one `failed to template email message` warning and then delivers +# Grafana's DEFAULT body — the value dump and label list this file exists to +# remove — while the rule still reports `health=ok`. So a change here is only +# verified by reading a real delivered notification, never by rule state, and +# never by the absence of an obvious error. # # --------------------------------------------------------------------------- # Two label traps @@ -47,7 +67,7 @@ templates: {{ if .Labels.service_instance_id }}{{ .Labels.service_instance_id }}{{ else }}unknown node (NoData/Error evaluation){{ end }} {{- end }} - {{- /* One line, used as the Slack title and the email subject. */ -}} + {{- /* One line. Slack title and email subject share this. */ -}} {{ define "xrpld.title" -}} {{ if .Alerts.Firing }}🔥 FIRING{{ else }}✅ RESOLVED{{ end }} {{- with .CommonLabels.alertname }} · {{ . }}{{ end }} @@ -55,8 +75,8 @@ templates: {{- with .CommonLabels.service_instance_id }} · {{ . }}{{ end }} {{- end }} - {{- /* Message body: one block per alert, prose only. */ -}} - {{ define "xrpld.body" -}} + {{- /* SLACK body: mrkdwn, emoji shortcodes, links. */ -}} + {{ define "xrpld.slack.body" -}} {{ with .Alerts.Firing }}{{ range . }} :rotating_light: *FIRING* · *{{ .Labels.alertname }}* on `{{ template "xrpld.node" . }}` {{ .Annotations.description }} @@ -73,3 +93,23 @@ templates: {{ .Annotations.description }} {{ end }}{{ end }} {{- end }} + + {{- /* EMAIL body: plain text only. Newlines are the sole layout tool, + and each link sits on its own labelled line. */ -}} + {{ define "xrpld.email.body" -}} + {{ with .Alerts.Firing }}{{ range . }} + FIRING - {{ .Labels.alertname }} on {{ template "xrpld.node" . }} + {{ .Annotations.description }} + {{- with .Annotations.action }} + Why it matters: {{ . }} + {{- end }} + {{- range $key, $url := .Annotations }} + {{- if match "^panel_" $key }} + {{ title (reReplaceAll "_" " " (reReplaceAll "^panel_" "" $key)) }}: {{ $url }} + {{- end }}{{ end }} + {{ end }}{{ end }} + {{- with .Alerts.Resolved }}{{ range . }} + RESOLVED - {{ .Labels.alertname }} on {{ template "xrpld.node" . }} + {{ .Annotations.description }} + {{ end }}{{ end }} + {{- end }}