From bca04bc2c3f9c818cc7f89ce3cb8576e2272b89a Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Wed, 23 Sep 2026 15:19:51 +0100 Subject: [PATCH] fix(telemetry): give email its own notification body, not the Slack one Sharing one body between the Slack and email receivers made the email unreadable. Email cannot render Slack markup, so the bold asterisks, the backticks and the `:rotating_light:` shortcodes all arrived as literal characters, and a `` link could not become a link at all -- it dumped the whole dashboard URL inline. Four panel links then buried the prose. Split the body in two. The Slack body keeps mrkdwn. The email body is plain text, one fact per line, with each dashboard link on its own labelled line. Both still share the title and the node-identity fallback. Measured while fixing this, and now recorded in templates.yaml: email escapes any HTML in the message, so `
` arrives as `<br>` and no tag or anchor is possible; but a newline in the template does become a real `
`, so line breaks are the only layout tool email has. Also correct two comments that were wrong. A missing template define does not ship raw template text: it logs one warning and silently delivers Grafana's default body, the value dump this file exists to remove, while the rule still reports health=ok. And the Cloud contact point is not email-only; it holds a Slack receiver and an email receiver, on an instance shared with other teams. --- .../provisioning/alerting/contactpoints.yaml | 19 ++++-- .../provisioning/alerting/templates.yaml | 60 +++++++++++++++---- 2 files changed, 63 insertions(+), 16 deletions(-) diff --git a/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml b/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml index 0083b584d4..2fd6ce6a52 100644 --- a/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml +++ b/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml @@ -41,11 +41,14 @@ # GF_SMTP_ENABLED=true and the relay settings point somewhere real. # # Grafana Cloud does NOT use this file — Cloud has no provisioning filesystem. -# Cloud delivery is created over the REST API instead: a single email-only -# contact point, attached to each rule via per-rule notification_settings. +# Cloud delivery is created over the REST API instead: one contact point named +# `xrpld-alerts` holding a Slack receiver AND an email receiver, attached to +# each rule via per-rule notification_settings. Keep this file's templates and +# the Cloud receivers' templates in step by hand; nothing syncs them. # The notification policy tree must never be pushed to a shared Cloud # instance -- there is exactly one tree per org and the PUT endpoint -# replaces it wholesale. +# replaces it wholesale. That Cloud instance is shared with other teams, so a +# change there must be scoped to the `xrpld-` uids and the xrpld folder. apiVersion: 1 @@ -66,7 +69,7 @@ contactPoints: # label and is absent on NoData/Error evaluations, so the templates # guard it rather than referencing it bare. title: '{{ template "xrpld.title" . }}' - text: '{{ template "xrpld.body" . }}' + text: '{{ template "xrpld.slack.body" . }}' disableResolveMessage: false # --- Critical tier: Slack + email --- @@ -81,7 +84,7 @@ contactPoints: # The title template already carries the severity label, so this tier # needs no separate "[CRITICAL]" prefix. title: '{{ template "xrpld.title" . }}' - text: '{{ template "xrpld.body" . }}' + text: '{{ template "xrpld.slack.body" . }}' disableResolveMessage: false - uid: xrpld-email-critical type: email @@ -97,8 +100,12 @@ contactPoints: # what removes the raw value dump and the label list. Grafana's own # header ("N firing alert instances", "Grouped by") and footer are # fixed chrome and stay. + # + # This uses the EMAIL body, not the Slack one. Email escapes HTML and + # cannot render Slack markup, so a shared body arrives as literal + # asterisks and raw URLs. See templates.yaml. subject: '{{ template "xrpld.title" . }}' - message: '{{ template "xrpld.body" . }}' + message: '{{ template "xrpld.email.body" . }}' disableResolveMessage: false # To retire a receiver that a running Grafana has already stored, uncomment diff --git a/docker/telemetry/grafana/provisioning/alerting/templates.yaml b/docker/telemetry/grafana/provisioning/alerting/templates.yaml index 0e533aee9b..071fd7499a 100644 --- a/docker/telemetry/grafana/provisioning/alerting/templates.yaml +++ b/docker/telemetry/grafana/provisioning/alerting/templates.yaml @@ -7,19 +7,39 @@ # says. These templates replace that body with the rule's own prose. # # --------------------------------------------------------------------------- +# Slack and email need SEPARATE bodies. Do not share one. +# --------------------------------------------------------------------------- +# Measured against a real email and a real Slack payload: +# +# * Email ESCAPES any HTML in the message: a literal `
` arrives as +# `<br>`. So no tags, and no clickable anchors are possible here. +# * Email DOES turn a newline in the template into a real `
`, so line +# breaks are the only layout tool available. +# * Slack markup is meaningless in email. Bold asterisks, backticks and +# `:rotating_light:` all arrive as literal characters, and worst of all a +# `` link cannot become a link, so it dumps the whole URL +# inline. Four panel links then bury the prose in a wall of text. +# * Real emoji CHARACTERS render in both channels; the `:shortcode:` form +# only works in Slack. +# +# So the email body is plain text with one fact per line and each link on its +# own labelled line, and the Slack body keeps mrkdwn. +# +# --------------------------------------------------------------------------- # What a template may and may not use # --------------------------------------------------------------------------- -# The function set here is much smaller than Go's text/template plus sprig. +# The function set is much smaller than Go's text/template plus sprig. # Available and used below: `match` (regexp), `reReplaceAll`, `title`, `printf`, # `len`, `index`, `eq`. NOT available — each fails the whole template with -# `function "X" not defined`, which silently ships the raw template text in the -# notification instead: `hasPrefix`, `hasSuffix`, `contains`, `humanize`, +# `function "X" not defined`: `hasPrefix`, `hasSuffix`, `contains`, `humanize`, # `humanizeDuration`, `humanizePercentage`. # -# A template error does NOT mark the rule unhealthy. Rules keep reporting -# `health=ok` and the broken text is only visible in the delivered message, so -# any change here must be checked against a real notification, not against rule -# state. +# A template failure is nearly silent, which is the trap. Measured: a missing +# define logs one `failed to template email message` warning and then delivers +# Grafana's DEFAULT body — the value dump and label list this file exists to +# remove — while the rule still reports `health=ok`. So a change here is only +# verified by reading a real delivered notification, never by rule state, and +# never by the absence of an obvious error. # # --------------------------------------------------------------------------- # Two label traps @@ -47,7 +67,7 @@ templates: {{ if .Labels.service_instance_id }}{{ .Labels.service_instance_id }}{{ else }}unknown node (NoData/Error evaluation){{ end }} {{- end }} - {{- /* One line, used as the Slack title and the email subject. */ -}} + {{- /* One line. Slack title and email subject share this. */ -}} {{ define "xrpld.title" -}} {{ if .Alerts.Firing }}🔥 FIRING{{ else }}✅ RESOLVED{{ end }} {{- with .CommonLabels.alertname }} · {{ . }}{{ end }} @@ -55,8 +75,8 @@ templates: {{- with .CommonLabels.service_instance_id }} · {{ . }}{{ end }} {{- end }} - {{- /* Message body: one block per alert, prose only. */ -}} - {{ define "xrpld.body" -}} + {{- /* SLACK body: mrkdwn, emoji shortcodes, links. */ -}} + {{ define "xrpld.slack.body" -}} {{ with .Alerts.Firing }}{{ range . }} :rotating_light: *FIRING* · *{{ .Labels.alertname }}* on `{{ template "xrpld.node" . }}` {{ .Annotations.description }} @@ -73,3 +93,23 @@ templates: {{ .Annotations.description }} {{ end }}{{ end }} {{- end }} + + {{- /* EMAIL body: plain text only. Newlines are the sole layout tool, + and each link sits on its own labelled line. */ -}} + {{ define "xrpld.email.body" -}} + {{ with .Alerts.Firing }}{{ range . }} + FIRING - {{ .Labels.alertname }} on {{ template "xrpld.node" . }} + {{ .Annotations.description }} + {{- with .Annotations.action }} + Why it matters: {{ . }} + {{- end }} + {{- range $key, $url := .Annotations }} + {{- if match "^panel_" $key }} + {{ title (reReplaceAll "_" " " (reReplaceAll "^panel_" "" $key)) }}: {{ $url }} + {{- end }}{{ end }} + {{ end }}{{ end }} + {{- with .Alerts.Resolved }}{{ range . }} + RESOLVED - {{ .Labels.alertname }} on {{ template "xrpld.node" . }} + {{ .Annotations.description }} + {{ end }}{{ end }} + {{- end }}