diff --git a/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml b/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml
index 0083b584d4..2fd6ce6a52 100644
--- a/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml
+++ b/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml
@@ -41,11 +41,14 @@
# GF_SMTP_ENABLED=true and the relay settings point somewhere real.
#
# Grafana Cloud does NOT use this file — Cloud has no provisioning filesystem.
-# Cloud delivery is created over the REST API instead: a single email-only
-# contact point, attached to each rule via per-rule notification_settings.
+# Cloud delivery is created over the REST API instead: one contact point named
+# `xrpld-alerts` holding a Slack receiver AND an email receiver, attached to
+# each rule via per-rule notification_settings. Keep this file's templates and
+# the Cloud receivers' templates in step by hand; nothing syncs them.
# The notification policy tree must never be pushed to a shared Cloud
# instance -- there is exactly one tree per org and the PUT endpoint
-# replaces it wholesale.
+# replaces it wholesale. That Cloud instance is shared with other teams, so a
+# change there must be scoped to the `xrpld-` uids and the xrpld folder.
apiVersion: 1
@@ -66,7 +69,7 @@ contactPoints:
# label and is absent on NoData/Error evaluations, so the templates
# guard it rather than referencing it bare.
title: '{{ template "xrpld.title" . }}'
- text: '{{ template "xrpld.body" . }}'
+ text: '{{ template "xrpld.slack.body" . }}'
disableResolveMessage: false
# --- Critical tier: Slack + email ---
@@ -81,7 +84,7 @@ contactPoints:
# The title template already carries the severity label, so this tier
# needs no separate "[CRITICAL]" prefix.
title: '{{ template "xrpld.title" . }}'
- text: '{{ template "xrpld.body" . }}'
+ text: '{{ template "xrpld.slack.body" . }}'
disableResolveMessage: false
- uid: xrpld-email-critical
type: email
@@ -97,8 +100,12 @@ contactPoints:
# what removes the raw value dump and the label list. Grafana's own
# header ("N firing alert instances", "Grouped by") and footer are
# fixed chrome and stay.
+ #
+ # This uses the EMAIL body, not the Slack one. Email escapes HTML and
+ # cannot render Slack markup, so a shared body arrives as literal
+ # asterisks and raw URLs. See templates.yaml.
subject: '{{ template "xrpld.title" . }}'
- message: '{{ template "xrpld.body" . }}'
+ message: '{{ template "xrpld.email.body" . }}'
disableResolveMessage: false
# To retire a receiver that a running Grafana has already stored, uncomment
diff --git a/docker/telemetry/grafana/provisioning/alerting/templates.yaml b/docker/telemetry/grafana/provisioning/alerting/templates.yaml
index 0e533aee9b..071fd7499a 100644
--- a/docker/telemetry/grafana/provisioning/alerting/templates.yaml
+++ b/docker/telemetry/grafana/provisioning/alerting/templates.yaml
@@ -7,19 +7,39 @@
# says. These templates replace that body with the rule's own prose.
#
# ---------------------------------------------------------------------------
+# Slack and email need SEPARATE bodies. Do not share one.
+# ---------------------------------------------------------------------------
+# Measured against a real email and a real Slack payload:
+#
+# * Email ESCAPES any HTML in the message: a literal `
` arrives as
+# `<br>`. So no tags, and no clickable anchors are possible here.
+# * Email DOES turn a newline in the template into a real `
`, so line
+# breaks are the only layout tool available.
+# * Slack markup is meaningless in email. Bold asterisks, backticks and
+# `:rotating_light:` all arrive as literal characters, and worst of all a
+# `` link cannot become a link, so it dumps the whole URL
+# inline. Four panel links then bury the prose in a wall of text.
+# * Real emoji CHARACTERS render in both channels; the `:shortcode:` form
+# only works in Slack.
+#
+# So the email body is plain text with one fact per line and each link on its
+# own labelled line, and the Slack body keeps mrkdwn.
+#
+# ---------------------------------------------------------------------------
# What a template may and may not use
# ---------------------------------------------------------------------------
-# The function set here is much smaller than Go's text/template plus sprig.
+# The function set is much smaller than Go's text/template plus sprig.
# Available and used below: `match` (regexp), `reReplaceAll`, `title`, `printf`,
# `len`, `index`, `eq`. NOT available — each fails the whole template with
-# `function "X" not defined`, which silently ships the raw template text in the
-# notification instead: `hasPrefix`, `hasSuffix`, `contains`, `humanize`,
+# `function "X" not defined`: `hasPrefix`, `hasSuffix`, `contains`, `humanize`,
# `humanizeDuration`, `humanizePercentage`.
#
-# A template error does NOT mark the rule unhealthy. Rules keep reporting
-# `health=ok` and the broken text is only visible in the delivered message, so
-# any change here must be checked against a real notification, not against rule
-# state.
+# A template failure is nearly silent, which is the trap. Measured: a missing
+# define logs one `failed to template email message` warning and then delivers
+# Grafana's DEFAULT body — the value dump and label list this file exists to
+# remove — while the rule still reports `health=ok`. So a change here is only
+# verified by reading a real delivered notification, never by rule state, and
+# never by the absence of an obvious error.
#
# ---------------------------------------------------------------------------
# Two label traps
@@ -47,7 +67,7 @@ templates:
{{ if .Labels.service_instance_id }}{{ .Labels.service_instance_id }}{{ else }}unknown node (NoData/Error evaluation){{ end }}
{{- end }}
- {{- /* One line, used as the Slack title and the email subject. */ -}}
+ {{- /* One line. Slack title and email subject share this. */ -}}
{{ define "xrpld.title" -}}
{{ if .Alerts.Firing }}🔥 FIRING{{ else }}✅ RESOLVED{{ end }}
{{- with .CommonLabels.alertname }} · {{ . }}{{ end }}
@@ -55,8 +75,8 @@ templates:
{{- with .CommonLabels.service_instance_id }} · {{ . }}{{ end }}
{{- end }}
- {{- /* Message body: one block per alert, prose only. */ -}}
- {{ define "xrpld.body" -}}
+ {{- /* SLACK body: mrkdwn, emoji shortcodes, links. */ -}}
+ {{ define "xrpld.slack.body" -}}
{{ with .Alerts.Firing }}{{ range . }}
:rotating_light: *FIRING* · *{{ .Labels.alertname }}* on `{{ template "xrpld.node" . }}`
{{ .Annotations.description }}
@@ -73,3 +93,23 @@ templates:
{{ .Annotations.description }}
{{ end }}{{ end }}
{{- end }}
+
+ {{- /* EMAIL body: plain text only. Newlines are the sole layout tool,
+ and each link sits on its own labelled line. */ -}}
+ {{ define "xrpld.email.body" -}}
+ {{ with .Alerts.Firing }}{{ range . }}
+ FIRING - {{ .Labels.alertname }} on {{ template "xrpld.node" . }}
+ {{ .Annotations.description }}
+ {{- with .Annotations.action }}
+ Why it matters: {{ . }}
+ {{- end }}
+ {{- range $key, $url := .Annotations }}
+ {{- if match "^panel_" $key }}
+ {{ title (reReplaceAll "_" " " (reReplaceAll "^panel_" "" $key)) }}: {{ $url }}
+ {{- end }}{{ end }}
+ {{ end }}{{ end }}
+ {{- with .Alerts.Resolved }}{{ range . }}
+ RESOLVED - {{ .Labels.alertname }} on {{ template "xrpld.node" . }}
+ {{ .Annotations.description }}
+ {{ end }}{{ end }}
+ {{- end }}