From 979f4b64e4572fa4fe247def47bc5837efd51f4d Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Tue, 7 Jul 2026 18:28:11 +0100 Subject: [PATCH 1/2] fix(telemetry): correct service.name label and make alerting boot without env - MetricsRegistry: service.name was recorded as boolean true because the string literal "xrpld" bound to the OTel AttributeValue variant's char-const* -> bool overload. Assign std::string so it selects the string alternative; Prometheus now shows service_name=xrpld (and exported_job round-trips correctly) on every MetricsRegistry series. - Grafana alerting contact points: the Slack url / email addresses used ${ENV_VAR} references that expand to empty when unset, and Grafana's provisioning validator (which lacks ${VAR:-default} support) then crashes the whole stack on startup. Use non-empty disabled placeholders (unroutable webhook host, .invalid email) so the stack boots with zero configuration; delivery stays off until a real destination is supplied. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../provisioning/alerting/contactpoints.yaml | 37 ++++++++++++------- src/xrpld/telemetry/MetricsRegistry.cpp | 6 ++- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml b/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml index a0c1694172..a9b9d25036 100644 --- a/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml +++ b/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml @@ -7,13 +7,20 @@ # xrpld-critical — Slack + email; receives critical-severity alerts. # The severity split is wired in policies.yaml. # -# Secrets and personal addresses are NOT hard-coded. Grafana expands -# ${ENV_VAR} references in provisioning files at load time, so the real -# Slack webhook and alert-email addresses come from environment variables -# supplied via docker/telemetry/.env.alerting (gitignored). See -# .env.alerting.example for the variable list and the Alerting section of +# Secrets and personal addresses are NOT hard-coded. To enable delivery, +# replace the disabled placeholder values below with a real Slack webhook +# and alert email — e.g. by copying docker/telemetry/.env.alerting.example +# to .env.alerting (gitignored) and swapping the placeholder lines to +# ${SLACK_WEBHOOK_URL} / ${ALERT_EMAIL_TO}. See the Alerting section of # docs/telemetry-runbook.md for setup. -# If a variable is unset, that receiver has no live destination. +# +# The defaults are deliberately non-empty, invalid-but-valid-shaped +# placeholders (an unroutable webhook host and a .invalid email). Grafana's +# alerting provisioning validator REQUIRES a non-empty Slack url and email +# addresses, and it does NOT support ${VAR:-default} expansion — an unset +# ${VAR} expands to empty and crashes Grafana on startup. The placeholders +# keep the whole stack booting with zero configuration; alerts simply route +# nowhere until a real destination is supplied. # # Email delivery additionally requires SMTP configured on the Grafana # service (GF_SMTP_* in docker-compose.yml). @@ -28,10 +35,11 @@ contactPoints: - uid: xrpld-slack-default type: slack settings: - # Incoming-webhook delivery: the channel is fixed by the webhook, so - # `recipient` is only here to satisfy Grafana's Slack validator. - url: ${SLACK_WEBHOOK_URL} - recipient: ${SLACK_CHANNEL} + # Disabled placeholder: an unroutable webhook host. A non-empty url + # selects Slack webhook mode (no recipient/token required) and keeps + # provisioning valid. Replace with a real ${SLACK_WEBHOOK_URL} to + # enable delivery. + url: https://hooks.slack.invalid/disabled title: "{{ .CommonLabels.alertname }} on {{ .CommonLabels.exported_instance }}" disableResolveMessage: false @@ -42,13 +50,16 @@ contactPoints: - uid: xrpld-slack-critical type: slack settings: - url: ${SLACK_WEBHOOK_URL} - recipient: ${SLACK_CHANNEL} + # Disabled placeholder — see xrpld-slack-default above. + url: https://hooks.slack.invalid/disabled title: "[CRITICAL] {{ .CommonLabels.alertname }} on {{ .CommonLabels.exported_instance }}" disableResolveMessage: false - uid: xrpld-email-critical type: email settings: - addresses: ${ALERT_EMAIL_TO} + # Disabled placeholder: a .invalid address keeps the required + # `addresses` field non-empty so provisioning validates. Replace + # with a real ${ALERT_EMAIL_TO} (and enable SMTP) to deliver. + addresses: alerts-disabled@xrpld.invalid singleEmail: true disableResolveMessage: false diff --git a/src/xrpld/telemetry/MetricsRegistry.cpp b/src/xrpld/telemetry/MetricsRegistry.cpp index bd51db3b51..693df1fd5e 100644 --- a/src/xrpld/telemetry/MetricsRegistry.cpp +++ b/src/xrpld/telemetry/MetricsRegistry.cpp @@ -182,7 +182,11 @@ MetricsRegistry::start(std::string const& endpoint, std::string const& instanceI // Configure resource attributes so Prometheus exported_instance labels // distinguish metrics from different nodes (matches OTelCollector setup). resource::ResourceAttributes attrs; - attrs[opentelemetry::semconv::service::kServiceName] = "xrpld"; + // Use std::string, not a string literal: ResourceAttributes stores an + // OTel AttributeValue variant whose char-const* overload binds to bool, + // so "xrpld" would be recorded as the boolean true. std::string selects + // the string alternative and the value round-trips as service.name=xrpld. + attrs[opentelemetry::semconv::service::kServiceName] = std::string("xrpld"); if (!instanceId.empty()) attrs[opentelemetry::semconv::service::kServiceInstanceId] = instanceId; auto resourceAttrs = resource::Resource::Create(attrs); From bc140d9e96867e706273208e47dc442d6b0f4f6f Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Tue, 7 Jul 2026 18:54:01 +0100 Subject: [PATCH 2/2] fix(telemetry): repoint node-health job panels to native job-duration metrics The Key Jobs execution/dequeue panels queried StatsD-era per-job metric names (xrpld__milliseconds_bucket, xrpld__q_milliseconds_bucket) that the native OTel path no longer emits. Phase 9's MetricsRegistry emits job timings as two label-dimensioned histograms instead: xrpld_job_running_duration_us_bucket{job_type=""} xrpld_job_queued_duration_us_bucket{job_type=""} Rewrite the 22 affected panel queries (11 job types x running/queued) to the label-dimensioned form, preserving the histogram_quantile / rate / sum-by structure and all template-variable filters. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../grafana/dashboards/node-health.json | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index e9e599ccc8..c249dcae85 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -433,77 +433,77 @@ "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_acceptLedger_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"acceptLedger\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Accept Ledger [{{quantile}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_advanceLedger_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"advanceLedger\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Advance Ledger [{{quantile}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_transaction_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"transaction\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Transaction [{{quantile}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_writeObjects_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"writeObjects\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Write Objects [{{quantile}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_heartbeat_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"heartbeat\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Heartbeat [{{quantile}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_sweep_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"sweep\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Sweep [{{quantile}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_trustedValidation_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"trustedValidation\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Trusted Validation [{{quantile}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_trustedProposal_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"trustedProposal\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Trusted Proposal [{{quantile}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_publishNewLedger_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"publishNewLedger\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Publish New Ledger [{{quantile}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_clientRPC_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"clientRPC\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Client RPC [{{quantile}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_ledgerData_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"ledgerData\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Ledger Data [{{quantile}}]" } ], @@ -542,77 +542,77 @@ "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_acceptLedger_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"acceptLedger\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Accept Ledger [{{quantile}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_advanceLedger_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"advanceLedger\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Advance Ledger [{{quantile}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_transaction_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"transaction\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Transaction [{{quantile}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_writeObjects_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"writeObjects\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Write Objects [{{quantile}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_heartbeat_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"heartbeat\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Heartbeat [{{quantile}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_sweep_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"sweep\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Sweep [{{quantile}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_trustedValidation_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"trustedValidation\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Trusted Validation [{{quantile}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_trustedProposal_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"trustedProposal\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Trusted Proposal [{{quantile}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_publishNewLedger_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"publishNewLedger\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Publish New Ledger [{{quantile}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_clientRPC_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"clientRPC\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Client RPC [{{quantile}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_ledgerData_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"ledgerData\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Ledger Data [{{quantile}}]" } ],