diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index 257c8ddd66..80f468cd44 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -1643,7 +1643,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_acceptLedger_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"acceptLedger\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Accept Ledger [{{quantile}}]" }, { @@ -1651,7 +1651,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_advanceLedger_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"advanceLedger\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Advance Ledger [{{quantile}}]" }, { @@ -1659,7 +1659,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_transaction_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"transaction\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Transaction [{{quantile}}]" }, { @@ -1667,7 +1667,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_writeObjects_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"writeObjects\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Write Objects [{{quantile}}]" }, { @@ -1675,7 +1675,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_heartbeat_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"heartbeat\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Heartbeat [{{quantile}}]" }, { @@ -1683,7 +1683,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_sweep_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"sweep\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Sweep [{{quantile}}]" }, { @@ -1691,7 +1691,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_trustedValidation_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"trustedValidation\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Trusted Validation [{{quantile}}]" }, { @@ -1699,7 +1699,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_trustedProposal_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"trustedProposal\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Trusted Proposal [{{quantile}}]" }, { @@ -1707,7 +1707,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_publishNewLedger_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"publishNewLedger\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Publish New Ledger [{{quantile}}]" }, { @@ -1715,7 +1715,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_clientRPC_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"clientRPC\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Client RPC [{{quantile}}]" }, { @@ -1723,7 +1723,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_ledgerData_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"ledgerData\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Ledger Data [{{quantile}}]" } ], @@ -1763,7 +1763,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_acceptLedger_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"acceptLedger\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Accept Ledger [{{quantile}}]" }, { @@ -1771,7 +1771,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_advanceLedger_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"advanceLedger\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Advance Ledger [{{quantile}}]" }, { @@ -1779,7 +1779,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_transaction_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"transaction\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Transaction [{{quantile}}]" }, { @@ -1787,7 +1787,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_writeObjects_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"writeObjects\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Write Objects [{{quantile}}]" }, { @@ -1795,7 +1795,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_heartbeat_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"heartbeat\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Heartbeat [{{quantile}}]" }, { @@ -1803,7 +1803,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_sweep_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"sweep\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Sweep [{{quantile}}]" }, { @@ -1811,7 +1811,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_trustedValidation_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"trustedValidation\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Trusted Validation [{{quantile}}]" }, { @@ -1819,7 +1819,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_trustedProposal_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"trustedProposal\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Trusted Proposal [{{quantile}}]" }, { @@ -1827,7 +1827,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_publishNewLedger_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"publishNewLedger\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Publish New Ledger [{{quantile}}]" }, { @@ -1835,7 +1835,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_clientRPC_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"clientRPC\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Client RPC [{{quantile}}]" }, { @@ -1843,7 +1843,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_ledgerData_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"ledgerData\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", "legendFormat": "Ledger Data [{{quantile}}]" } ], diff --git a/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml b/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml index a0c1694172..a9b9d25036 100644 --- a/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml +++ b/docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml @@ -7,13 +7,20 @@ # xrpld-critical — Slack + email; receives critical-severity alerts. # The severity split is wired in policies.yaml. # -# Secrets and personal addresses are NOT hard-coded. Grafana expands -# ${ENV_VAR} references in provisioning files at load time, so the real -# Slack webhook and alert-email addresses come from environment variables -# supplied via docker/telemetry/.env.alerting (gitignored). See -# .env.alerting.example for the variable list and the Alerting section of +# Secrets and personal addresses are NOT hard-coded. To enable delivery, +# replace the disabled placeholder values below with a real Slack webhook +# and alert email — e.g. by copying docker/telemetry/.env.alerting.example +# to .env.alerting (gitignored) and swapping the placeholder lines to +# ${SLACK_WEBHOOK_URL} / ${ALERT_EMAIL_TO}. See the Alerting section of # docs/telemetry-runbook.md for setup. -# If a variable is unset, that receiver has no live destination. +# +# The defaults are deliberately non-empty, invalid-but-valid-shaped +# placeholders (an unroutable webhook host and a .invalid email). Grafana's +# alerting provisioning validator REQUIRES a non-empty Slack url and email +# addresses, and it does NOT support ${VAR:-default} expansion — an unset +# ${VAR} expands to empty and crashes Grafana on startup. The placeholders +# keep the whole stack booting with zero configuration; alerts simply route +# nowhere until a real destination is supplied. # # Email delivery additionally requires SMTP configured on the Grafana # service (GF_SMTP_* in docker-compose.yml). @@ -28,10 +35,11 @@ contactPoints: - uid: xrpld-slack-default type: slack settings: - # Incoming-webhook delivery: the channel is fixed by the webhook, so - # `recipient` is only here to satisfy Grafana's Slack validator. - url: ${SLACK_WEBHOOK_URL} - recipient: ${SLACK_CHANNEL} + # Disabled placeholder: an unroutable webhook host. A non-empty url + # selects Slack webhook mode (no recipient/token required) and keeps + # provisioning valid. Replace with a real ${SLACK_WEBHOOK_URL} to + # enable delivery. + url: https://hooks.slack.invalid/disabled title: "{{ .CommonLabels.alertname }} on {{ .CommonLabels.exported_instance }}" disableResolveMessage: false @@ -42,13 +50,16 @@ contactPoints: - uid: xrpld-slack-critical type: slack settings: - url: ${SLACK_WEBHOOK_URL} - recipient: ${SLACK_CHANNEL} + # Disabled placeholder — see xrpld-slack-default above. + url: https://hooks.slack.invalid/disabled title: "[CRITICAL] {{ .CommonLabels.alertname }} on {{ .CommonLabels.exported_instance }}" disableResolveMessage: false - uid: xrpld-email-critical type: email settings: - addresses: ${ALERT_EMAIL_TO} + # Disabled placeholder: a .invalid address keeps the required + # `addresses` field non-empty so provisioning validates. Replace + # with a real ${ALERT_EMAIL_TO} (and enable SMTP) to deliver. + addresses: alerts-disabled@xrpld.invalid singleEmail: true disableResolveMessage: false diff --git a/src/xrpld/telemetry/MetricsRegistry.cpp b/src/xrpld/telemetry/MetricsRegistry.cpp index bd51db3b51..693df1fd5e 100644 --- a/src/xrpld/telemetry/MetricsRegistry.cpp +++ b/src/xrpld/telemetry/MetricsRegistry.cpp @@ -182,7 +182,11 @@ MetricsRegistry::start(std::string const& endpoint, std::string const& instanceI // Configure resource attributes so Prometheus exported_instance labels // distinguish metrics from different nodes (matches OTelCollector setup). resource::ResourceAttributes attrs; - attrs[opentelemetry::semconv::service::kServiceName] = "xrpld"; + // Use std::string, not a string literal: ResourceAttributes stores an + // OTel AttributeValue variant whose char-const* overload binds to bool, + // so "xrpld" would be recorded as the boolean true. std::string selects + // the string alternative and the value round-trips as service.name=xrpld. + attrs[opentelemetry::semconv::service::kServiceName] = std::string("xrpld"); if (!instanceId.empty()) attrs[opentelemetry::semconv::service::kServiceInstanceId] = instanceId; auto resourceAttrs = resource::Resource::Create(attrs);