ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2026-07-23 15:10:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'pratik/otel-phase9-metric-gap-fill' into pratik/otel-phase10-workload-validation

Browse Source
This commit is contained in:
Pratik Mankawde
2026-07-07 18:55:23 +01:00
parent a84c9d0e76 b15c549f63
commit 54e4be9a8f
3 changed files with 51 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
docker/telemetry/grafana/dashboards/node-health.json
View File

@@ -1643,7 +1643,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_acceptLedger_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"acceptLedger\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"legendFormat": "Accept Ledger [{{quantile}}]"
},
{
@@ -1651,7 +1651,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_advanceLedger_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"advanceLedger\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"legendFormat": "Advance Ledger [{{quantile}}]"
},
{
@@ -1659,7 +1659,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_transaction_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"transaction\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"legendFormat": "Transaction [{{quantile}}]"
},
{
@@ -1667,7 +1667,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_writeObjects_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"writeObjects\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"legendFormat": "Write Objects [{{quantile}}]"
},
{
@@ -1675,7 +1675,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_heartbeat_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"heartbeat\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"legendFormat": "Heartbeat [{{quantile}}]"
},
{
@@ -1683,7 +1683,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_sweep_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"sweep\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"legendFormat": "Sweep [{{quantile}}]"
},
{
@@ -1691,7 +1691,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_trustedValidation_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"trustedValidation\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"legendFormat": "Trusted Validation [{{quantile}}]"
},
{
@@ -1699,7 +1699,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_trustedProposal_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"trustedProposal\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"legendFormat": "Trusted Proposal [{{quantile}}]"
},
{
@@ -1707,7 +1707,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_publishNewLedger_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"publishNewLedger\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"legendFormat": "Publish New Ledger [{{quantile}}]"
},
{
@@ -1715,7 +1715,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_clientRPC_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"clientRPC\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"legendFormat": "Client RPC [{{quantile}}]"
},
{
@@ -1723,7 +1723,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_ledgerData_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_running_duration_us_bucket{job_type=\"ledgerData\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"legendFormat": "Ledger Data [{{quantile}}]"
}
],
@@ -1763,7 +1763,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_acceptLedger_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"acceptLedger\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"legendFormat": "Accept Ledger [{{quantile}}]"
},
{
@@ -1771,7 +1771,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_advanceLedger_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"advanceLedger\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"legendFormat": "Advance Ledger [{{quantile}}]"
},
{
@@ -1779,7 +1779,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_transaction_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"transaction\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"legendFormat": "Transaction [{{quantile}}]"
},
{
@@ -1787,7 +1787,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_writeObjects_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"writeObjects\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"legendFormat": "Write Objects [{{quantile}}]"
},
{
@@ -1795,7 +1795,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_heartbeat_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"heartbeat\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"legendFormat": "Heartbeat [{{quantile}}]"
},
{
@@ -1803,7 +1803,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_sweep_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"sweep\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"legendFormat": "Sweep [{{quantile}}]"
},
{
@@ -1811,7 +1811,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_trustedValidation_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"trustedValidation\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"legendFormat": "Trusted Validation [{{quantile}}]"
},
{
@@ -1819,7 +1819,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_trustedProposal_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"trustedProposal\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"legendFormat": "Trusted Proposal [{{quantile}}]"
},
{
@@ -1827,7 +1827,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_publishNewLedger_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"publishNewLedger\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"legendFormat": "Publish New Ledger [{{quantile}}]"
},
{
@@ -1835,7 +1835,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_clientRPC_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"clientRPC\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"legendFormat": "Client RPC [{{quantile}}]"
},
{
@@ -1843,7 +1843,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_ledgerData_q_milliseconds_bucket{exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"expr": "histogram_quantile($quantile, sum by (le, exported_instance) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"ledgerData\", exported_instance=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))",
"legendFormat": "Ledger Data [{{quantile}}]"
}
],

37
docker/telemetry/grafana/provisioning/alerting/contactpoints.yaml
View File

@@ -7,13 +7,20 @@
# xrpld-critical — Slack + email; receives critical-severity alerts.
# The severity split is wired in policies.yaml.
#
# Secrets and personal addresses are NOT hard-coded. Grafana expands
# ${ENV_VAR} references in provisioning files at load time, so the real
# Slack webhook and alert-email addresses come from environment variables
# supplied via docker/telemetry/.env.alerting (gitignored). See
# .env.alerting.example for the variable list and the Alerting section of
# Secrets and personal addresses are NOT hard-coded. To enable delivery,
# replace the disabled placeholder values below with a real Slack webhook
# and alert email — e.g. by copying docker/telemetry/.env.alerting.example
# to .env.alerting (gitignored) and swapping the placeholder lines to
# ${SLACK_WEBHOOK_URL} / ${ALERT_EMAIL_TO}. See the Alerting section of
# docs/telemetry-runbook.md for setup.
# If a variable is unset, that receiver has no live destination.
#
# The defaults are deliberately non-empty, invalid-but-valid-shaped
# placeholders (an unroutable webhook host and a .invalid email). Grafana's
# alerting provisioning validator REQUIRES a non-empty Slack url and email
# addresses, and it does NOT support ${VAR:-default} expansion — an unset
# ${VAR} expands to empty and crashes Grafana on startup. The placeholders
# keep the whole stack booting with zero configuration; alerts simply route
# nowhere until a real destination is supplied.
#
# Email delivery additionally requires SMTP configured on the Grafana
# service (GF_SMTP_* in docker-compose.yml).
@@ -28,10 +35,11 @@ contactPoints:
- uid: xrpld-slack-default
type: slack
settings:
# Incoming-webhook delivery: the channel is fixed by the webhook, so
# `recipient` is only here to satisfy Grafana's Slack validator.
url: ${SLACK_WEBHOOK_URL}
recipient: ${SLACK_CHANNEL}
# Disabled placeholder: an unroutable webhook host. A non-empty url
# selects Slack webhook mode (no recipient/token required) and keeps
# provisioning valid. Replace with a real ${SLACK_WEBHOOK_URL} to
# enable delivery.
url: https://hooks.slack.invalid/disabled
title: "{{ .CommonLabels.alertname }} on {{ .CommonLabels.exported_instance }}"
disableResolveMessage: false
@@ -42,13 +50,16 @@ contactPoints:
- uid: xrpld-slack-critical
type: slack
settings:
url: ${SLACK_WEBHOOK_URL}
recipient: ${SLACK_CHANNEL}
# Disabled placeholder — see xrpld-slack-default above.
url: https://hooks.slack.invalid/disabled
title: "[CRITICAL] {{ .CommonLabels.alertname }} on {{ .CommonLabels.exported_instance }}"
disableResolveMessage: false
- uid: xrpld-email-critical
type: email
settings:
addresses: ${ALERT_EMAIL_TO}
# Disabled placeholder: a .invalid address keeps the required
# `addresses` field non-empty so provisioning validates. Replace
# with a real ${ALERT_EMAIL_TO} (and enable SMTP) to deliver.
addresses: alerts-disabled@xrpld.invalid
singleEmail: true
disableResolveMessage: false

6
src/xrpld/telemetry/MetricsRegistry.cpp
View File

@@ -182,7 +182,11 @@ MetricsRegistry::start(std::string const& endpoint, std::string const& instanceI
// Configure resource attributes so Prometheus exported_instance labels
// distinguish metrics from different nodes (matches OTelCollector setup).
resource::ResourceAttributes attrs;
attrs[opentelemetry::semconv::service::kServiceName] = "xrpld";
// Use std::string, not a string literal: ResourceAttributes stores an
// OTel AttributeValue variant whose char-const* overload binds to bool,
// so "xrpld" would be recorded as the boolean true. std::string selects
// the string alternative and the value round-trips as service.name=xrpld.
attrs[opentelemetry::semconv::service::kServiceName] = std::string("xrpld");
if (!instanceId.empty())
attrs[opentelemetry::semconv::service::kServiceInstanceId] = instanceId;
auto resourceAttrs = resource::Resource::Create(attrs);