diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json index 981d8e01ea..04b44027ae 100644 --- a/docker/telemetry/grafana/dashboards/consensus-health.json +++ b/docker/telemetry/grafana/dashboards/consensus-health.json @@ -235,7 +235,7 @@ { "datasource": { "type": "tempo", - "uid": "tempo" + "uid": "${DS_TEMPO}" }, "queryType": "traceql", "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\"} | quantile_over_time(span.round_time_ms, .5)", @@ -245,7 +245,7 @@ { "datasource": { "type": "tempo", - "uid": "tempo" + "uid": "${DS_TEMPO}" }, "queryType": "traceql", "refId": "B", @@ -288,7 +288,7 @@ { "datasource": { "type": "tempo", - "uid": "tempo" + "uid": "${DS_TEMPO}" }, "queryType": "traceql", "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\"} | avg_over_time(span.round_time_ms)", @@ -337,7 +337,7 @@ { "datasource": { "type": "tempo", - "uid": "tempo" + "uid": "${DS_TEMPO}" }, "queryType": "traceql", "refId": "A", @@ -381,7 +381,7 @@ { "datasource": { "type": "tempo", - "uid": "tempo" + "uid": "${DS_TEMPO}" }, "queryType": "traceql", "query": "{name=\"consensus.round\" && resource.service.instance.id=~\"$node\"} | quantile_over_time(span.previous_round_time_ms, .95)", @@ -676,7 +676,7 @@ { "datasource": { "type": "tempo", - "uid": "tempo" + "uid": "${DS_TEMPO}" }, "queryType": "traceql", "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\"} | avg_over_time(span.close_time_vote_bins)", @@ -686,7 +686,7 @@ { "datasource": { "type": "tempo", - "uid": "tempo" + "uid": "${DS_TEMPO}" }, "queryType": "traceql", "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\"} | avg_over_time(span.close_resolution_ms)", @@ -734,7 +734,7 @@ { "datasource": { "type": "tempo", - "uid": "tempo" + "uid": "${DS_TEMPO}" }, "queryType": "traceql", "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && span.resolution_direction=~\"$resolution_direction\"} | count_over_time() by (span.resolution_direction)", @@ -805,7 +805,7 @@ { "datasource": { "type": "tempo", - "uid": "tempo" + "uid": "${DS_TEMPO}" }, "queryType": "traceql", "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\"} | count_over_time() by (span.close_time_vote_bins)", @@ -1061,6 +1061,32 @@ "tags": ["xrpld", "consensus"], "templating": { "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 2, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "DS_TEMPO", + "type": "datasource", + "label": "Tempo", + "query": "tempo", + "regex": "", + "current": {}, + "hide": 2, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, { "name": "node", "label": "Node", @@ -1069,7 +1095,7 @@ "query": "label_values(traces_span_metrics_calls_total, exported_instance)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", @@ -1089,7 +1115,7 @@ "query": "label_values(traces_span_metrics_calls_total{span_name=\"consensus.ledger_close\"}, consensus_mode)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", diff --git a/docker/telemetry/grafana/dashboards/ledger-operations.json b/docker/telemetry/grafana/dashboards/ledger-operations.json index 6ac980bb90..1e3cbea661 100644 --- a/docker/telemetry/grafana/dashboards/ledger-operations.json +++ b/docker/telemetry/grafana/dashboards/ledger-operations.json @@ -308,6 +308,19 @@ "tags": ["xrpld", "ledger"], "templating": { "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 2, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, { "name": "node", "label": "Node", @@ -316,7 +329,7 @@ "query": "label_values(traces_span_metrics_calls_total, exported_instance)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", diff --git a/docker/telemetry/grafana/dashboards/peer-network.json b/docker/telemetry/grafana/dashboards/peer-network.json index dfbc751cb8..cf3d6017fb 100644 --- a/docker/telemetry/grafana/dashboards/peer-network.json +++ b/docker/telemetry/grafana/dashboards/peer-network.json @@ -248,6 +248,19 @@ "tags": ["xrpld", "peer"], "templating": { "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 2, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, { "name": "node", "label": "Node", @@ -256,7 +269,7 @@ "query": "label_values(traces_span_metrics_calls_total, exported_instance)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", @@ -276,7 +289,7 @@ "query": "label_values(traces_span_metrics_calls_total{span_name=\"peer.proposal.receive\"}, proposal_trusted)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", @@ -296,7 +309,7 @@ "query": "label_values(traces_span_metrics_calls_total{span_name=\"peer.validation.receive\"}, validation_trusted)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", diff --git a/docker/telemetry/grafana/dashboards/rpc-performance.json b/docker/telemetry/grafana/dashboards/rpc-performance.json index 45074c0eac..e3f0c70afd 100644 --- a/docker/telemetry/grafana/dashboards/rpc-performance.json +++ b/docker/telemetry/grafana/dashboards/rpc-performance.json @@ -421,6 +421,19 @@ "tags": ["xrpld", "rpc"], "templating": { "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 2, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, { "name": "node", "label": "Node", @@ -429,7 +442,7 @@ "query": "label_values(traces_span_metrics_calls_total, exported_instance)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", @@ -449,7 +462,7 @@ "query": "label_values(traces_span_metrics_calls_total{span_name=~\"rpc.command.*\"}, command)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", diff --git a/docker/telemetry/grafana/dashboards/system-ledger-data-sync.json b/docker/telemetry/grafana/dashboards/system-ledger-data-sync.json index 006155f672..950150ef27 100644 --- a/docker/telemetry/grafana/dashboards/system-ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/system-ledger-data-sync.json @@ -496,6 +496,19 @@ "tags": ["xrpld", "statsd", "ledger", "sync"], "templating": { "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 2, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, { "name": "node", "label": "Node", @@ -504,7 +517,7 @@ "query": "label_values(xrpld_ledger_data_get_Bytes_In, exported_instance)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", diff --git a/docker/telemetry/grafana/dashboards/system-network-traffic.json b/docker/telemetry/grafana/dashboards/system-network-traffic.json index 9fb061b5a5..81f0e4261c 100644 --- a/docker/telemetry/grafana/dashboards/system-network-traffic.json +++ b/docker/telemetry/grafana/dashboards/system-network-traffic.json @@ -774,6 +774,19 @@ "tags": ["xrpld", "statsd", "network"], "templating": { "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 2, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, { "name": "node", "label": "Node", @@ -782,7 +795,7 @@ "query": "label_values(xrpld_Peer_Finder_Active_Inbound_Peers, exported_instance)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", diff --git a/docker/telemetry/grafana/dashboards/system-node-health.json b/docker/telemetry/grafana/dashboards/system-node-health.json index 9fa958ad27..f7acd31219 100644 --- a/docker/telemetry/grafana/dashboards/system-node-health.json +++ b/docker/telemetry/grafana/dashboards/system-node-health.json @@ -2128,6 +2128,19 @@ "tags": ["xrpld", "statsd", "node-health"], "templating": { "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 2, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, { "name": "node", "label": "Node", @@ -2136,7 +2149,7 @@ "query": "label_values(xrpld_LedgerMaster_Validated_Ledger_Age, exported_instance)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", @@ -2156,7 +2169,7 @@ "query": "label_values(xrpld_object_count, type)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", diff --git a/docker/telemetry/grafana/dashboards/system-overlay-traffic-detail.json b/docker/telemetry/grafana/dashboards/system-overlay-traffic-detail.json index 496ec56c22..e46f699030 100644 --- a/docker/telemetry/grafana/dashboards/system-overlay-traffic-detail.json +++ b/docker/telemetry/grafana/dashboards/system-overlay-traffic-detail.json @@ -556,6 +556,19 @@ "tags": ["xrpld", "statsd", "overlay", "network"], "templating": { "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 2, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, { "name": "node", "label": "Node", @@ -564,7 +577,7 @@ "query": "label_values(xrpld_squelch_Messages_In, exported_instance)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", diff --git a/docker/telemetry/grafana/dashboards/system-rpc-pathfinding.json b/docker/telemetry/grafana/dashboards/system-rpc-pathfinding.json index 3e2a7651df..a9405c7e19 100644 --- a/docker/telemetry/grafana/dashboards/system-rpc-pathfinding.json +++ b/docker/telemetry/grafana/dashboards/system-rpc-pathfinding.json @@ -553,6 +553,19 @@ "tags": ["xrpld", "statsd", "rpc", "pathfinding"], "templating": { "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 2, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, { "name": "node", "label": "Node", @@ -561,7 +574,7 @@ "query": "label_values(xrpld_rpc_requests_total, exported_instance)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", @@ -581,7 +594,7 @@ "query": "label_values(traces_span_metrics_calls_total{span_name=~\"grpc\\\\..*\"}, method)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json index 24476d8451..c811d19100 100644 --- a/docker/telemetry/grafana/dashboards/transaction-overview.json +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -870,6 +870,19 @@ "tags": ["xrpld", "transactions"], "templating": { "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 2, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, { "name": "node", "label": "Node", @@ -878,7 +891,7 @@ "query": "label_values(traces_span_metrics_calls_total, exported_instance)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", @@ -898,7 +911,7 @@ "query": "label_values(traces_span_metrics_calls_total{span_name=\"tx.process\"}, local)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", diff --git a/docker/telemetry/grafana/dashboards/xrpld-fee-market.json b/docker/telemetry/grafana/dashboards/xrpld-fee-market.json index dfeac283e7..318146a127 100644 --- a/docker/telemetry/grafana/dashboards/xrpld-fee-market.json +++ b/docker/telemetry/grafana/dashboards/xrpld-fee-market.json @@ -459,6 +459,19 @@ "tags": ["xrpld", "fee-market"], "templating": { "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 2, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, { "name": "node", "label": "Node", @@ -467,7 +480,7 @@ "query": "label_values(exported_instance)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", diff --git a/docker/telemetry/grafana/dashboards/xrpld-job-queue.json b/docker/telemetry/grafana/dashboards/xrpld-job-queue.json index f19fdedce1..8ccff34c48 100644 --- a/docker/telemetry/grafana/dashboards/xrpld-job-queue.json +++ b/docker/telemetry/grafana/dashboards/xrpld-job-queue.json @@ -457,6 +457,19 @@ "tags": ["xrpld", "job-queue"], "templating": { "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 2, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, { "name": "node", "label": "Node", @@ -465,7 +478,7 @@ "query": "label_values(exported_instance)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", @@ -485,7 +498,7 @@ "query": "label_values(xrpld_job_queued_total, job_type)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", diff --git a/docker/telemetry/grafana/dashboards/xrpld-peer-quality.json b/docker/telemetry/grafana/dashboards/xrpld-peer-quality.json index 72995c7996..25558c8424 100644 --- a/docker/telemetry/grafana/dashboards/xrpld-peer-quality.json +++ b/docker/telemetry/grafana/dashboards/xrpld-peer-quality.json @@ -366,6 +366,19 @@ "tags": ["xrpld", "peer", "network"], "templating": { "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 2, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, { "name": "node", "label": "Node", @@ -374,7 +387,7 @@ "query": "label_values(xrpld_peer_quality, exported_instance)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", diff --git a/docker/telemetry/grafana/dashboards/xrpld-rpc-perf-otel.json b/docker/telemetry/grafana/dashboards/xrpld-rpc-perf-otel.json index b7fc34fde1..508d068192 100644 --- a/docker/telemetry/grafana/dashboards/xrpld-rpc-perf-otel.json +++ b/docker/telemetry/grafana/dashboards/xrpld-rpc-perf-otel.json @@ -397,6 +397,19 @@ "tags": ["xrpld", "rpc"], "templating": { "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 2, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, { "name": "node", "label": "Node", @@ -405,7 +418,7 @@ "query": "label_values(exported_instance)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", @@ -425,7 +438,7 @@ "query": "label_values(xrpld_rpc_method_started_total, method)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", diff --git a/docker/telemetry/grafana/dashboards/xrpld-validator-health.json b/docker/telemetry/grafana/dashboards/xrpld-validator-health.json index 439eb1bf43..8278554456 100644 --- a/docker/telemetry/grafana/dashboards/xrpld-validator-health.json +++ b/docker/telemetry/grafana/dashboards/xrpld-validator-health.json @@ -919,6 +919,19 @@ "tags": ["xrpld", "validator", "health"], "templating": { "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 2, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, { "name": "node", "label": "Node", @@ -927,7 +940,7 @@ "query": "label_values(xrpld_validation_agreement, exported_instance)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 69dc99fdad..da0dd348f8 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -42,6 +42,39 @@ cmake --preset default -Dtelemetry=ON cmake --build --preset default ``` +### 4. Run against a live network + +Two ready-made configs connect a tracking node (no validator credentials) to a +public network with all tracing and native metrics enabled: + +| Config | Network | +| ---------------------------------------------- | ------- | +| `docker/telemetry/xrpld-telemetry.cfg` | Devnet | +| `docker/telemetry/xrpld-telemetry-mainnet.cfg` | Mainnet | + +```bash +.build/xrpld --conf docker/telemetry/xrpld-telemetry-mainnet.cfg +``` + +Both set `[insight] server=otel` (native metrics → collector → Prometheus, which +drives the dashboards) and `service_instance_id`, exposed by Prometheus as the +`exported_instance` label that the `$node` dashboard variable filters on. The +mainnet config logs to `/home/pratik/xrpld-logs/mainnet/debug.log` — the path +the collector's filelog receiver tails for log-trace correlation. + +Metrics begin flowing as soon as the node connects to peers (`server_state` +≥ `connected`); full ledger and consensus panels populate after sync +(`server_state` = `full`). Check progress with: + +```bash +curl -s http://localhost:5005 -d '{"method":"server_info"}' | + jq '.result.info | {server_state, peers, complete_ledgers}' +``` + +> Mainnet sync is bandwidth- and disk-heavy. For a quick check use the devnet +> config or the standalone test in `docker/telemetry/TESTING.md`, which +> generates spans without waiting for a live sync. + ## Configuration Reference | Option | Default | Description | @@ -124,6 +157,36 @@ remove the local exporters (`debug`, `otlp/tempo`, `prometheus`, > logging on (at least `warning` level) so the collector's filelog receiver > has a `debug.log` to tail. Traces and metrics are unaffected by log level. +### Importing dashboards to Grafana Cloud + +Shipping data (above) is independent of installing the dashboards. The local +stack auto-provisions dashboards from a mounted folder +(`grafana/provisioning/dashboards/dashboards.yaml`, `type: file`); Grafana +Cloud cannot read your filesystem, so its dashboards must be imported over the +HTTP API or the UI. + +The dashboard JSON in `docker/telemetry/grafana/dashboards/` references its +backends through datasource **template variables** (`${DS_PROMETHEUS}`, +`${DS_TEMPO}`) rather than fixed UIDs. On import, Grafana binds each variable +to a datasource of the matching type — auto-selecting it when only one exists +(the usual case: one Mimir, one Tempo). This is what makes the same files work +unchanged on both the local stack and Cloud. + +> Dashboards are parameterized by `grafana/parameterize-datasources.py`. If you +> add a dashboard exported with hardcoded UIDs, re-run that script (idempotent) +> before committing so it stays portable. + +To import: + +1. In Grafana Cloud, go to **Dashboards → New → Import**. +2. Upload a file from `docker/telemetry/grafana/dashboards/` (or paste its + JSON), then click **Load**. +3. At the datasource prompt, confirm the auto-selected **Prometheus/Mimir** + datasource — and **Tempo** for dashboards that query traces — then + **Import**. +4. Repeat per dashboard. Only `consensus-health` uses Tempo; the rest need + only the Prometheus/Mimir datasource. + ## Span Reference All spans instrumented in xrpld, grouped by subsystem: