From 0dccf29b6478ce96d6da7c1d523ad87a84aea149 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Tue, 22 Sep 2026 13:58:20 +0100 Subject: [PATCH 1/6] fix(telemetry): publish the Loki port on the host loopback Grafana reaches Loki as loki:3100 over the compose network and the collector exports to it in-network, so the published port was reachable off-host for no consumer. --- docker/telemetry/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/telemetry/docker-compose.yml b/docker/telemetry/docker-compose.yml index 6d247d2840..0c342b942e 100644 --- a/docker/telemetry/docker-compose.yml +++ b/docker/telemetry/docker-compose.yml @@ -134,7 +134,7 @@ services: loki: image: grafana/loki:3.4.2 ports: - - "3100:3100" + - "127.0.0.1:3100:3100" command: -config.file=/etc/loki/local-config.yaml volumes: - loki-data:/loki From 89be8fd551927ef13f748660ec343e075ccab55c Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Tue, 22 Sep 2026 13:58:21 +0100 Subject: [PATCH 2/6] fix(telemetry): publish workload stack ports on the host loopback Same change as the base compose file, applied to the validation stack. The harness reaches every service via localhost, verified across the workload scripts and integration-test.sh. --- docker/telemetry/docker-compose.workload.yaml | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/docker/telemetry/docker-compose.workload.yaml b/docker/telemetry/docker-compose.workload.yaml index 5e81570f95..59a9f9047c 100644 --- a/docker/telemetry/docker-compose.workload.yaml +++ b/docker/telemetry/docker-compose.workload.yaml @@ -40,11 +40,17 @@ services: otel-collector: image: otel/opentelemetry-collector-contrib:0.158.0 command: ["--config=/etc/otel-collector-config.yaml"] + # Published on the host loopback only. The receivers have no auth and no + # TLS, so only processes on this host may reach them. Note this 127.0.0.1 + # is the HOST interface docker listens on; the container-side bind lives in + # the collector config and is a separate choice. Upstream asks for a + # specific interface rather than 0.0.0.0 on either side (CWE-1327): + # https://opentelemetry.io/docs/security/config-best-practices/ ports: - - "4317:4317" # OTLP gRPC - - "4318:4318" # OTLP HTTP (traces + beast::insight metrics) - - "8889:8889" # Prometheus metrics endpoint - - "13133:13133" # Health check + - "127.0.0.1:4317:4317" # OTLP gRPC + - "127.0.0.1:4318:4318" # OTLP HTTP (traces + beast::insight metrics) + - "127.0.0.1:8889:8889" # Prometheus metrics endpoint + - "127.0.0.1:13133:13133" # Health check volumes: - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro # Mount the validation workdir so the file_log receiver can tail node @@ -60,7 +66,7 @@ services: image: grafana/tempo:2.9.4 command: ["-config.file=/etc/tempo.yaml"] ports: - - "3200:3200" # Tempo HTTP API + - "127.0.0.1:3200:3200" # Tempo HTTP API volumes: - ./tempo.yaml:/etc/tempo.yaml:ro - tempo-data:/var/tempo @@ -70,7 +76,7 @@ services: prometheus: image: prom/prometheus:v3.13.2 ports: - - "9090:9090" + - "127.0.0.1:9090:9090" volumes: - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro depends_on: @@ -81,7 +87,7 @@ services: loki: image: grafana/loki:3.7.6 ports: - - "3100:3100" # Loki HTTP API + - "127.0.0.1:3100:3100" # Loki HTTP API command: ["-config.file=/etc/loki/local-config.yaml"] networks: - workload-net @@ -98,7 +104,7 @@ services: - GF_AUTH_ANONYMOUS_ENABLED=true # No login required for local dev - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin # Full access without auth ports: - - "3000:3000" # Grafana web UI + - "127.0.0.1:3000:3000" # Grafana web UI volumes: - ./grafana/provisioning:/etc/grafana/provisioning:ro - ./grafana/dashboards:/var/lib/grafana/dashboards:ro From 74706020e8de7cb162d1982515e12432986fe572 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Tue, 22 Sep 2026 13:58:33 +0100 Subject: [PATCH 3/6] docs(telemetry): drop the site figure from the rotation keep-policy comment The comment stated a measured rotation interval. The repo is public, so a one-site measurement belongs in an internal note, and it rots the day the config changes. The reason for keeping every rotation trace is unchanged and now stated without the number. --- docker/telemetry/otel-collector-config.grafanacloud.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/telemetry/otel-collector-config.grafanacloud.yaml b/docker/telemetry/otel-collector-config.grafanacloud.yaml index 6ebdc81610..99c1bbbaa3 100644 --- a/docker/telemetry/otel-collector-config.grafanacloud.yaml +++ b/docker/telemetry/otel-collector-config.grafanacloud.yaml @@ -90,9 +90,10 @@ processors: type: probabilistic probabilistic: sampling_percentage: 0.5 - # A rotation happens every ~2 h and is the trace that proves a - # full->syncing flap. Keep every one; children arrive first, so match - # on the family prefix rather than the root name. + # A nodestore rotation is the trace that proves a full->syncing flap, + # and it is rare, so keep every one rather than leaving it to the + # probabilistic policy. Children arrive before the root, so match on + # the family prefix rather than the root name. - name: keep-rotation-traces type: ottl_condition ottl_condition: From 850ed7ec2e1b97c060b51fca0149dfbd1701b416 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:06:47 +0100 Subject: [PATCH 4/6] fix(telemetry): slow the Job Queue dashboard refresh to 30s Every other dashboard refreshes at 30s. This one was at 5s, which re-ran all of its queries twelve times a minute for no benefit. --- docker/telemetry/grafana/dashboards/job-queue.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/telemetry/grafana/dashboards/job-queue.json b/docker/telemetry/grafana/dashboards/job-queue.json index 0516079768..0b1a5f32dd 100644 --- a/docker/telemetry/grafana/dashboards/job-queue.json +++ b/docker/telemetry/grafana/dashboards/job-queue.json @@ -804,5 +804,5 @@ "title": "Job Queue Analysis", "uid": "job-queue", "version": 1, - "refresh": "5s" + "refresh": "30s" } From 07ec48d717343b96be35cb8686e29050902e382e Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:06:49 +0100 Subject: [PATCH 5/6] fix(telemetry): point template variables at the datasource variable Sixteen template variables named a datasource by the literal uid "prometheus" or "loki". Neither exists on the Grafana instance, so those variables resolved to nothing and their pickers offered only All. That also broke the panels. With an empty variable the selector becomes job_type=~"" rather than job_type=~".*", which matches no series, so the Current Job Latency gauge had no data and its renderer raised the plugin error banner on every refresh. Every working dashboard already references ${DS_PROMETHEUS} or ${DS_LOKI}, which the dashboard declares in templating.list. These now match. job-queue 6, ledger-data-sync 5, log-derived-insights 3, overlay-traffic-detail 2. --- docker/telemetry/grafana/dashboards/job-queue.json | 12 ++++++------ .../grafana/dashboards/ledger-data-sync.json | 10 +++++----- .../grafana/dashboards/log-derived-insights.json | 6 +++--- .../grafana/dashboards/overlay-traffic-detail.json | 4 ++-- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/job-queue.json b/docker/telemetry/grafana/dashboards/job-queue.json index 0b1a5f32dd..f31a87be24 100644 --- a/docker/telemetry/grafana/dashboards/job-queue.json +++ b/docker/telemetry/grafana/dashboards/job-queue.json @@ -621,7 +621,7 @@ "query": "label_values(service_name)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", @@ -641,7 +641,7 @@ "query": "label_values(deployment_environment)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", @@ -661,7 +661,7 @@ "query": "label_values(xrpl_network_type)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", @@ -741,7 +741,7 @@ "query": "label_values(target_info, service_instance_id)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", @@ -761,7 +761,7 @@ "query": "label_values(job_queued_total, job_type)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", @@ -781,7 +781,7 @@ "query": "label_values(job_queued_total, handler)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index 3827c902d2..6fe4cbc84d 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -2365,7 +2365,7 @@ "query": "label_values(service_name)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", @@ -2385,7 +2385,7 @@ "query": "label_values(deployment_environment)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", @@ -2405,7 +2405,7 @@ "query": "label_values(xrpl_network_type)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", @@ -2485,7 +2485,7 @@ "query": "label_values(target_info, service_instance_id)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", @@ -2505,7 +2505,7 @@ "query": "label_values(job_queued_total, handler)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", diff --git a/docker/telemetry/grafana/dashboards/log-derived-insights.json b/docker/telemetry/grafana/dashboards/log-derived-insights.json index dc26f76a27..ceb4a9f9cd 100644 --- a/docker/telemetry/grafana/dashboards/log-derived-insights.json +++ b/docker/telemetry/grafana/dashboards/log-derived-insights.json @@ -90,7 +90,7 @@ "query": "label_values(service_name)", "datasource": { "type": "loki", - "uid": "loki" + "uid": "${DS_LOKI}" }, "includeAll": true, "allValue": ".*", @@ -110,7 +110,7 @@ "query": "label_values({service_name=\"xrpld\"}, deployment_environment)", "datasource": { "type": "loki", - "uid": "loki" + "uid": "${DS_LOKI}" }, "includeAll": true, "allValue": ".*", @@ -130,7 +130,7 @@ "query": "label_values({service_name=\"xrpld\"}, service_instance_id)", "datasource": { "type": "loki", - "uid": "loki" + "uid": "${DS_LOKI}" }, "includeAll": true, "allValue": ".*", diff --git a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json index dd088632da..cb60c3e348 100644 --- a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json +++ b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json @@ -1135,7 +1135,7 @@ "query": "label_values(getobject_lookups_total, result)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", @@ -1155,7 +1155,7 @@ "query": "label_values(getobject_rejected_total, reason)", "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "allValue": ".*", From 2de992f334d13380d13005d9d7ea4416f5e4cc3f Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:07:02 +0100 Subject: [PATCH 6/6] fix(telemetry): give Ledger Data & Sync an auto-refresh interval The dashboard had no top-level refresh key at all, so auto-refresh was off while every other dashboard refreshes at 30s. --- docker/telemetry/grafana/dashboards/ledger-data-sync.json | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index 204f9c1b96..a66030b107 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -1724,6 +1724,7 @@ "id": 27 } ], + "refresh": "30s", "schemaVersion": 39, "tags": ["ledger", "sync"], "templating": {