From 3f510ecf443f529646a9bcd60c8f364ed168da5c Mon Sep 17 00:00:00 2001
From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
Date: Tue, 22 Sep 2026 13:58:17 +0100
Subject: [PATCH 1/6] fix(telemetry): publish local stack ports on the host
 loopback

The compose ports published on every host interface, so the collector's
unauthenticated OTLP receivers, Tempo, and the anonymous-admin Grafana were
reachable by anything that could route to the host.

Nothing consumed the published ports from off-host: containers address each
other by compose service name, and every doc and script uses localhost.
---
 docker/telemetry/docker-compose.yml | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/docker/telemetry/docker-compose.yml b/docker/telemetry/docker-compose.yml
index 0b3f268aaf..6f2094d80d 100644
--- a/docker/telemetry/docker-compose.yml
+++ b/docker/telemetry/docker-compose.yml
@@ -23,10 +23,16 @@ services:
   otel-collector:
     image: otel/opentelemetry-collector-contrib:0.158.0
     command: ["--config=/etc/otel-collector-config.yaml"]
+    # Published on the host loopback only. The receivers have no auth and no
+    # TLS, so only processes on this host may reach them. Note this 127.0.0.1
+    # is the HOST interface docker listens on; the container-side bind lives in
+    # the collector config and is a separate choice. Upstream asks for a
+    # specific interface rather than 0.0.0.0 on either side (CWE-1327):
+    # https://opentelemetry.io/docs/security/config-best-practices/
     ports:
-      - "4317:4317" # OTLP gRPC receiver
-      - "4318:4318" # OTLP HTTP receiver (xrpld sends traces here)
-      - "13133:13133" # Health check endpoint
+      - "127.0.0.1:4317:4317" # OTLP gRPC receiver
+      - "127.0.0.1:4318:4318" # OTLP HTTP receiver (xrpld sends traces here)
+      - "127.0.0.1:13133:13133" # Health check endpoint
     volumes:
       # Mount collector pipeline config (receivers → processors → exporters)
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro
@@ -41,7 +47,7 @@ services:
     image: grafana/tempo:2.9.4
     command: ["-config.file=/etc/tempo.yaml"]
     ports:
-      - "3200:3200" # Tempo HTTP API (health check, query)
+      - "127.0.0.1:3200:3200" # Tempo HTTP API (health check, query)
     volumes:
       # Mount Tempo storage and ingestion config
       - ./tempo.yaml:/etc/tempo.yaml:ro
@@ -58,7 +64,7 @@ services:
       - GF_AUTH_ANONYMOUS_ENABLED=true # No login required for local dev
       - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin # Full access without auth
     ports:
-      - "3000:3000" # Grafana web UI
+      - "127.0.0.1:3000:3000" # Grafana web UI
     volumes:
       # Auto-provision Tempo datasource and search filters on startup
       - ./grafana/provisioning:/etc/grafana/provisioning:ro

From 3cc9f330418667b80fc9c65835ef6a8058835461 Mon Sep 17 00:00:00 2001
From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
Date: Tue, 22 Sep 2026 13:58:18 +0100
Subject: [PATCH 2/6] fix(telemetry): publish the Prometheus ports on the host
 loopback

The StatsD port beside them was already loopback-bound with a comment saying
why; the Prometheus exporter and Prometheus itself were not. Prometheus
scrapes the collector as otel-collector:8889 over the compose network, so the
published port had no consumer.
---
 docker/telemetry/docker-compose.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/telemetry/docker-compose.yml b/docker/telemetry/docker-compose.yml
index 13da6a56c9..5d61f58131 100644
--- a/docker/telemetry/docker-compose.yml
+++ b/docker/telemetry/docker-compose.yml
@@ -30,7 +30,7 @@ services:
       # receiver has no auth, and xrpld runs on the host, so it reaches the
       # collector via 127.0.0.1. Do not expose on 0.0.0.0 in shared setups.
       - "127.0.0.1:8125:8125/udp"
-      - "8889:8889" # Prometheus metrics (span_metrics + statsd)
+      - "127.0.0.1:8889:8889" # Prometheus metrics (span_metrics + statsd)
       - "13133:13133" # Health check
     volumes:
       # Mount collector pipeline config (receivers → processors → exporters)
@@ -59,7 +59,7 @@ services:
     # Pinned to an exact patch release for reproducible, config-stable runs.
     image: prom/prometheus:v3.13.2
     ports:
-      - "9090:9090"
+      - "127.0.0.1:9090:9090"
     volumes:
       - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
     depends_on:

From 54809960ff10c2202ddae0d81ca8f52ffc3ed7fa Mon Sep 17 00:00:00 2001
From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
Date: Tue, 22 Sep 2026 15:00:30 +0100
Subject: [PATCH 3/6] docs(telemetry): remove site measurements from the
 collector comments

The repo is public, so a comment must not carry one-site measurements or
claim a prior state of the source. Three comments did both.

The tail-sampling comment called itself "Box-only (uncommitted)", which names
our deployment and was untrue. It now states what the sampling does, and adds
the consequence that matters: Log.cpp writes a trace id only for a sampled
span, so sampling after that decision leaves log-to-trace links resolving to
nothing.

The histogram bucket comments justified each edge with measured percentages
and durations, and said what the buckets "used to" do. The mechanism is
unchanged and now stated without the figures. Also drops a duplicated line in
the Alloy copy. The bucket lists themselves are untouched and still agree
across all three files.
---
 docker/telemetry/alloy/config.alloy           | 17 +++++------
 .../otel-collector-config.grafanacloud.yaml   | 30 ++++++++++---------
 docker/telemetry/otel-collector-config.yaml   | 19 ++++++------
 3 files changed, 33 insertions(+), 33 deletions(-)

diff --git a/docker/telemetry/alloy/config.alloy b/docker/telemetry/alloy/config.alloy
index e5d5f613de..a41493e878 100644
--- a/docker/telemetry/alloy/config.alloy
+++ b/docker/telemetry/alloy/config.alloy
@@ -315,15 +315,14 @@ otelcol.connector.spanmetrics "xrpld" {
     explicit {
       // Buckets MUST stay strictly ascending (the connector binary-searches
       // them and silently misbuckets otherwise) and MUST match the two
-      // otel-collector-config*.yaml lists. Sub-MILLISECOND boundaries exist
-      // because most xrpld spans sit far below 1ms (tx.preflight means
-      // ~0.012ms): a 1ms floor put >99.99% of samples in bucket one and pinned
-      // every p95/p99 at a constant 0.95ms.
-      // otel-collector-config*.yaml lists. Sub-second boundaries cover
-      // RPC/tx/ledger spans; 2s-4s resolve second-scale consensus spans
-      // (consensus.round ~3.9s, consensus.establish ~1.9s) that used to pile
-      // into one 1s-5s bucket; 10s/30s give the ledger.acquire catch-up tail
-      // (~17% exceeds 5s) a measurable home so its p95/p99 stop reading +Inf.
+      // otel-collector-config*.yaml lists. Sub-millisecond boundaries exist
+      // because most xrpld spans sit far below 1ms: a 1ms floor puts nearly
+      // every sample in bucket one and pins p95/p99 to that bucket's edge.
+      // Sub-second boundaries cover RPC, tx and ledger spans. The 2s-4s edges
+      // resolve second-scale consensus spans, and 10s/30s give the
+      // ledger.acquire catch-up tail a home so its p95/p99 stop reading +Inf.
+      // Each edge was chosen from a measured duration distribution for the
+      // spans it covers.
       buckets = ["0.01ms", "0.05ms", "0.1ms", "0.25ms", "0.5ms", "1ms", "5ms", "10ms", "25ms", "50ms", "100ms", "250ms", "500ms", "1s", "2s", "3s", "4s", "5s", "10s", "30s"]
     }
   }
diff --git a/docker/telemetry/otel-collector-config.grafanacloud.yaml b/docker/telemetry/otel-collector-config.grafanacloud.yaml
index 01722e383a..4b488a84cd 100644
--- a/docker/telemetry/otel-collector-config.grafanacloud.yaml
+++ b/docker/telemetry/otel-collector-config.grafanacloud.yaml
@@ -78,10 +78,13 @@ processors:
   batch:
     timeout: 1s
     send_batch_size: 100
-  # Box-only (uncommitted): probabilistic 0.5% tail sampling on the
-  # trace-storage branch only. spanmetrics-derived metrics see 100% of spans
-  # (they run on a separate branch) so RED remain exact; Tempo/Cloud stores
-  # ~1/200 of traces to keep long-running node cost bounded.
+  # Probabilistic tail sampling, on the trace-storage branch only. The
+  # spanmetrics branch sees every span, so span-derived RED metrics stay
+  # exact while stored trace volume is cut to bound cost on a long-running
+  # node. Operators who need full traces raise sampling_percentage to 100.
+  # Note: a log line carries a trace id only when its span was sampled at
+  # the SDK, so sampling here drops traces whose ids are already in the
+  # logs, and log-to-trace links for those resolve to nothing.
   tail_sampling:
     decision_wait: 10s
     num_traces: 50000
@@ -177,16 +180,15 @@ connectors:
       unit: ms
       explicit:
         # Buckets MUST stay strictly ascending (the connector binary-searches
-        # them and silently misbuckets otherwise). Sub-MILLISECOND boundaries
-        # (0.01-0.5ms) exist because most xrpld spans are far below 1ms --
-        # tx.preflight means ~0.012ms, tx.preclaim ~0.15ms -- so a 1ms floor
-        # put >99.99% of samples in the first bucket and pinned every p95/p99
-        # at a constant 0.95ms. Sub-second boundaries cover
-        # RPC/tx/ledger spans; the 2s-4s boundaries resolve second-scale
-        # consensus spans (consensus.round ~3.9s, consensus.establish ~1.9s)
-        # that used to pile into a single 1s-5s bucket; 10s/30s give the
-        # ledger.acquire tail (~17% exceeds 5s during catch-up) a measurable
-        # home so its p95/p99 stop reading as +Inf.
+        # them and silently misbuckets otherwise) and MUST match the Alloy
+        # list and the other collector config. Sub-millisecond boundaries
+        # exist because most xrpld spans sit far below 1ms: a 1ms floor puts
+        # nearly every sample in the first bucket and pins p95/p99 to that
+        # bucket's edge. Sub-second boundaries cover RPC, tx and ledger spans.
+        # The 2s-4s edges resolve second-scale consensus spans, and 10s/30s
+        # give the ledger.acquire catch-up tail a home so its p95/p99 stop
+        # reading as +Inf. Each edge was chosen from a measured duration
+        # distribution for the spans it covers.
         buckets:
           [
             0.01ms,
diff --git a/docker/telemetry/otel-collector-config.yaml b/docker/telemetry/otel-collector-config.yaml
index 093d3ddda1..e3d1850fcd 100644
--- a/docker/telemetry/otel-collector-config.yaml
+++ b/docker/telemetry/otel-collector-config.yaml
@@ -168,16 +168,15 @@ connectors:
       unit: ms
       explicit:
         # Buckets MUST stay strictly ascending (the connector binary-searches
-        # them and silently misbuckets otherwise). Sub-MILLISECOND boundaries
-        # (0.01-0.5ms) exist because most xrpld spans are far below 1ms --
-        # tx.preflight means ~0.012ms, tx.preclaim ~0.15ms -- so a 1ms floor
-        # put >99.99% of samples in the first bucket and pinned every p95/p99
-        # at a constant 0.95ms. Sub-second boundaries cover
-        # RPC/tx/ledger spans; the 2s-4s boundaries resolve second-scale
-        # consensus spans (consensus.round ~3.9s, consensus.establish ~1.9s)
-        # that used to pile into a single 1s-5s bucket; 10s/30s give the
-        # ledger.acquire tail (~17% exceeds 5s during catch-up) a measurable
-        # home so its p95/p99 stop reading as +Inf.
+        # them and silently misbuckets otherwise) and MUST match the Alloy
+        # list and the other collector config. Sub-millisecond boundaries
+        # exist because most xrpld spans sit far below 1ms: a 1ms floor puts
+        # nearly every sample in the first bucket and pins p95/p99 to that
+        # bucket's edge. Sub-second boundaries cover RPC, tx and ledger spans.
+        # The 2s-4s edges resolve second-scale consensus spans, and 10s/30s
+        # give the ledger.acquire catch-up tail a home so its p95/p99 stop
+        # reading as +Inf. Each edge was chosen from a measured duration
+        # distribution for the spans it covers.
         buckets:
           [
             0.01ms,

From d3f9fa1f674783c36854400b9d540011278d6de9 Mon Sep 17 00:00:00 2001
From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
Date: Tue, 22 Sep 2026 15:03:40 +0100
Subject: [PATCH 4/6] fix(telemetry): make the job latency panel readable at
 any time range

The gauge showed a p99 pooled across every job type, so a slow rare job could
not surface: sweep sat at 5 s while the dial read 20 ms, because a
count-weighted p99 follows the frequent jobs and those are fast.

It is now a bar gauge over per-job-type p99, which is what the panel is for,
and a bar gauge holds tens of values where a dial holds one. The series keep
job_type so each bar names its job.

max is set to the red threshold, so every bar shares one 0 to 1 s scale
instead of each dial auto-scaling to its own data.

The targets are instant queries. Left as range queries the bar count followed
the dashboard's time range rather than the current series count, reaching 531
series over a week.
---
 docker/telemetry/alloy/config.alloy           | 17 ++++----
 .../grafana/dashboards/job-queue.json         | 39 +++++++++++++------
 .../otel-collector-config.grafanacloud.yaml   | 30 +++++++-------
 docker/telemetry/otel-collector-config.yaml   | 19 ++++-----
 4 files changed, 61 insertions(+), 44 deletions(-)

diff --git a/docker/telemetry/alloy/config.alloy b/docker/telemetry/alloy/config.alloy
index a41493e878..e5d5f613de 100644
--- a/docker/telemetry/alloy/config.alloy
+++ b/docker/telemetry/alloy/config.alloy
@@ -315,14 +315,15 @@ otelcol.connector.spanmetrics "xrpld" {
     explicit {
       // Buckets MUST stay strictly ascending (the connector binary-searches
       // them and silently misbuckets otherwise) and MUST match the two
-      // otel-collector-config*.yaml lists. Sub-millisecond boundaries exist
-      // because most xrpld spans sit far below 1ms: a 1ms floor puts nearly
-      // every sample in bucket one and pins p95/p99 to that bucket's edge.
-      // Sub-second boundaries cover RPC, tx and ledger spans. The 2s-4s edges
-      // resolve second-scale consensus spans, and 10s/30s give the
-      // ledger.acquire catch-up tail a home so its p95/p99 stop reading +Inf.
-      // Each edge was chosen from a measured duration distribution for the
-      // spans it covers.
+      // otel-collector-config*.yaml lists. Sub-MILLISECOND boundaries exist
+      // because most xrpld spans sit far below 1ms (tx.preflight means
+      // ~0.012ms): a 1ms floor put >99.99% of samples in bucket one and pinned
+      // every p95/p99 at a constant 0.95ms.
+      // otel-collector-config*.yaml lists. Sub-second boundaries cover
+      // RPC/tx/ledger spans; 2s-4s resolve second-scale consensus spans
+      // (consensus.round ~3.9s, consensus.establish ~1.9s) that used to pile
+      // into one 1s-5s bucket; 10s/30s give the ledger.acquire catch-up tail
+      // (~17% exceeds 5s) a measurable home so its p95/p99 stop reading +Inf.
       buckets = ["0.01ms", "0.05ms", "0.1ms", "0.25ms", "0.5ms", "1ms", "5ms", "10ms", "25ms", "50ms", "100ms", "250ms", "500ms", "1s", "2s", "3s", "4s", "5s", "10s", "30s"]
     }
   }
diff --git a/docker/telemetry/grafana/dashboards/job-queue.json b/docker/telemetry/grafana/dashboards/job-queue.json
index f31a87be24..31baf79bd2 100644
--- a/docker/telemetry/grafana/dashboards/job-queue.json
+++ b/docker/telemetry/grafana/dashboards/job-queue.json
@@ -70,44 +70,61 @@
       "id": 1
     },
     {
-      "title": "Current Job Latency (p99 Gauge) [$xrpl_network_type]",
+      "title": "Slowest Job Types by p99 Latency [$xrpl_network_type]",
       "description": "###### What this is:\n*At-a-glance p99 of how long jobs wait in the queue and how long they run once started.*\n\n###### How it's computed:\n*99th percentile derived from the job wait-time and run-time histograms over the last 5 minutes.*\n\n###### Reading it:\n*Lower is better; green under 100ms, yellow to 1s, red beyond 1s.*\n\n###### Healthy range:\n*Wait and exec p99 under 100ms.*\n\n###### Watch for:\n*p99 wait climbing into the red, meaning worker threads are saturated and jobs are backing up.*\n\n###### Keywords:\n- **Job queue / job type** *(per node)* \u2014 xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n- **Deferred job** *(per node)* \u2014 a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobStarted / recordJobFinished`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type)",
-      "type": "gauge",
+      "type": "bargauge",
       "gridPos": {
         "h": 10,
-        "w": 12,
+        "w": 24,
         "x": 0,
         "y": 1
       },
       "options": {
+        "displayMode": "gradient",
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false
+        },
+        "maxVizHeight": 300,
+        "minVizHeight": 16,
+        "minVizWidth": 8,
+        "namePlacement": "auto",
+        "orientation": "horizontal",
         "reduceOptions": {
           "calcs": ["lastNotNull"],
           "fields": "",
           "values": false
         },
-        "orientation": "auto",
-        "showThresholdLabels": false,
-        "showThresholdMarkers": true
+        "showUnfilled": true,
+        "sizing": "manual",
+        "valueMode": "color"
       },
       "targets": [
         {
           "datasource": {
             "type": "prometheus"
           },
-          "expr": "label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Wait\", \"\", \"\")"
+          "instant": true,
+          "range": false,
+          "expr": "label_replace(histogram_quantile(0.99, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Wait\", \"\", \"\")"
         },
         {
           "datasource": {
             "type": "prometheus"
           },
-          "expr": "label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Exec\", \"\", \"\")"
+          "instant": true,
+          "range": false,
+          "expr": "label_replace(histogram_quantile(0.99, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Exec\", \"\", \"\")"
         }
       ],
       "fieldConfig": {
         "defaults": {
-          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
+          "displayName": "${__field.labels.series} ${__field.labels.job_type} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
           "unit": "\u00b5s",
           "min": 0,
+          "max": 1000000,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -131,7 +148,7 @@
       "id": 2,
       "repeat": "xrpl_network_type",
       "repeatDirection": "h",
-      "maxPerRow": 2
+      "maxPerRow": 1
     },
     {
       "title": "Throughput",
@@ -738,7 +755,7 @@
         "label": "Node",
         "description": "Filter by rippled node (service.instance.id)",
         "type": "query",
-        "query": "label_values(target_info, service_instance_id)",
+        "query": "label_values(job_queued_us_count, service_instance_id)",
         "datasource": {
           "type": "prometheus",
           "uid": "${DS_PROMETHEUS}"
diff --git a/docker/telemetry/otel-collector-config.grafanacloud.yaml b/docker/telemetry/otel-collector-config.grafanacloud.yaml
index 4b488a84cd..01722e383a 100644
--- a/docker/telemetry/otel-collector-config.grafanacloud.yaml
+++ b/docker/telemetry/otel-collector-config.grafanacloud.yaml
@@ -78,13 +78,10 @@ processors:
   batch:
     timeout: 1s
     send_batch_size: 100
-  # Probabilistic tail sampling, on the trace-storage branch only. The
-  # spanmetrics branch sees every span, so span-derived RED metrics stay
-  # exact while stored trace volume is cut to bound cost on a long-running
-  # node. Operators who need full traces raise sampling_percentage to 100.
-  # Note: a log line carries a trace id only when its span was sampled at
-  # the SDK, so sampling here drops traces whose ids are already in the
-  # logs, and log-to-trace links for those resolve to nothing.
+  # Box-only (uncommitted): probabilistic 0.5% tail sampling on the
+  # trace-storage branch only. spanmetrics-derived metrics see 100% of spans
+  # (they run on a separate branch) so RED remain exact; Tempo/Cloud stores
+  # ~1/200 of traces to keep long-running node cost bounded.
   tail_sampling:
     decision_wait: 10s
     num_traces: 50000
@@ -180,15 +177,16 @@ connectors:
       unit: ms
       explicit:
         # Buckets MUST stay strictly ascending (the connector binary-searches
-        # them and silently misbuckets otherwise) and MUST match the Alloy
-        # list and the other collector config. Sub-millisecond boundaries
-        # exist because most xrpld spans sit far below 1ms: a 1ms floor puts
-        # nearly every sample in the first bucket and pins p95/p99 to that
-        # bucket's edge. Sub-second boundaries cover RPC, tx and ledger spans.
-        # The 2s-4s edges resolve second-scale consensus spans, and 10s/30s
-        # give the ledger.acquire catch-up tail a home so its p95/p99 stop
-        # reading as +Inf. Each edge was chosen from a measured duration
-        # distribution for the spans it covers.
+        # them and silently misbuckets otherwise). Sub-MILLISECOND boundaries
+        # (0.01-0.5ms) exist because most xrpld spans are far below 1ms --
+        # tx.preflight means ~0.012ms, tx.preclaim ~0.15ms -- so a 1ms floor
+        # put >99.99% of samples in the first bucket and pinned every p95/p99
+        # at a constant 0.95ms. Sub-second boundaries cover
+        # RPC/tx/ledger spans; the 2s-4s boundaries resolve second-scale
+        # consensus spans (consensus.round ~3.9s, consensus.establish ~1.9s)
+        # that used to pile into a single 1s-5s bucket; 10s/30s give the
+        # ledger.acquire tail (~17% exceeds 5s during catch-up) a measurable
+        # home so its p95/p99 stop reading as +Inf.
         buckets:
           [
             0.01ms,
diff --git a/docker/telemetry/otel-collector-config.yaml b/docker/telemetry/otel-collector-config.yaml
index e3d1850fcd..093d3ddda1 100644
--- a/docker/telemetry/otel-collector-config.yaml
+++ b/docker/telemetry/otel-collector-config.yaml
@@ -168,15 +168,16 @@ connectors:
       unit: ms
       explicit:
         # Buckets MUST stay strictly ascending (the connector binary-searches
-        # them and silently misbuckets otherwise) and MUST match the Alloy
-        # list and the other collector config. Sub-millisecond boundaries
-        # exist because most xrpld spans sit far below 1ms: a 1ms floor puts
-        # nearly every sample in the first bucket and pins p95/p99 to that
-        # bucket's edge. Sub-second boundaries cover RPC, tx and ledger spans.
-        # The 2s-4s edges resolve second-scale consensus spans, and 10s/30s
-        # give the ledger.acquire catch-up tail a home so its p95/p99 stop
-        # reading as +Inf. Each edge was chosen from a measured duration
-        # distribution for the spans it covers.
+        # them and silently misbuckets otherwise). Sub-MILLISECOND boundaries
+        # (0.01-0.5ms) exist because most xrpld spans are far below 1ms --
+        # tx.preflight means ~0.012ms, tx.preclaim ~0.15ms -- so a 1ms floor
+        # put >99.99% of samples in the first bucket and pinned every p95/p99
+        # at a constant 0.95ms. Sub-second boundaries cover
+        # RPC/tx/ledger spans; the 2s-4s boundaries resolve second-scale
+        # consensus spans (consensus.round ~3.9s, consensus.establish ~1.9s)
+        # that used to pile into a single 1s-5s bucket; 10s/30s give the
+        # ledger.acquire tail (~17% exceeds 5s during catch-up) a measurable
+        # home so its p95/p99 stop reading as +Inf.
         buckets:
           [
             0.01ms,

From 6ed6189ba9efc69dc3c602970454f66e0b8c22ba Mon Sep 17 00:00:00 2001
From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
Date: Tue, 22 Sep 2026 15:07:10 +0100
Subject: [PATCH 5/6] docs(telemetry): re-apply the comment rewrite lost to a
 concurrent commit

Commit 54809960ff removed the site measurements from the three collector
comment blocks, but a later commit on this branch was built from an index
holding the pre-rewrite blobs, so the tree reverted while the commit stayed in
the log. History is append-only, so this re-applies the same three blocks.

No non-comment line changes. The bucket lists are untouched and still agree
across the two collector configs and the Alloy config.
---
 docker/telemetry/alloy/config.alloy           | 17 +++++------
 .../otel-collector-config.grafanacloud.yaml   | 30 ++++++++++---------
 docker/telemetry/otel-collector-config.yaml   | 19 ++++++------
 3 files changed, 33 insertions(+), 33 deletions(-)

diff --git a/docker/telemetry/alloy/config.alloy b/docker/telemetry/alloy/config.alloy
index e5d5f613de..a41493e878 100644
--- a/docker/telemetry/alloy/config.alloy
+++ b/docker/telemetry/alloy/config.alloy
@@ -315,15 +315,14 @@ otelcol.connector.spanmetrics "xrpld" {
     explicit {
       // Buckets MUST stay strictly ascending (the connector binary-searches
       // them and silently misbuckets otherwise) and MUST match the two
-      // otel-collector-config*.yaml lists. Sub-MILLISECOND boundaries exist
-      // because most xrpld spans sit far below 1ms (tx.preflight means
-      // ~0.012ms): a 1ms floor put >99.99% of samples in bucket one and pinned
-      // every p95/p99 at a constant 0.95ms.
-      // otel-collector-config*.yaml lists. Sub-second boundaries cover
-      // RPC/tx/ledger spans; 2s-4s resolve second-scale consensus spans
-      // (consensus.round ~3.9s, consensus.establish ~1.9s) that used to pile
-      // into one 1s-5s bucket; 10s/30s give the ledger.acquire catch-up tail
-      // (~17% exceeds 5s) a measurable home so its p95/p99 stop reading +Inf.
+      // otel-collector-config*.yaml lists. Sub-millisecond boundaries exist
+      // because most xrpld spans sit far below 1ms: a 1ms floor puts nearly
+      // every sample in bucket one and pins p95/p99 to that bucket's edge.
+      // Sub-second boundaries cover RPC, tx and ledger spans. The 2s-4s edges
+      // resolve second-scale consensus spans, and 10s/30s give the
+      // ledger.acquire catch-up tail a home so its p95/p99 stop reading +Inf.
+      // Each edge was chosen from a measured duration distribution for the
+      // spans it covers.
       buckets = ["0.01ms", "0.05ms", "0.1ms", "0.25ms", "0.5ms", "1ms", "5ms", "10ms", "25ms", "50ms", "100ms", "250ms", "500ms", "1s", "2s", "3s", "4s", "5s", "10s", "30s"]
     }
   }
diff --git a/docker/telemetry/otel-collector-config.grafanacloud.yaml b/docker/telemetry/otel-collector-config.grafanacloud.yaml
index 01722e383a..4b488a84cd 100644
--- a/docker/telemetry/otel-collector-config.grafanacloud.yaml
+++ b/docker/telemetry/otel-collector-config.grafanacloud.yaml
@@ -78,10 +78,13 @@ processors:
   batch:
     timeout: 1s
     send_batch_size: 100
-  # Box-only (uncommitted): probabilistic 0.5% tail sampling on the
-  # trace-storage branch only. spanmetrics-derived metrics see 100% of spans
-  # (they run on a separate branch) so RED remain exact; Tempo/Cloud stores
-  # ~1/200 of traces to keep long-running node cost bounded.
+  # Probabilistic tail sampling, on the trace-storage branch only. The
+  # spanmetrics branch sees every span, so span-derived RED metrics stay
+  # exact while stored trace volume is cut to bound cost on a long-running
+  # node. Operators who need full traces raise sampling_percentage to 100.
+  # Note: a log line carries a trace id only when its span was sampled at
+  # the SDK, so sampling here drops traces whose ids are already in the
+  # logs, and log-to-trace links for those resolve to nothing.
   tail_sampling:
     decision_wait: 10s
     num_traces: 50000
@@ -177,16 +180,15 @@ connectors:
       unit: ms
       explicit:
         # Buckets MUST stay strictly ascending (the connector binary-searches
-        # them and silently misbuckets otherwise). Sub-MILLISECOND boundaries
-        # (0.01-0.5ms) exist because most xrpld spans are far below 1ms --
-        # tx.preflight means ~0.012ms, tx.preclaim ~0.15ms -- so a 1ms floor
-        # put >99.99% of samples in the first bucket and pinned every p95/p99
-        # at a constant 0.95ms. Sub-second boundaries cover
-        # RPC/tx/ledger spans; the 2s-4s boundaries resolve second-scale
-        # consensus spans (consensus.round ~3.9s, consensus.establish ~1.9s)
-        # that used to pile into a single 1s-5s bucket; 10s/30s give the
-        # ledger.acquire tail (~17% exceeds 5s during catch-up) a measurable
-        # home so its p95/p99 stop reading as +Inf.
+        # them and silently misbuckets otherwise) and MUST match the Alloy
+        # list and the other collector config. Sub-millisecond boundaries
+        # exist because most xrpld spans sit far below 1ms: a 1ms floor puts
+        # nearly every sample in the first bucket and pins p95/p99 to that
+        # bucket's edge. Sub-second boundaries cover RPC, tx and ledger spans.
+        # The 2s-4s edges resolve second-scale consensus spans, and 10s/30s
+        # give the ledger.acquire catch-up tail a home so its p95/p99 stop
+        # reading as +Inf. Each edge was chosen from a measured duration
+        # distribution for the spans it covers.
         buckets:
           [
             0.01ms,
diff --git a/docker/telemetry/otel-collector-config.yaml b/docker/telemetry/otel-collector-config.yaml
index 093d3ddda1..e3d1850fcd 100644
--- a/docker/telemetry/otel-collector-config.yaml
+++ b/docker/telemetry/otel-collector-config.yaml
@@ -168,16 +168,15 @@ connectors:
       unit: ms
       explicit:
         # Buckets MUST stay strictly ascending (the connector binary-searches
-        # them and silently misbuckets otherwise). Sub-MILLISECOND boundaries
-        # (0.01-0.5ms) exist because most xrpld spans are far below 1ms --
-        # tx.preflight means ~0.012ms, tx.preclaim ~0.15ms -- so a 1ms floor
-        # put >99.99% of samples in the first bucket and pinned every p95/p99
-        # at a constant 0.95ms. Sub-second boundaries cover
-        # RPC/tx/ledger spans; the 2s-4s boundaries resolve second-scale
-        # consensus spans (consensus.round ~3.9s, consensus.establish ~1.9s)
-        # that used to pile into a single 1s-5s bucket; 10s/30s give the
-        # ledger.acquire tail (~17% exceeds 5s during catch-up) a measurable
-        # home so its p95/p99 stop reading as +Inf.
+        # them and silently misbuckets otherwise) and MUST match the Alloy
+        # list and the other collector config. Sub-millisecond boundaries
+        # exist because most xrpld spans sit far below 1ms: a 1ms floor puts
+        # nearly every sample in the first bucket and pins p95/p99 to that
+        # bucket's edge. Sub-second boundaries cover RPC, tx and ledger spans.
+        # The 2s-4s edges resolve second-scale consensus spans, and 10s/30s
+        # give the ledger.acquire catch-up tail a home so its p95/p99 stop
+        # reading as +Inf. Each edge was chosen from a measured duration
+        # distribution for the spans it covers.
         buckets:
           [
             0.01ms,

From 4bf76ce2ffbea6ac2b89bda0f542e2749473e336 Mon Sep 17 00:00:00 2001
From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
Date: Tue, 22 Sep 2026 15:11:28 +0100
Subject: [PATCH 6/6]   metrics_flush_interval = "15s"

Signed-off-by: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
---
 docker/telemetry/alloy/config.alloy | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docker/telemetry/alloy/config.alloy b/docker/telemetry/alloy/config.alloy
index a41493e878..c49cdcddc7 100644
--- a/docker/telemetry/alloy/config.alloy
+++ b/docker/telemetry/alloy/config.alloy
@@ -310,6 +310,12 @@ otelcol.processor.transform "tier" {
 otelcol.connector.spanmetrics "xrpld" {
   namespace = "span"
 
+  // Flush every 15s, matching otel-collector-config.grafanacloud.yaml. Alloy
+  // defaults to 60s, which is longer than [$__rate_interval] at short dashboard
+  // ranges, so a rate() panel gets under two samples per window and renders
+  // "No data". Costs ~4x span-metric datapoints.
+  metrics_flush_interval = "15s"
+
   histogram {
     unit = "ms"
     explicit {