From 844248339d9ad9dc3687e4f0614e6c4a5fbe5363 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Thu, 6 Aug 2026 14:20:53 +0100 Subject: [PATCH 1/4] chore(telemetry): update collector, Tempo and Grafana to current releases Bumps the three images this branch pins: otel-collector-contrib 0.121.0 -> 0.158.0 tempo 2.7.2 -> 2.9.4 grafana 11.5.2 -> 13.1.2 Verified locally against the new images: the collector config passes `validate` under 0.158.0 and tempo.yaml passes `-config.verify=true` under 2.9.4, both unchanged. The collector's log path uses the generic otlphttp exporter (otlphttp/loki), not the dedicated loki exporter removed upstream in v0.147.0, so the pipeline is unaffected by that removal. Grafana crosses two majors. Operator-visible consequences are handled on the branches that own the affected files: Grafana 13 enables the renderAuthJWT feature toggle by default, so the image renderer now requires a matching renderer_token on both the server and the renderer container. --- docker/telemetry/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/telemetry/docker-compose.yml b/docker/telemetry/docker-compose.yml index bf74dad9be..a910e1a940 100644 --- a/docker/telemetry/docker-compose.yml +++ b/docker/telemetry/docker-compose.yml @@ -21,7 +21,7 @@ services: # OpenTelemetry Collector: receives spans from xrpld via OTLP protocol, # batches them for efficiency, and forwards to Tempo for storage. otel-collector: - image: otel/opentelemetry-collector-contrib:0.121.0 + image: otel/opentelemetry-collector-contrib:0.158.0 command: ["--config=/etc/otel-collector-config.yaml"] ports: - "4317:4317" # OTLP gRPC receiver @@ -38,7 +38,7 @@ services: # Grafana Tempo: distributed tracing backend that stores and indexes # spans. Queryable via TraceQL in Grafana Explore. tempo: - image: grafana/tempo:2.7.2 + image: grafana/tempo:2.9.4 command: ["-config.file=/etc/tempo.yaml"] ports: - "3200:3200" # Tempo HTTP API (health check, query) @@ -53,7 +53,7 @@ services: # Grafana: visualization UI with Tempo pre-configured as a datasource. # Anonymous admin access enabled for local development convenience. grafana: - image: grafana/grafana:11.5.2 + image: grafana/grafana:13.1.2 environment: - GF_AUTH_ANONYMOUS_ENABLED=true # No login required for local dev - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin # Full access without auth From bf5aae2f24c550cc46708292d8ed0582432db8fa Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Thu, 6 Aug 2026 14:23:13 +0100 Subject: [PATCH 2/4] chore(telemetry): update Prometheus to v3.13.2 Moves off the v2.53 LTS line, which this branch pinned, onto the current v3 release, and corrects the accompanying comment that named the old line. Verified against the new image rather than assumed, since this crosses a major version: - prometheus.yml passes `promtool check config` under v3.13.2 - all 391 unique dashboard PromQL queries parse under the v3 parser - all 13 alert-rule expressions parse under the v3 parser The config uses none of the surfaces v3 changed: no remote_write, no holt_winters, no offset modifiers. --- docker/telemetry/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/telemetry/docker-compose.yml b/docker/telemetry/docker-compose.yml index bbfb0c9db9..03c951cad5 100644 --- a/docker/telemetry/docker-compose.yml +++ b/docker/telemetry/docker-compose.yml @@ -56,8 +56,8 @@ services: - xrpld-telemetry prometheus: - # Pinned to the v2.53 LTS line for reproducible, config-stable runs. - image: prom/prometheus:v2.53.3 + # Pinned to an exact patch release for reproducible, config-stable runs. + image: prom/prometheus:v3.13.2 ports: - "9090:9090" volumes: From ef05e1b2a172f3d17e4a302c17a10a587d2a1225 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Thu, 6 Aug 2026 19:01:00 +0100 Subject: [PATCH 3/4] fix(telemetry): filter zero denominators instead of clamping them Panels 21 (NuDB Read Latency) and 23 (NuDB Read Found Ratio) on the Ledger Data & Sync dashboard guarded their divisor with clamp_min(, 1). clamp_min raises the value, not just the zero case, so any node reading fewer than 1 block per second was divided by a fabricated 1 instead of by its real read count. Replace the clamp with the filter ( > 0). A comparison without the bool modifier drops the sample rather than rewriting it, so these panels now show no data instead of a wrong number. Measured over 7 days: five nodes fall below 1 read/s. On validator-0 the clamp reported 2.726 us/read against a true 5.493, and on nonUNLmalloc-tc it reported 0 us/read, which cannot occur. The error is largest exactly when panel 21 is used as the bottleneck discriminator during a stall, because that is when the read rate collapses toward zero. Matches the existing idiom on the same nodestore_state metric family in the NodeStore Write vs Read Latency panel. --- docker/telemetry/grafana/dashboards/ledger-data-sync.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index 2f01f834d9..5f02984c72 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -1248,7 +1248,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_duration_us\"}[$__rate_interval])) / clamp_min(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])), 1), \"series\", \"NuDB us/read\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_duration_us\"}[$__rate_interval])) / (sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])) > 0), \"series\", \"NuDB us/read\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" } ], "fieldConfig": { @@ -1332,7 +1332,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval])) / clamp_min(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])), 1), \"series\", \"NuDB Found Ratio\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval])) / (sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])) > 0), \"series\", \"NuDB Found Ratio\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" } ], "fieldConfig": { From 84ef8cbf3355a752ced56916d377ad4e664343d4 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Thu, 6 Aug 2026 19:01:15 +0100 Subject: [PATCH 4/4] fix(telemetry): drop unmeasurable Queue Bypass Ratio panel The Transaction Overview panel "Queue Bypass Ratio (Direct Apply vs Enqueue)" reported a confident 0.50 on every node while the true bypass rate was zero. The two spans it divided are not disjoint alternatives: txq.apply_direct is a child of txq.enqueue. TxQ.cpp creates the apply_direct span as the first statement of tryDirectApply(), ahead of the account, sequence and fee-level guards, and tryDirectApply() is called from inside the live enqueue scope. The span therefore counts attempts, so the denominator direct + enqueue counts each transaction twice and pins the ratio to one half algebraically. Measured on a four-node fleet: 6082443 direct against 6082877 enqueue over the same population, panel output 0.5000170 on three nodes and 0.5000000 on the fourth. Grouping txq.enqueue by txq_status over seven days returns only "rejected" -- no transaction has ever taken the direct-apply path. Remove the panel rather than repoint it. A correct expression using txq_status as the disjoint discriminator would render permanently empty on this fleet, which reads no better than a wrong number. Widen the band partner "TxQ Enqueue Rate by Transaction Type" from 12 to 24 columns so the y=48 band still fills the grid. Every band in all ten dashboards sums to 24 columns; leaving a half-width hole would be the only exception. Panel order and every other panel's position, width and height are unchanged. The runbook already listed txq.apply_direct as available but not paneled, so that row becomes accurate. Rows describing the span itself are untouched -- the span and its metric are unchanged. --- .../dashboards/transaction-overview.json | 42 +------------------ 1 file changed, 1 insertion(+), 41 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json index 4efec5de79..94cc8be828 100644 --- a/docker/telemetry/grafana/dashboards/transaction-overview.json +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -555,7 +555,7 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 12, + "w": 24, "x": 0, "y": 48 }, @@ -589,46 +589,6 @@ "overrides": [] } }, - { - "title": "Queue Bypass Ratio (Direct Apply vs Enqueue)", - "description": "**What:** Fraction of transactions that applied straight to the open ledger versus those that had to be queued.\n**How it's computed:** Ratio of direct applies to direct-plus-queued over a 5-minute window, per node.\n**Reading it:** A high fraction means the network is keeping up without escalation.\n**Healthy range:** Near 1.0 in normal conditions.\n**Watch for:** A falling fraction is the cleanest single signal the network has entered sustained fee escalation.\n**Source:** src/xrpld/app/misc/detail/TxQ.cpp:TxQ::tryDirectApply, TxQ::apply", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 48 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc", - "maxHeight": 600 - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "sum by (service_instance_id) (rate(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"txq.apply_direct\"}[5m])) / clamp_min(sum by (service_instance_id) (rate(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"txq.apply_direct\"}[5m])) + sum by (service_instance_id) (rate(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"txq.enqueue\"}[5m])), 1)", - "legendFormat": "Direct-Apply Fraction [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "custom": { - "axisLabel": "Bypass Fraction", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 3 - } - }, - "overrides": [] - } - }, { "title": "Queue Accept (Drain) Duration per Ledger", "description": "**What:** Time spent draining queued transactions into a newly closed ledger.\n**How it's computed:** 95th and 50th percentile of per-ledger drain durations over a 5-minute window, per node.\n**Reading it:** Rises as the queue holds more transactions to process at close.\n**Healthy range:** A few milliseconds when the queue is light.\n**Watch for:** Rising drain time signals queue pressure at ledger close.\n**Source:** src/xrpld/app/misc/detail/TxQ.cpp:TxQ::accept",