From 844248339d9ad9dc3687e4f0614e6c4a5fbe5363 Mon Sep 17 00:00:00 2001
From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
Date: Thu, 6 Aug 2026 14:20:53 +0100
Subject: [PATCH 1/9] chore(telemetry): update collector, Tempo and Grafana to
 current releases

Bumps the three images this branch pins:
  otel-collector-contrib  0.121.0 -> 0.158.0
  tempo                   2.7.2   -> 2.9.4
  grafana                 11.5.2  -> 13.1.2

Verified locally against the new images: the collector config passes
`validate` under 0.158.0 and tempo.yaml passes `-config.verify=true` under
2.9.4, both unchanged. The collector's log path uses the generic otlphttp
exporter (otlphttp/loki), not the dedicated loki exporter removed upstream in
v0.147.0, so the pipeline is unaffected by that removal.

Grafana crosses two majors. Operator-visible consequences are handled on the
branches that own the affected files: Grafana 13 enables the renderAuthJWT
feature toggle by default, so the image renderer now requires a matching
renderer_token on both the server and the renderer container.
---
 docker/telemetry/docker-compose.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docker/telemetry/docker-compose.yml b/docker/telemetry/docker-compose.yml
index bf74dad9be..a910e1a940 100644
--- a/docker/telemetry/docker-compose.yml
+++ b/docker/telemetry/docker-compose.yml
@@ -21,7 +21,7 @@ services:
   # OpenTelemetry Collector: receives spans from xrpld via OTLP protocol,
   # batches them for efficiency, and forwards to Tempo for storage.
   otel-collector:
-    image: otel/opentelemetry-collector-contrib:0.121.0
+    image: otel/opentelemetry-collector-contrib:0.158.0
     command: ["--config=/etc/otel-collector-config.yaml"]
     ports:
       - "4317:4317" # OTLP gRPC receiver
@@ -38,7 +38,7 @@ services:
   # Grafana Tempo: distributed tracing backend that stores and indexes
   # spans. Queryable via TraceQL in Grafana Explore.
   tempo:
-    image: grafana/tempo:2.7.2
+    image: grafana/tempo:2.9.4
     command: ["-config.file=/etc/tempo.yaml"]
     ports:
       - "3200:3200" # Tempo HTTP API (health check, query)
@@ -53,7 +53,7 @@ services:
   # Grafana: visualization UI with Tempo pre-configured as a datasource.
   # Anonymous admin access enabled for local development convenience.
   grafana:
-    image: grafana/grafana:11.5.2
+    image: grafana/grafana:13.1.2
     environment:
       - GF_AUTH_ANONYMOUS_ENABLED=true # No login required for local dev
       - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin # Full access without auth

From bf5aae2f24c550cc46708292d8ed0582432db8fa Mon Sep 17 00:00:00 2001
From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
Date: Thu, 6 Aug 2026 14:23:13 +0100
Subject: [PATCH 2/9] chore(telemetry): update Prometheus to v3.13.2

Moves off the v2.53 LTS line, which this branch pinned, onto the current
v3 release, and corrects the accompanying comment that named the old line.

Verified against the new image rather than assumed, since this crosses a
major version:
  - prometheus.yml passes `promtool check config` under v3.13.2
  - all 391 unique dashboard PromQL queries parse under the v3 parser
  - all 13 alert-rule expressions parse under the v3 parser

The config uses none of the surfaces v3 changed: no remote_write, no
holt_winters, no offset modifiers.
---
 docker/telemetry/docker-compose.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/telemetry/docker-compose.yml b/docker/telemetry/docker-compose.yml
index bbfb0c9db9..03c951cad5 100644
--- a/docker/telemetry/docker-compose.yml
+++ b/docker/telemetry/docker-compose.yml
@@ -56,8 +56,8 @@ services:
       - xrpld-telemetry
 
   prometheus:
-    # Pinned to the v2.53 LTS line for reproducible, config-stable runs.
-    image: prom/prometheus:v2.53.3
+    # Pinned to an exact patch release for reproducible, config-stable runs.
+    image: prom/prometheus:v3.13.2
     ports:
       - "9090:9090"
     volumes:

From 81915498d75a14f4bd7cc0c22f246b707cb5bc0f Mon Sep 17 00:00:00 2001
From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
Date: Thu, 6 Aug 2026 14:24:22 +0100
Subject: [PATCH 3/9] chore(telemetry): update Loki to 3.7.6 and refresh
 version references

Bumps the Loki pin this branch introduced, and updates the two docs that
name the version so they do not drift from the compose file:
  - OpenTelemetryPlan/09-data-collection-reference.md (log backend section)
  - OpenTelemetryPlan/Phase8_taskList.md (compose snippet)

The service runs the config bundled in the image rather than one from this
repo; `-verify-config` reports "config is valid" under 3.7.6, and native OTLP
ingestion at /otlp is unchanged across 3.4 -> 3.7.
---
 OpenTelemetryPlan/09-data-collection-reference.md | 2 +-
 OpenTelemetryPlan/Phase8_taskList.md              | 2 +-
 docker/telemetry/docker-compose.yml               | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/OpenTelemetryPlan/09-data-collection-reference.md b/OpenTelemetryPlan/09-data-collection-reference.md
index 03c32fb428..f2b3d336a9 100644
--- a/OpenTelemetryPlan/09-data-collection-reference.md
+++ b/OpenTelemetryPlan/09-data-collection-reference.md
@@ -967,7 +967,7 @@ Bidirectional linking between logs and traces is configured via Grafana datasour
 
 ### Loki Backend
 
-Grafana Loki (v3.4.2) serves as the log storage backend. It receives log entries from the OTel Collector's `otlphttp/loki` exporter via the native OTLP endpoint at `http://loki:3100/otlp`.
+Grafana Loki (v3.7.6) serves as the log storage backend. It receives log entries from the OTel Collector's `otlphttp/loki` exporter via the native OTLP endpoint at `http://loki:3100/otlp`.
 
 ### LogQL Query Examples
 
diff --git a/OpenTelemetryPlan/Phase8_taskList.md b/OpenTelemetryPlan/Phase8_taskList.md
index 7849990858..8c4d80e80b 100644
--- a/OpenTelemetryPlan/Phase8_taskList.md
+++ b/OpenTelemetryPlan/Phase8_taskList.md
@@ -76,7 +76,7 @@
   - Add Loki service:
     ```yaml
     loki:
-      image: grafana/loki:3.4.2
+      image: grafana/loki:3.7.6
       ports:
         - "3100:3100"
       command: -config.file=/etc/loki/local-config.yaml
diff --git a/docker/telemetry/docker-compose.yml b/docker/telemetry/docker-compose.yml
index 39ca0e34dd..612be4321b 100644
--- a/docker/telemetry/docker-compose.yml
+++ b/docker/telemetry/docker-compose.yml
@@ -69,7 +69,7 @@ services:
   # Collector exports via otlphttp to Loki's /otlp endpoint.
   # Query logs via Grafana Explore -> Loki at http://localhost:3000.
   loki:
-    image: grafana/loki:3.4.2
+    image: grafana/loki:3.7.6
     ports:
       - "3100:3100"
     command: -config.file=/etc/loki/local-config.yaml

From 78b29dbd0e0300106f5f84bf075276b3946b1657 Mon Sep 17 00:00:00 2001
From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
Date: Thu, 6 Aug 2026 14:25:59 +0100
Subject: [PATCH 4/9] chore(telemetry): pin the image renderer and wire its
 auth token

The renderer was the only service in the stack on a floating :latest tag, so
the stack was not reproducible: a renderer release could change under a
checkout that had not been touched. Pins it to v5.12.0, matching how every
other service here is pinned.

Also wires the shared render token. Grafana 13 enables the renderAuthJWT
feature toggle by default, and the renderer rejects requests unless both sides
present the same non-default token, so pinning alone would leave rendering
broken once the Grafana bump merges forward. Both values come from the same
GF_RENDERING_RENDERER_TOKEN variable and cannot drift apart; the local default
keeps `docker compose up` working with no setup.

Verified `docker compose config` resolves and both AUTH_TOKEN and
GF_RENDERING_RENDERER_TOKEN expand to the same value.
---
 docker/telemetry/docker-compose.yml | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/docker/telemetry/docker-compose.yml b/docker/telemetry/docker-compose.yml
index cffe3d139a..78564d3de9 100644
--- a/docker/telemetry/docker-compose.yml
+++ b/docker/telemetry/docker-compose.yml
@@ -110,6 +110,10 @@ services:
       # the callback URL is how the renderer fetches the panel from grafana.
       - GF_RENDERING_SERVER_URL=http://renderer:8081/render
       - GF_RENDERING_CALLBACK_URL=http://grafana:3000/
+      # Grafana 13 turns the renderAuthJWT feature toggle on by default, which
+      # makes the renderer reject requests unless both sides share a token that
+      # is not the "-" default. Must match AUTH_TOKEN on the renderer service.
+      - GF_RENDERING_RENDERER_TOKEN=${GF_RENDERING_RENDERER_TOKEN:-xrpld-local-render}
     # SMTP for the critical-tier email receiver and the Slack webhook / email
     # address for the contact points all come from the env_file above, which
     # injects them straight into the container environment for Grafana to
@@ -133,7 +137,14 @@ services:
   # Grafana image renderer: a sidecar that renders panels/dashboards to PNG
   # for image export and alerting. Grafana calls it at http://renderer:8081.
   renderer:
-    image: grafana/grafana-image-renderer:latest
+    # Pinned like every other image in this stack so a stack that worked
+    # yesterday still works today. Upstream maintains only the newest release,
+    # which is expected to serve all supported Grafana versions.
+    image: grafana/grafana-image-renderer:v5.12.0
+    environment:
+      # Shared secret for the JWT-authenticated render requests Grafana 13
+      # sends. Must match GF_RENDERING_RENDERER_TOKEN on the grafana service.
+      - AUTH_TOKEN=${GF_RENDERING_RENDERER_TOKEN:-xrpld-local-render}
     ports:
       - "8081:8081" # Renderer HTTP endpoint (called by grafana)
     networks:

From af3767cdb807068a2a730c2967789af45d86d952 Mon Sep 17 00:00:00 2001
From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
Date: Thu, 6 Aug 2026 14:27:38 +0100
Subject: [PATCH 5/9] docs(telemetry): document the render token alert
 screenshots now need

Grafana 13 enables the renderAuthJWT feature toggle by default, so the image
renderer silently drops render requests carrying the default token and alert
notifications arrive with no panel image. The runbook's alerting section
covered delivery but not this failure mode.

Adds how the token is wired (one variable feeding both services, so they cannot
drift), how to override it, and the two-command check for when images stop
appearing.
---
 docs/telemetry-runbook.md | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md
index 1ae7059aa7..49b2827a45 100644
--- a/docs/telemetry-runbook.md
+++ b/docs/telemetry-runbook.md
@@ -2286,6 +2286,33 @@ Three traps worth knowing before you edit this file:
 To add a third destination (PagerDuty, Opsgenie, a custom webhook), add a receiver
 to the relevant contact point.
 
+#### Panel screenshots on alerts need a matching render token
+
+Alert notifications that carry a panel image are rendered by the `renderer`
+sidecar, not by Grafana itself. Grafana 13 enables the `renderAuthJWT` feature
+toggle by default, so the renderer rejects any request whose token is missing or
+still the `-` default — notifications then arrive with no image.
+
+`docker-compose.yml` feeds both sides from one variable, so they cannot drift:
+`GF_RENDERING_RENDERER_TOKEN` on the `grafana` service and `AUTH_TOKEN` on the
+`renderer` service both read `${GF_RENDERING_RENDERER_TOKEN}`, defaulting to a
+local development value. Override it in the environment to use your own:
+
+```bash
+GF_RENDERING_RENDERER_TOKEN=$(openssl rand -hex 16) \
+    docker compose -f docker/telemetry/docker-compose.yml up -d grafana renderer
+```
+
+If images stop appearing, check that the two containers agree — a token set on
+only one side fails exactly this way:
+
+```bash
+docker compose -f docker/telemetry/docker-compose.yml exec grafana \
+    printenv GF_RENDERING_RENDERER_TOKEN
+docker compose -f docker/telemetry/docker-compose.yml exec renderer \
+    printenv AUTH_TOKEN
+```
+
 #### Deploying alerts to Grafana Cloud
 
 Grafana Cloud has **no provisioning filesystem**, so these `apiVersion: 1` files

From d45bdeee9d79d154e23d3b95c393a49a65e73bd7 Mon Sep 17 00:00:00 2001
From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
Date: Thu, 6 Aug 2026 18:35:51 +0100
Subject: [PATCH 6/9] added core filters to queries and legends

Signed-off-by: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
---
 .../grafana/dashboards/consensus-health.json  | 18 ++---
 .../dashboards/log-derived-insights.json      | 72 +++++++++----------
 .../grafana/dashboards/node-health.json       | 26 +++----
 .../dashboards/transaction-overview.json      |  4 +-
 docs/telemetry-runbook.md                     | 53 +++++++-------
 5 files changed, 87 insertions(+), 86 deletions(-)

diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json
index cd1e796a33..e77e207c10 100644
--- a/docker/telemetry/grafana/dashboards/consensus-health.json
+++ b/docker/telemetry/grafana/dashboards/consensus-health.json
@@ -279,7 +279,7 @@
             "uid": "${DS_TEMPO}"
           },
           "queryType": "traceql",
-          "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\"} | quantile_over_time(span.round_time_ms, .5)",
+          "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | quantile_over_time(span.round_time_ms, .5)",
           "legendFormat": "P50 Time to Consensus [{{resource.service.instance.id}}]",
           "refId": "A"
         },
@@ -290,7 +290,7 @@
           },
           "queryType": "traceql",
           "refId": "B",
-          "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\"} | quantile_over_time(span.round_time_ms, .95)",
+          "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | quantile_over_time(span.round_time_ms, .95)",
           "legendFormat": "P95 Time to Consensus [{{resource.service.instance.id}}]"
         }
       ],
@@ -333,7 +333,7 @@
             "uid": "${DS_TEMPO}"
           },
           "queryType": "traceql",
-          "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\"} | avg_over_time(span.round_time_ms)",
+          "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | avg_over_time(span.round_time_ms)",
           "legendFormat": "Avg Time to Consensus [{{resource.service.instance.id}}]",
           "refId": "A"
         }
@@ -383,7 +383,7 @@
           },
           "queryType": "traceql",
           "refId": "A",
-          "query": "{name=\"consensus.establish\" && resource.service.instance.id=~\"$node\"} | count_over_time() by (span.establish_count)",
+          "query": "{name=\"consensus.establish\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | count_over_time() by (span.establish_count)",
           "legendFormat": "{{span.establish_count}} Rounds"
         }
       ],
@@ -427,7 +427,7 @@
             "uid": "${DS_TEMPO}"
           },
           "queryType": "traceql",
-          "query": "{name=\"consensus.round\" && resource.service.instance.id=~\"$node\"} | quantile_over_time(span.previous_round_time_ms, .95)",
+          "query": "{name=\"consensus.round\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | quantile_over_time(span.previous_round_time_ms, .95)",
           "legendFormat": "P95 Previous Round [{{resource.service.instance.id}}]",
           "refId": "A"
         }
@@ -730,7 +730,7 @@
             "uid": "${DS_TEMPO}"
           },
           "queryType": "traceql",
-          "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\"} | avg_over_time(span.close_time_vote_bins)",
+          "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | avg_over_time(span.close_time_vote_bins)",
           "legendFormat": "Avg Vote Bins",
           "refId": "A"
         },
@@ -740,7 +740,7 @@
             "uid": "${DS_TEMPO}"
           },
           "queryType": "traceql",
-          "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\"} | avg_over_time(span.close_resolution_ms)",
+          "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | avg_over_time(span.close_resolution_ms)",
           "legendFormat": "Avg Resolution (ms)",
           "refId": "B"
         }
@@ -789,7 +789,7 @@
             "uid": "${DS_TEMPO}"
           },
           "queryType": "traceql",
-          "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && span.resolution_direction=~\"$resolution_direction\"} | count_over_time() by (span.resolution_direction)",
+          "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\" && span.resolution_direction=~\"$resolution_direction\"} | count_over_time() by (span.resolution_direction)",
           "legendFormat": "{{span.resolution_direction}}",
           "refId": "A"
         }
@@ -861,7 +861,7 @@
             "uid": "${DS_TEMPO}"
           },
           "queryType": "traceql",
-          "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\"} | count_over_time() by (span.close_time_vote_bins)",
+          "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | count_over_time() by (span.close_time_vote_bins)",
           "legendFormat": "{{span.close_time_vote_bins}} Distinct Position(s)",
           "refId": "A"
         }
diff --git a/docker/telemetry/grafana/dashboards/log-derived-insights.json b/docker/telemetry/grafana/dashboards/log-derived-insights.json
index 6937748bc8..8f7aedfbd5 100644
--- a/docker/telemetry/grafana/dashboards/log-derived-insights.json
+++ b/docker/telemetry/grafana/dashboards/log-derived-insights.json
@@ -312,10 +312,10 @@
             "uid": "${DS_LOKI}"
           },
           "refId": "A",
-          "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | severity =~ `ERR|FTL` [$__range])))",
+          "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | severity =~ `ERR|FTL` [$__range])))",
           "instant": true,
           "queryType": "instant",
-          "legendFormat": "Errors [{{service_instance_id}}]"
+          "legendFormat": "Errors [{{service_instance_id}}] [{{xrpl_network_type}}]"
         }
       ],
       "id": 3
@@ -383,10 +383,10 @@
             "uid": "${DS_LOKI}"
           },
           "refId": "A",
-          "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `Resource` |~ `useless data|unwanted data|init drop` [$__range])))",
+          "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `Resource` |~ `useless data|unwanted data|init drop` [$__range])))",
           "instant": true,
           "queryType": "instant",
-          "legendFormat": "Charges [{{service_instance_id}}]"
+          "legendFormat": "Charges [{{service_instance_id}}] [{{xrpl_network_type}}]"
         }
       ],
       "id": 4
@@ -454,10 +454,10 @@
             "uid": "${DS_LOKI}"
           },
           "refId": "A",
-          "expr": "topk($topn, sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `Resource` | regexp `\\(\\$(?P<fee>[0-9]+)\\)` | unwrap fee [$__range])))",
+          "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `Resource` | regexp `\\(\\$(?P<fee>[0-9]+)\\)` | unwrap fee [$__range])))",
           "instant": true,
           "queryType": "instant",
-          "legendFormat": "Fee Charged [{{service_instance_id}}]"
+          "legendFormat": "Fee Charged [{{service_instance_id}}] [{{xrpl_network_type}}]"
         }
       ],
       "id": 5
@@ -525,10 +525,10 @@
             "uid": "${DS_LOKI}"
           },
           "refId": "A",
-          "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ManifestCache` |~ `Manifest: (Stale|Invalid|Revoked|UntrustedCapacity)` [$__range])))",
+          "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ManifestCache` |~ `Manifest: (Stale|Invalid|Revoked|UntrustedCapacity)` [$__range])))",
           "instant": true,
           "queryType": "instant",
-          "legendFormat": "Rejections [{{service_instance_id}}]"
+          "legendFormat": "Rejections [{{service_instance_id}}] [{{xrpl_network_type}}]"
         }
       ],
       "id": 6
@@ -596,10 +596,10 @@
             "uid": "${DS_LOKI}"
           },
           "refId": "A",
-          "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `LedgerConsensus` | severity =~ `WRN|ERR|FTL` [$__range])))",
+          "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `LedgerConsensus` | severity =~ `WRN|ERR|FTL` [$__range])))",
           "instant": true,
           "queryType": "instant",
-          "legendFormat": "Consensus Events [{{service_instance_id}}]"
+          "legendFormat": "Consensus Events [{{service_instance_id}}] [{{xrpl_network_type}}]"
         }
       ],
       "id": 7
@@ -667,10 +667,10 @@
             "uid": "${DS_LOKI}"
           },
           "refId": "A",
-          "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `LoadMonitor` |~ `Job: .* run: [0-9]+ms wait: [0-9]+ms` [$__range])))",
+          "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `LoadMonitor` |~ `Job: .* run: [0-9]+ms wait: [0-9]+ms` [$__range])))",
           "instant": true,
           "queryType": "instant",
-          "legendFormat": "Breaches [{{service_instance_id}}]"
+          "legendFormat": "Breaches [{{service_instance_id}}] [{{xrpl_network_type}}]"
         }
       ],
       "id": 8
@@ -738,10 +738,10 @@
             "uid": "${DS_LOKI}"
           },
           "refId": "A",
-          "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" |~ `STATE->(syncing|connected|tracking|disconnected)` [$__range])))",
+          "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" |~ `STATE->(syncing|connected|tracking|disconnected)` [$__range])))",
           "instant": true,
           "queryType": "instant",
-          "legendFormat": "Transitions [{{service_instance_id}}]"
+          "legendFormat": "Transitions [{{service_instance_id}}] [{{xrpl_network_type}}]"
         }
       ],
       "id": 9
@@ -809,10 +809,10 @@
             "uid": "${DS_LOKI}"
           },
           "refId": "A",
-          "expr": "topk($topn, sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | regexp `dupe:(?P<dupe>[0-9]+)` | unwrap dupe [$__range])))",
+          "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | regexp `dupe:(?P<dupe>[0-9]+)` | unwrap dupe [$__range])))",
           "instant": true,
           "queryType": "instant",
-          "legendFormat": "Duplicates [{{service_instance_id}}]"
+          "legendFormat": "Duplicates [{{service_instance_id}}] [{{xrpl_network_type}}]"
         }
       ],
       "id": 10
@@ -1199,8 +1199,8 @@
             "uid": "${DS_LOKI}"
           },
           "refId": "A",
-          "legendFormat": "Accepted [{{service_instance_id}}]",
-          "expr": "sum by (service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ManifestCache` | severity =~ \"$severity\" |~ `Manifest: Accepted` [$__auto])) or vector(0)"
+          "legendFormat": "Accepted [{{service_instance_id}}] [{{xrpl_network_type}}]",
+          "expr": "sum by (service_instance_id, xrpl_network_type) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ManifestCache` | severity =~ \"$severity\" |~ `Manifest: Accepted` [$__auto])) or vector(0)"
         },
         {
           "datasource": {
@@ -1208,8 +1208,8 @@
             "uid": "${DS_LOKI}"
           },
           "refId": "B",
-          "legendFormat": "Rejected [{{service_instance_id}}]",
-          "expr": "sum by (service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ManifestCache` | severity =~ \"$severity\" |~ `Manifest: (Stale|Invalid|Revoked|UntrustedCapacity)` [$__auto])) or vector(0)"
+          "legendFormat": "Rejected [{{service_instance_id}}] [{{xrpl_network_type}}]",
+          "expr": "sum by (service_instance_id, xrpl_network_type) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ManifestCache` | severity =~ \"$severity\" |~ `Manifest: (Stale|Invalid|Revoked|UntrustedCapacity)` [$__auto])) or vector(0)"
         }
       ],
       "id": 19,
@@ -1716,8 +1716,8 @@
             "uid": "${DS_LOKI}"
           },
           "refId": "A",
-          "legendFormat": "Duplicate Ratio [{{service_instance_id}}]",
-          "expr": "sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `dupe:(?P<dupe>[0-9]+)` | unwrap dupe [$__auto])) / (sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `dupe:(?P<dupe>[0-9]+)` | unwrap dupe [$__auto])) + sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `good:(?P<good>[0-9]+)` | unwrap good [$__auto])))"
+          "legendFormat": "Duplicate Ratio [{{service_instance_id}}] [{{xrpl_network_type}}]",
+          "expr": "sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `dupe:(?P<dupe>[0-9]+)` | unwrap dupe [$__auto])) / (sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `dupe:(?P<dupe>[0-9]+)` | unwrap dupe [$__auto])) + sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `good:(?P<good>[0-9]+)` | unwrap good [$__auto])))"
         }
       ],
       "id": 26,
@@ -1775,8 +1775,8 @@
             "uid": "${DS_LOKI}"
           },
           "refId": "A",
-          "legendFormat": "Good [{{service_instance_id}}]",
-          "expr": "sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `good:(?P<good>[0-9]+)` | unwrap good [$__auto]))"
+          "legendFormat": "Good [{{service_instance_id}}] [{{xrpl_network_type}}]",
+          "expr": "sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `good:(?P<good>[0-9]+)` | unwrap good [$__auto]))"
         },
         {
           "datasource": {
@@ -1784,8 +1784,8 @@
             "uid": "${DS_LOKI}"
           },
           "refId": "B",
-          "legendFormat": "Duplicate [{{service_instance_id}}]",
-          "expr": "sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `dupe:(?P<dupe>[0-9]+)` | unwrap dupe [$__auto]))"
+          "legendFormat": "Duplicate [{{service_instance_id}}] [{{xrpl_network_type}}]",
+          "expr": "sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `dupe:(?P<dupe>[0-9]+)` | unwrap dupe [$__auto]))"
         },
         {
           "datasource": {
@@ -1793,8 +1793,8 @@
             "uid": "${DS_LOKI}"
           },
           "refId": "C",
-          "legendFormat": "Acquire Timeouts [{{service_instance_id}}]",
-          "expr": "sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `Acquire \\S+ timeouts:(?P<timeouts>[0-9]+)` | unwrap timeouts [$__auto]))"
+          "legendFormat": "Acquire Timeouts [{{service_instance_id}}] [{{xrpl_network_type}}]",
+          "expr": "sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `Acquire \\S+ timeouts:(?P<timeouts>[0-9]+)` | unwrap timeouts [$__auto]))"
         }
       ],
       "id": 27,
@@ -1924,8 +1924,8 @@
             "uid": "${DS_LOKI}"
           },
           "refId": "A",
-          "legendFormat": "Handshake [{{service_instance_id}}]",
-          "expr": "sum by (service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `PeerFinder` | severity =~ \"$severity\" |~ `Logic handshake` [$__auto]))"
+          "legendFormat": "Handshake [{{service_instance_id}}] [{{xrpl_network_type}}]",
+          "expr": "sum by (service_instance_id, xrpl_network_type) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `PeerFinder` | severity =~ \"$severity\" |~ `Logic handshake` [$__auto]))"
         },
         {
           "datasource": {
@@ -1933,8 +1933,8 @@
             "uid": "${DS_LOKI}"
           },
           "refId": "B",
-          "legendFormat": "Accept [{{service_instance_id}}]",
-          "expr": "sum by (service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `PeerFinder` | severity =~ \"$severity\" |~ `Logic accept` [$__auto]))"
+          "legendFormat": "Accept [{{service_instance_id}}] [{{xrpl_network_type}}]",
+          "expr": "sum by (service_instance_id, xrpl_network_type) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `PeerFinder` | severity =~ \"$severity\" |~ `Logic accept` [$__auto]))"
         }
       ],
       "id": 30,
@@ -2124,8 +2124,8 @@
             "uid": "${DS_LOKI}"
           },
           "refId": "A",
-          "legendFormat": "Quorum [{{service_instance_id}}]",
-          "expr": "max by (service_instance_id) (max_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ValidatorList` | severity =~ \"$severity\" | regexp `Using quorum of (?P<quorum>[0-9]+) for new set of (?P<total>[0-9]+) trusted` | unwrap quorum [$__auto]))"
+          "legendFormat": "Quorum [{{service_instance_id}}] [{{xrpl_network_type}}]",
+          "expr": "max by (service_instance_id, xrpl_network_type) (max_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ValidatorList` | severity =~ \"$severity\" | regexp `Using quorum of (?P<quorum>[0-9]+) for new set of (?P<total>[0-9]+) trusted` | unwrap quorum [$__auto]))"
         },
         {
           "datasource": {
@@ -2133,8 +2133,8 @@
             "uid": "${DS_LOKI}"
           },
           "refId": "B",
-          "legendFormat": "Trusted Validators [{{service_instance_id}}]",
-          "expr": "max by (service_instance_id) (max_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ValidatorList` | severity =~ \"$severity\" | regexp `Using quorum of (?P<quorum>[0-9]+) for new set of (?P<total>[0-9]+) trusted` | unwrap total [$__auto]))"
+          "legendFormat": "Trusted Validators [{{service_instance_id}}] [{{xrpl_network_type}}]",
+          "expr": "max by (service_instance_id, xrpl_network_type) (max_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ValidatorList` | severity =~ \"$severity\" | regexp `Using quorum of (?P<quorum>[0-9]+) for new set of (?P<total>[0-9]+) trusted` | unwrap total [$__auto]))"
         }
       ],
       "id": 34,
diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json
index 41597900e1..e73637bf26 100644
--- a/docker/telemetry/grafana/dashboards/node-health.json
+++ b/docker/telemetry/grafana/dashboards/node-health.json
@@ -1177,7 +1177,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"server_state\"}",
-          "legendFormat": "State [{{service_instance_id}}]",
+          "legendFormat": "State [{{service_instance_id}}] [{{xrpl_network_type}}]",
           "refId": "A"
         }
       ],
@@ -3177,7 +3177,7 @@
           },
           "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"server_state\"}",
           "instant": false,
-          "legendFormat": "{{service_instance_id}} [{{xrpl_branch}}]",
+          "legendFormat": "{{service_instance_id}} [{{xrpl_network_type}}] [{{xrpl_branch}}]",
           "queryType": "range",
           "range": true,
           "refId": "A"
@@ -3241,7 +3241,7 @@
           },
           "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"uptime\"}",
           "instant": true,
-          "legendFormat": "{{service_instance_id}}",
+          "legendFormat": "{{service_instance_id}} [{{xrpl_network_type}}]",
           "refId": "A"
         }
       ],
@@ -3311,7 +3311,7 @@
           },
           "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"peers\"}",
           "instant": true,
-          "legendFormat": "{{service_instance_id}}",
+          "legendFormat": "{{service_instance_id}} [{{xrpl_network_type}}]",
           "refId": "A"
         }
       ],
@@ -3385,7 +3385,7 @@
         "type": "prometheus",
         "uid": "${DS_PROMETHEUS}"
       },
-      "description": "###### What this is:\n*The spread in validated ledger sequence across all selected nodes.*\n\n###### How it's computed:\n*Highest validated ledger sequence minus the lowest, across the selected nodes.*\n\n###### Reading it:\n*0 means every node agrees on the same validated ledger; larger means they diverge.*\n\n###### Healthy range:\n*0 to 1 ledger in steady state.*\n\n###### Watch for:\n*A sustained spread above a few ledgers means some nodes are lagging or the fleet is diverging.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* \u2014 a ledger confirmed final by the trusted validator quorum; its contents never change.\n\n###### Computation boundary:\n*Result: Across the selected nodes \u2014 the query aggregates instances into one series.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)",
+      "description": "###### What this is:\n*The spread in validated ledger sequence across all selected nodes.*\n\n###### How it's computed:\n*Highest validated ledger sequence minus the lowest, among the selected nodes on the same network.*\n\n###### Reading it:\n*0 means every node agrees on the same validated ledger; larger means they diverge.*\n\n###### Healthy range:\n*0 to 1 ledger in steady state.*\n\n###### Watch for:\n*A sustained spread above a few ledgers means some nodes are lagging or the fleet is diverging.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* \u2014 a ledger confirmed final by the trusted validator quorum; its contents never change.\n\n###### Computation boundary:\n*Result: Per network \u2014 the query aggregates the selected nodes into one series for each `xrpl_network_type`, so mainnet and devnet sequences are never subtracted from each other.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -3467,15 +3467,15 @@
             "type": "prometheus",
             "uid": "${DS_PROMETHEUS}"
           },
-          "expr": "max(server_info{metric=\"validated_ledger_seq\",service_name=~\"$service_name\",service_instance_id=~\"$node\",xrpl_network_type=~\"$xrpl_network_type\",xrpl_branch=~\"$xrpl_branch\",xrpl_node_role=~\"$xrpl_node_role\",deployment_environment=~\"$deployment_environment\",xrpl_work_item=~\"$xrpl_work_item\"}) - min(server_info{metric=\"validated_ledger_seq\",service_name=~\"$service_name\",service_instance_id=~\"$node\",xrpl_network_type=~\"$xrpl_network_type\",xrpl_branch=~\"$xrpl_branch\",xrpl_node_role=~\"$xrpl_node_role\",deployment_environment=~\"$deployment_environment\",xrpl_work_item=~\"$xrpl_work_item\"})",
+          "expr": "max by (xrpl_network_type) (server_info{metric=\"validated_ledger_seq\",service_name=~\"$service_name\",service_instance_id=~\"$node\",xrpl_network_type=~\"$xrpl_network_type\",xrpl_branch=~\"$xrpl_branch\",xrpl_node_role=~\"$xrpl_node_role\",deployment_environment=~\"$deployment_environment\",xrpl_work_item=~\"$xrpl_work_item\"}) - min by (xrpl_network_type) (server_info{metric=\"validated_ledger_seq\",service_name=~\"$service_name\",service_instance_id=~\"$node\",xrpl_network_type=~\"$xrpl_network_type\",xrpl_branch=~\"$xrpl_branch\",xrpl_node_role=~\"$xrpl_node_role\",deployment_environment=~\"$deployment_environment\",xrpl_work_item=~\"$xrpl_work_item\"})",
           "instant": true,
-          "legendFormat": "Spread",
+          "legendFormat": "Spread [{{xrpl_network_type}}]",
           "queryType": "instant",
           "range": false,
           "refId": "A"
         }
       ],
-      "title": "Validated Ledger Seq \u2014 Convergence (Max \u2212 Min)",
+      "title": "Validated Ledger Seq \u2014 Convergence (Max \u2212 Min, per network)",
       "type": "stat"
     },
     {
@@ -3483,7 +3483,7 @@
         "type": "prometheus",
         "uid": "${DS_PROMETHEUS}"
       },
-      "description": "###### What this is:\n*How far each node's validated ledger lags behind the network tip, in ledgers.*\n\n###### How it's computed:\n*Highest validated ledger sequence across the selected nodes, minus each node's own sequence.*\n\n###### Reading it:\n*0 means the node is at the tip; larger values mean it trails further behind.*\n\n###### Healthy range:\n*0 to 1 ledger on a synced node.*\n\n###### Watch for:\n*A node stuck at a growing value is falling behind and not keeping up with consensus.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* \u2014 a ledger confirmed final by the trusted validator quorum; its contents never change.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Across the selected nodes \u2014 the query aggregates instances into one series.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)",
+      "description": "###### What this is:\n*How far each node's validated ledger lags behind the network tip, in ledgers.*\n\n###### How it's computed:\n*Highest validated ledger sequence within the node's own network, minus each node's own sequence.*\n\n###### Reading it:\n*0 means the node is at the tip; larger values mean it trails further behind.*\n\n###### Healthy range:\n*0 to 1 ledger on a synced node.*\n\n###### Watch for:\n*A node stuck at a growing value is falling behind and not keeping up with consensus.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* \u2014 a ledger confirmed final by the trusted validator quorum; its contents never change.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each node is compared against the highest sequence on its own `xrpl_network_type`, so mainnet and devnet sequences are never subtracted from each other.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -3533,15 +3533,15 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "editorMode": "code",
-          "expr": "sort_desc(scalar(max(server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"validated_ledger_seq\"})) - server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"validated_ledger_seq\"})",
+          "expr": "sort_desc(max by (xrpl_network_type) (server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"validated_ledger_seq\"}) - on(xrpl_network_type) group_right() server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"validated_ledger_seq\"})",
           "instant": true,
-          "legendFormat": "{{service_instance_id}}",
+          "legendFormat": "{{service_instance_id}} [{{xrpl_network_type}}]",
           "queryType": "instant",
           "range": false,
           "refId": "A"
         }
       ],
-      "title": "Validated Ledger Seq \u2014 Current (Stat)",
+      "title": "Validated Ledger Seq \u2014 Lag Behind Network Tip",
       "type": "stat"
     },
     {
@@ -4218,7 +4218,7 @@
           "expr": "complete_ledgers{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}",
           "format": "table",
           "instant": true,
-          "legendFormat": "{{bound}} [range {{index}}] [{{service_instance_id}}]",
+          "legendFormat": "{{bound}} [range {{index}}] [{{service_instance_id}}] [{{xrpl_network_type}}]",
           "refId": "A"
         }
       ],
diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json
index ecb9450172..f8aa73bad7 100644
--- a/docker/telemetry/grafana/dashboards/transaction-overview.json
+++ b/docker/telemetry/grafana/dashboards/transaction-overview.json
@@ -474,9 +474,9 @@
             "type": "prometheus",
             "uid": "${DS_PROMETHEUS}"
           },
-          "expr": "sum by (service_instance_id) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.accept_tx\", txq_status=\"applied\"}[$__rate_interval]))\n/\nsum by (service_instance_id) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.accept_tx\", txq_status=~\"applied|failed\"}[$__rate_interval]))",
+          "expr": "sum by (service_instance_id, xrpl_network_type) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.accept_tx\", txq_status=\"applied\"}[$__rate_interval]))\n/\nsum by (service_instance_id, xrpl_network_type) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.accept_tx\", txq_status=~\"applied|failed\"}[$__rate_interval]))",
           "interval": "15s",
-          "legendFormat": "{{service_instance_id}}"
+          "legendFormat": "{{service_instance_id}} [{{xrpl_network_type}}]"
         }
       ],
       "fieldConfig": {
diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md
index 49b2827a45..d1a8c25152 100644
--- a/docs/telemetry-runbook.md
+++ b/docs/telemetry-runbook.md
@@ -1929,32 +1929,33 @@ Requires `trace_peer=1` in the `[telemetry]` config section.
 
 ### Node Health -- System Metrics (`node-health`)
 
-| Panel                                  | Type       | PromQL                                                     | Labels Used      |
-| -------------------------------------- | ---------- | ---------------------------------------------------------- | ---------------- |
-| Validated Ledger Age                   | stat       | `ledgermaster_validated_ledger_age`                        | —                |
-| Published Ledger Age                   | stat       | `ledgermaster_published_ledger_age`                        | —                |
-| Operating Mode (Time Share)            | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))` | —                |
-| Operating Mode Transitions             | timeseries | `state_accounting_*_transitions`                           | —                |
-| I/O Latency                            | timeseries | `histogram_quantile(0.95, ios_latency_bucket)`             | —                |
-| Job Queue Depth                        | timeseries | `jobq_job_count`                                           | —                |
-| Ledger Fetch Rate                      | stat       | `rate(ledger_fetches[5m])`                                 | —                |
-| Ledger History Mismatches              | stat       | `rate(ledger_history_mismatch[5m])`                        | —                |
-| Key Jobs Execution Time                | timeseries | `acceptledger{quantile="$quantile"}` (+ 10 more key jobs)  | `quantile`       |
-| Key Jobs Dequeue Wait Time             | timeseries | `acceptledger_q{quantile="$quantile"}` (+ 10 more)         | `quantile`       |
-| FullBelowCache Size                    | timeseries | `node_family_full_below_cache_size`                        | —                |
-| FullBelowCache Hit Rate                | gauge      | `node_family_full_below_cache_hit_rate`                    | —                |
-| Ledger Publish Gap                     | stat       | `Published_Ledger_Age - Validated_Ledger_Age`              | —                |
-| State Duration Rate (Full vs Tracking) | timeseries | `rate(state_accounting_full_duration[5m]) / 1000000`       | —                |
-| All Jobs Execution Time (Detail)       | timeseries | `{__name__=~"<all_jobs>", quantile="$quantile"}`           | `quantile`       |
-| All Jobs Dequeue Wait (Detail)         | timeseries | `{__name__=~"<all_jobs>_q", quantile="$quantile"}`         | `quantile`       |
-| Server State                           | stat       | `server_info{metric="server_state"}`                       | `metric`         |
-| Uptime                                 | stat       | `server_info{metric="uptime"}`                             | `metric`         |
-| Peer Count                             | stat       | `server_info{metric="peers"}`                              | `metric`         |
-| Validated Ledger Seq                   | stat       | `server_info{metric="validated_ledger_seq"}`               | `metric`         |
-| Build Version                          | stat       | `build_info`                                               | `version`        |
-| Complete Ledger Ranges                 | table      | `complete_ledgers`                                         | `bound`, `index` |
-| Database Sizes                         | timeseries | `db_metrics{metric=~"db_kb_.*"}`                           | `metric`         |
-| Historical Fetch Rate                  | stat       | `db_metrics{metric="historical_perminute"}`                | `metric`         |
+| Panel                                                        | Type       | PromQL                                                                                                                                                     | Labels Used      |
+| ------------------------------------------------------------ | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- |
+| Validated Ledger Age                                         | stat       | `ledgermaster_validated_ledger_age`                                                                                                                        | —                |
+| Published Ledger Age                                         | stat       | `ledgermaster_published_ledger_age`                                                                                                                        | —                |
+| Operating Mode (Time Share)                                  | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))`                                                                                                 | —                |
+| Operating Mode Transitions                                   | timeseries | `state_accounting_*_transitions`                                                                                                                           | —                |
+| I/O Latency                                                  | timeseries | `histogram_quantile(0.95, ios_latency_bucket)`                                                                                                             | —                |
+| Job Queue Depth                                              | timeseries | `jobq_job_count`                                                                                                                                           | —                |
+| Ledger Fetch Rate                                            | stat       | `rate(ledger_fetches[5m])`                                                                                                                                 | —                |
+| Ledger History Mismatches                                    | stat       | `rate(ledger_history_mismatch[5m])`                                                                                                                        | —                |
+| Key Jobs Execution Time                                      | timeseries | `acceptledger{quantile="$quantile"}` (+ 10 more key jobs)                                                                                                  | `quantile`       |
+| Key Jobs Dequeue Wait Time                                   | timeseries | `acceptledger_q{quantile="$quantile"}` (+ 10 more)                                                                                                         | `quantile`       |
+| FullBelowCache Size                                          | timeseries | `node_family_full_below_cache_size`                                                                                                                        | —                |
+| FullBelowCache Hit Rate                                      | gauge      | `node_family_full_below_cache_hit_rate`                                                                                                                    | —                |
+| Ledger Publish Gap                                           | stat       | `Published_Ledger_Age - Validated_Ledger_Age`                                                                                                              | —                |
+| State Duration Rate (Full vs Tracking)                       | timeseries | `rate(state_accounting_full_duration[5m]) / 1000000`                                                                                                       | —                |
+| All Jobs Execution Time (Detail)                             | timeseries | `{__name__=~"<all_jobs>", quantile="$quantile"}`                                                                                                           | `quantile`       |
+| All Jobs Dequeue Wait (Detail)                               | timeseries | `{__name__=~"<all_jobs>_q", quantile="$quantile"}`                                                                                                         | `quantile`       |
+| Server State                                                 | stat       | `server_info{metric="server_state"}`                                                                                                                       | `metric`         |
+| Uptime                                                       | stat       | `server_info{metric="uptime"}`                                                                                                                             | `metric`         |
+| Peer Count                                                   | stat       | `server_info{metric="peers"}`                                                                                                                              | `metric`         |
+| Validated Ledger Seq -- Lag Behind Network Tip               | stat       | `max by (xrpl_network_type) (server_info{metric="validated_ledger_seq"}) - on(xrpl_network_type) group_right() server_info{metric="validated_ledger_seq"}` | `metric`         |
+| Validated Ledger Seq -- Convergence (Max - Min, per network) | stat       | `max by (xrpl_network_type) (server_info{metric="validated_ledger_seq"}) - min by (xrpl_network_type) (server_info{metric="validated_ledger_seq"})`        | `metric`         |
+| Build Version                                                | stat       | `build_info`                                                                                                                                               | `version`        |
+| Complete Ledger Ranges                                       | table      | `complete_ledgers`                                                                                                                                         | `bound`, `index` |
+| Database Sizes                                               | timeseries | `db_metrics{metric=~"db_kb_.*"}`                                                                                                                           | `metric`         |
+| Historical Fetch Rate                                        | stat       | `db_metrics{metric="historical_perminute"}`                                                                                                                | `metric`         |
 
 ### Network Traffic -- System Metrics (`network-traffic`)
 

From ef05e1b2a172f3d17e4a302c17a10a587d2a1225 Mon Sep 17 00:00:00 2001
From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
Date: Thu, 6 Aug 2026 19:01:00 +0100
Subject: [PATCH 7/9] fix(telemetry): filter zero denominators instead of
 clamping them

Panels 21 (NuDB Read Latency) and 23 (NuDB Read Found Ratio) on the
Ledger Data & Sync dashboard guarded their divisor with
clamp_min(<denominator>, 1). clamp_min raises the value, not just the
zero case, so any node reading fewer than 1 block per second was
divided by a fabricated 1 instead of by its real read count.

Replace the clamp with the filter (<denominator> > 0). A comparison
without the bool modifier drops the sample rather than rewriting it, so
these panels now show no data instead of a wrong number.

Measured over 7 days: five nodes fall below 1 read/s. On validator-0 the
clamp reported 2.726 us/read against a true 5.493, and on nonUNLmalloc-tc
it reported 0 us/read, which cannot occur. The error is largest exactly
when panel 21 is used as the bottleneck discriminator during a stall,
because that is when the read rate collapses toward zero.

Matches the existing idiom on the same nodestore_state metric family in
the NodeStore Write vs Read Latency panel.
---
 docker/telemetry/grafana/dashboards/ledger-data-sync.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json
index 2f01f834d9..5f02984c72 100644
--- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json
+++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json
@@ -1248,7 +1248,7 @@
             "type": "prometheus",
             "uid": "${DS_PROMETHEUS}"
           },
-          "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_duration_us\"}[$__rate_interval])) / clamp_min(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])), 1), \"series\", \"NuDB us/read\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
+          "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_duration_us\"}[$__rate_interval])) / (sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])) > 0), \"series\", \"NuDB us/read\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
         }
       ],
       "fieldConfig": {
@@ -1332,7 +1332,7 @@
             "type": "prometheus",
             "uid": "${DS_PROMETHEUS}"
           },
-          "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval])) / clamp_min(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])), 1), \"series\", \"NuDB Found Ratio\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
+          "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval])) / (sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])) > 0), \"series\", \"NuDB Found Ratio\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
         }
       ],
       "fieldConfig": {

From 84ef8cbf3355a752ced56916d377ad4e664343d4 Mon Sep 17 00:00:00 2001
From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
Date: Thu, 6 Aug 2026 19:01:15 +0100
Subject: [PATCH 8/9] fix(telemetry): drop unmeasurable Queue Bypass Ratio
 panel

The Transaction Overview panel "Queue Bypass Ratio (Direct Apply vs
Enqueue)" reported a confident 0.50 on every node while the true bypass
rate was zero. The two spans it divided are not disjoint alternatives:
txq.apply_direct is a child of txq.enqueue. TxQ.cpp creates the
apply_direct span as the first statement of tryDirectApply(), ahead of
the account, sequence and fee-level guards, and tryDirectApply() is
called from inside the live enqueue scope. The span therefore counts
attempts, so the denominator direct + enqueue counts each transaction
twice and pins the ratio to one half algebraically.

Measured on a four-node fleet: 6082443 direct against 6082877 enqueue
over the same population, panel output 0.5000170 on three nodes and
0.5000000 on the fourth. Grouping txq.enqueue by txq_status over seven
days returns only "rejected" -- no transaction has ever taken the
direct-apply path.

Remove the panel rather than repoint it. A correct expression using
txq_status as the disjoint discriminator would render permanently
empty on this fleet, which reads no better than a wrong number.

Widen the band partner "TxQ Enqueue Rate by Transaction Type" from 12
to 24 columns so the y=48 band still fills the grid. Every band in all
ten dashboards sums to 24 columns; leaving a half-width hole would be
the only exception. Panel order and every other panel's position,
width and height are unchanged.

The runbook already listed txq.apply_direct as available but not
paneled, so that row becomes accurate. Rows describing the span itself
are untouched -- the span and its metric are unchanged.
---
 .../dashboards/transaction-overview.json      | 42 +------------------
 1 file changed, 1 insertion(+), 41 deletions(-)

diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json
index 4efec5de79..94cc8be828 100644
--- a/docker/telemetry/grafana/dashboards/transaction-overview.json
+++ b/docker/telemetry/grafana/dashboards/transaction-overview.json
@@ -555,7 +555,7 @@
       "type": "timeseries",
       "gridPos": {
         "h": 8,
-        "w": 12,
+        "w": 24,
         "x": 0,
         "y": 48
       },
@@ -589,46 +589,6 @@
         "overrides": []
       }
     },
-    {
-      "title": "Queue Bypass Ratio (Direct Apply vs Enqueue)",
-      "description": "**What:** Fraction of transactions that applied straight to the open ledger versus those that had to be queued.\n**How it's computed:** Ratio of direct applies to direct-plus-queued over a 5-minute window, per node.\n**Reading it:** A high fraction means the network is keeping up without escalation.\n**Healthy range:** Near 1.0 in normal conditions.\n**Watch for:** A falling fraction is the cleanest single signal the network has entered sustained fee escalation.\n**Source:** src/xrpld/app/misc/detail/TxQ.cpp:TxQ::tryDirectApply, TxQ::apply",
-      "type": "timeseries",
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 48
-      },
-      "options": {
-        "tooltip": {
-          "mode": "multi",
-          "sort": "desc",
-          "maxHeight": 600
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus"
-          },
-          "expr": "sum by (service_instance_id) (rate(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"txq.apply_direct\"}[5m])) / clamp_min(sum by (service_instance_id) (rate(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"txq.apply_direct\"}[5m])) + sum by (service_instance_id) (rate(traces_span_metrics_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"txq.enqueue\"}[5m])), 1)",
-          "legendFormat": "Direct-Apply Fraction [{{service_instance_id}}]"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "unit": "percentunit",
-          "custom": {
-            "axisLabel": "Bypass Fraction",
-            "spanNulls": 1800000,
-            "insertNulls": false,
-            "showPoints": "auto",
-            "pointSize": 3
-          }
-        },
-        "overrides": []
-      }
-    },
     {
       "title": "Queue Accept (Drain) Duration per Ledger",
       "description": "**What:** Time spent draining queued transactions into a newly closed ledger.\n**How it's computed:** 95th and 50th percentile of per-ledger drain durations over a 5-minute window, per node.\n**Reading it:** Rises as the queue holds more transactions to process at close.\n**Healthy range:** A few milliseconds when the queue is light.\n**Watch for:** Rising drain time signals queue pressure at ledger close.\n**Source:** src/xrpld/app/misc/detail/TxQ.cpp:TxQ::accept",

From fd006ec3ac4b1b7ebe18bae8e79b7d2d6aff9dd7 Mon Sep 17 00:00:00 2001
From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
Date: Thu, 6 Aug 2026 18:44:06 +0100
Subject: [PATCH 9/9] fix(telemetry): correct dashboard query guards,
 thresholds and stale panel docs

Four defects found by a query-correctness audit of all 16 dashboards, all
scoped to panels this branch owns.

job-queue "Current Job Latency (p99 Gauge)": the histogram by-clause dropped
service_instance_id, collapsing every node into one fleet-wide p99 and hiding
a slow node. Measured: gauge read 1325us while the worst node was 1868us. The
panel's displayName already referenced service_instance_id, so it was also
rendering an empty label. Now groups by service_instance_id and xrpl_work_item,
matching sibling panels 5, 6 and 7.

ledger-data-sync "NodeStore Read Latency (Bottleneck Discriminator)": replaced
clamp_min(<denominator>, 1) with (<denominator> > 0). clamp_min clamps the
value, not just the zero case, so any node reading below 1/s got a fabricated
denominator. Demonstrated with a zero-rate denominator: clamp_min invents
40.2/40.3/11.2/9.2 where the > 0 guard correctly returns no data. This panel is
the bottleneck discriminator, read during a stall, which is exactly when the
read rate collapses and the clamp is most wrong. Panels 21 and 23 carry the
same defect but originate on phase-7 and are fixed there.

node-health thresholds: percentunit fields are compared against the raw value,
so a step of 80 needed 8000% and could never fire. Rescaled panels 74, 81 and
85 to 0.8. Panel 81 is a found-ratio where high is healthy, so its bands were
also inverted. Note these three panels use palette-classic with thresholdsStyle
off, so the steps are currently dormant rather than visibly wrong.

node-health panels 81 and 85 descriptions: both described the multi-series
panels they were split from. Panel 81 carried a byte-identical copy of panel
80's text, promising three plotted rate lines where it draws a single bounded
ratio; panel 85's text described read-thread gauges absent from its expression.
Rewritten to match the actual queries.
---
 docker/telemetry/grafana/dashboards/job-queue.json |  4 ++--
 .../grafana/dashboards/ledger-data-sync.json       |  2 +-
 .../telemetry/grafana/dashboards/node-health.json  | 14 +++++++-------
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/docker/telemetry/grafana/dashboards/job-queue.json b/docker/telemetry/grafana/dashboards/job-queue.json
index f9b5376ece..c7e745e7e9 100644
--- a/docker/telemetry/grafana/dashboards/job-queue.json
+++ b/docker/telemetry/grafana/dashboards/job-queue.json
@@ -34,13 +34,13 @@
           "datasource": {
             "type": "prometheus"
           },
-          "expr": "label_replace(histogram_quantile(0.99, sum by (le, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Wait\", \"\", \"\")"
+          "expr": "label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Wait\", \"\", \"\")"
         },
         {
           "datasource": {
             "type": "prometheus"
           },
-          "expr": "label_replace(histogram_quantile(0.99, sum by (le, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Exec\", \"\", \"\")"
+          "expr": "label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Exec\", \"\", \"\")"
         }
       ],
       "fieldConfig": {
diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json
index 640b5b464c..8a7d208999 100644
--- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json
+++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json
@@ -1685,7 +1685,7 @@
             "type": "prometheus",
             "uid": "${DS_PROMETHEUS}"
           },
-          "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_duration_us\"}[$__rate_interval])) / clamp_min(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])), 1), \"series\", \"Read Mean (Windowed)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
+          "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_duration_us\"}[$__rate_interval])) / (sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])) > 0), \"series\", \"Read Mean (Windowed)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
         }
       ],
       "fieldConfig": {
diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json
index e73637bf26..518f7bd5ca 100644
--- a/docker/telemetry/grafana/dashboards/node-health.json
+++ b/docker/telemetry/grafana/dashboards/node-health.json
@@ -632,7 +632,7 @@
               },
               {
                 "color": "red",
-                "value": 80
+                "value": 0.8
               }
             ]
           },
@@ -1320,7 +1320,7 @@
         "type": "prometheus",
         "uid": "${DS_PROMETHEUS}"
       },
-      "description": "###### What this is:\n*Object-store read, found and write operation rates. The found series is `node_reads_hit`, which counts fetches that returned an object whatever served them, so it is not a cache-hit count.*\n\n###### How it's computed:\n*Per-second rates of the read, found and write counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope reflects store activity.*\n\n###### Healthy range:\n*Reads and writes rising smoothly, with found tracking almost all reads on a node that has the data.*\n\n###### Watch for:\n*A sudden surge in reads or writes signals heavy back-end I/O, from sync, replay, or query load.*\n\n###### Keywords:\n- **NodeStore** *(per node)* \u2014 the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#nodestore)",
+      "description": "###### What this is:\n*Fraction of object-store reads that returned an object. `node_reads_hit` counts fetches that found the object whatever served them, so this is not a cache-hit rate.*\n\n###### How it's computed:\n*Per-second rate of `node_reads_hit` divided by the per-second rate of `node_reads_total`, as a single ratio per node.*\n\n###### Reading it:\n*A value near 1.0 means almost every read finds its object; dips mean reads are missing.*\n\n###### Healthy range:\n*Close to 1.0 on a node that holds the data it is being asked for.*\n\n###### Watch for:\n*A sustained drop means the node is repeatedly asked for objects it does not have, which usually accompanies backfill or a gap in history.*\n\n###### Keywords:\n- **NodeStore** *(per node)* \u2014 the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#nodestore)",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -1365,12 +1365,12 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
+                "color": "red",
                 "value": 0
               },
               {
-                "color": "red",
-                "value": 80
+                "color": "green",
+                "value": 0.8
               }
             ]
           },
@@ -1768,7 +1768,7 @@
         "type": "prometheus",
         "uid": "${DS_PROMETHEUS}"
       },
-      "description": "###### What this is:\n*Read-thread utilization, bundled read count, and cumulative read time for the object store.*\n\n###### How it's computed:\n*Current values of the running/total read-thread gauges, read-bundle gauge, and cumulative read-duration counter, plotted as lines.*\n\n###### Reading it:\n*Running threads well below total means spare capacity; a rising duration line reflects time spent in read I/O.*\n\n###### Healthy range:\n*Running threads below the total count most of the time.*\n\n###### Watch for:\n*Running threads pinned at the total for long periods means read I/O is saturated.*\n\n###### Keywords:\n- **Read threads / read queue / write load** *(per node)* \u2014 NodeStore back-end I/O internals \u2014 worker threads reading, their queue depth, and write pressure.\n- **NodeStore** *(per node)* \u2014 the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#read-threads-read-queue-write-load)",
+      "description": "###### What this is:\n*Share of wall-clock time the object store spent inside read I/O.*\n\n###### How it's computed:\n*Per-second rate of the cumulative `node_reads_duration_us` counter, converted from microseconds to seconds, as a single ratio per node.*\n\n###### Reading it:\n*1.0 means the store spent a full second per second in reads; well below 1.0 means spare read capacity.*\n\n###### Healthy range:\n*Below roughly 0.8 in steady state.*\n\n###### Watch for:\n*Sustained values at or above 1.0 mean read I/O is saturated and reads are queueing.*\n\n###### Keywords:\n- **Read threads / read queue / write load** *(per node)* \u2014 NodeStore back-end I/O internals \u2014 worker threads reading, their queue depth, and write pressure.\n- **NodeStore** *(per node)* \u2014 the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#read-threads-read-queue-write-load)",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -1818,7 +1818,7 @@
               },
               {
                 "color": "red",
-                "value": 80
+                "value": 0.8
               }
             ]
           },