Merge branch 'pratik/otel-phase8-log-correlation' into pratik/otel-phase9-metric-gap-fill

# Conflicts: # docker/telemetry/grafana/dashboards/consensus-health.json # docker/telemetry/grafana/dashboards/ledger-operations.json # docker/telemetry/grafana/dashboards/node-health.json # docker/telemetry/grafana/dashboards/peer-network.json # docker/telemetry/grafana/dashboards/rpc-pathfinding.json # docker/telemetry/grafana/dashboards/rpc-performance.json # docker/telemetry/grafana/dashboards/transaction-overview.json # docs/telemetry-runbook.md
2026-08-21 22:30:57 +00:00 · 2026-07-30 19:15:15 +01:00
parent 370130a9f9 09c9dc4ec8
commit b31183cf78
11 changed files with 182 additions and 127 deletions
--- a/docker/telemetry/grafana/dashboards/consensus-health.json
+++ b/docker/telemetry/grafana/dashboards/consensus-health.json
@@ -36,7 +36,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops"
+          "unit": "suffix: validations/s"
        },
        "overrides": []
      },
@@ -70,9 +70,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: closes/s",
          "custom": {
-            "axisLabel": "Events / Sec",
+            "axisLabel": "Closes / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -111,7 +111,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: proposals/s",
          "custom": {
            "axisLabel": "Proposals / Sec",
            "spanNulls": 1800000,
@@ -158,7 +158,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: events/s",
          "custom": {
            "axisLabel": "Events / Sec",
            "spanNulls": 1800000,
@@ -205,7 +205,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: events/s",
          "custom": {
            "axisLabel": "Events / Sec",
            "spanNulls": 1800000,
@@ -524,7 +524,14 @@
        },
        "overrides": []
      },
-      "id": 12
+      "id": 12,
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 600
+        }
+      }
    },
    {
      "title": "Consensus Accept Duration Heatmap",
@@ -539,7 +546,8 @@
      "options": {
        "tooltip": {
          "mode": "multi",
-          "sort": "desc"
+          "sort": "desc",
+          "maxHeight": 600
        },
        "yAxis": {
          "axisLabel": "Duration (ms)",
@@ -584,7 +592,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: rounds/s",
          "custom": {
            "axisLabel": "Rounds / Sec",
            "spanNulls": 1800000,
@@ -595,7 +603,14 @@
        },
        "overrides": []
      },
-      "id": 14
+      "id": 14,
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 600
+        }
+      }
    },
    {
      "title": "Close Time Vote Bins & Resolution",
@@ -657,7 +672,8 @@
      "options": {
        "tooltip": {
          "mode": "multi",
-          "sort": "desc"
+          "sort": "desc",
+          "maxHeight": 600
        },
        "legend": {
          "displayMode": "table",
@@ -880,7 +896,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: failures/s",
          "custom": {
            "axisLabel": "Failures / Sec",
            "spanNulls": 1800000,
@@ -927,7 +943,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: checks/s",
          "custom": {
            "axisLabel": "Checks / Sec",
            "spanNulls": 1800000,
@@ -968,9 +984,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: changes/s",
          "custom": {
-            "axisLabel": "Mode Changes / Sec",
+            "axisLabel": "Changes / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
--- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json
+++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json
@@ -496,9 +496,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "cps",
+          "unit": "suffix: messages/s",
          "custom": {
-            "axisLabel": "Messages In",
+            "axisLabel": "Messages / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -543,9 +543,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "cps",
+          "unit": "suffix: messages/s",
          "custom": {
-            "axisLabel": "Messages In",
+            "axisLabel": "Messages / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -584,9 +584,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "cps",
+          "unit": "suffix: messages/s",
          "custom": {
-            "axisLabel": "Messages In",
+            "axisLabel": "Messages / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -637,9 +637,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "cps",
+          "unit": "suffix: messages/s",
          "custom": {
-            "axisLabel": "Messages In",
+            "axisLabel": "Messages / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -985,8 +985,9 @@
        "alignValue": "left",
        "rowHeight": 0.9,
        "tooltip": {
-          "mode": "single",
-          "sort": "none"
+          "maxHeight": 600,
+          "mode": "multi",
+          "sort": "desc"
        },
        "legend": {
          "displayMode": "list",
@@ -1012,7 +1013,7 @@
            "fillOpacity": 80,
            "lineWidth": 0,
            "insertNulls": false,
-            "spanNulls": false
+            "spanNulls": 1800000
          },
          "mappings": [
            {
@@ -1393,16 +1394,49 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "short",
+          "unit": "suffix: reads/s",
          "custom": {
-            "axisLabel": "Reads/s & Depth",
+            "axisLabel": "Reads / Sec & Depth",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
            "pointSize": 3
          }
        },
-        "overrides": []
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "/^Read Queue/"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "short"
+              },
+              {
+                "id": "custom.axisPlacement",
+                "value": "right"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "/^Read Threads Running/"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "short"
+              },
+              {
+                "id": "custom.axisPlacement",
+                "value": "right"
+              }
+            ]
+          }
+        ]
      },
      "id": 24
    },
--- a/docker/telemetry/grafana/dashboards/ledger-operations.json
+++ b/docker/telemetry/grafana/dashboards/ledger-operations.json
@@ -36,7 +36,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops"
+          "unit": "suffix: ledgers/s"
        },
        "overrides": []
      }
@@ -109,7 +109,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops"
+          "unit": "suffix: ledgers/s"
        },
        "overrides": []
      }
@@ -127,7 +127,8 @@
      "options": {
        "tooltip": {
          "mode": "multi",
-          "sort": "desc"
+          "sort": "desc",
+          "maxHeight": 600
        },
        "yAxis": {
          "axisLabel": "Duration (ms)"
@@ -218,9 +219,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: transactions/s",
          "custom": {
-            "axisLabel": "Operations / Sec",
+            "axisLabel": "Transactions / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -258,7 +259,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops"
+          "unit": "suffix: ledgers/s"
        },
        "overrides": []
      }
--- a/docker/telemetry/grafana/dashboards/network-traffic.json
+++ b/docker/telemetry/grafana/dashboards/network-traffic.json
@@ -63,7 +63,7 @@
        "h": 8,
        "w": 12,
        "x": 12,
-        "y": 0
+        "y": 24
      },
      "options": {
        "tooltip": {
@@ -175,9 +175,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "cps",
+          "unit": "suffix: messages/s",
          "custom": {
-            "axisLabel": "Messages",
+            "axisLabel": "Messages / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -227,9 +227,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "cps",
+          "unit": "suffix: messages/s",
          "custom": {
-            "axisLabel": "Messages",
+            "axisLabel": "Messages / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -285,9 +285,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "cps",
+          "unit": "suffix: messages/s",
          "custom": {
-            "axisLabel": "Messages",
+            "axisLabel": "Messages / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -343,9 +343,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "cps",
+          "unit": "suffix: messages/s",
          "custom": {
-            "axisLabel": "Messages",
+            "axisLabel": "Messages / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -363,7 +363,7 @@
        "h": 8,
        "w": 12,
        "x": 12,
-        "y": 24
+        "y": 0
      },
      "options": {
        "tooltip": {
--- a/docker/telemetry/grafana/dashboards/node-health.json
+++ b/docker/telemetry/grafana/dashboards/node-health.json
@@ -419,7 +419,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: fetches/s",
          "custom": {}
        },
        "overrides": []
@@ -453,7 +453,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: mismatches/s",
          "thresholds": {
            "steps": [
              {
@@ -1886,9 +1886,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "µs",
+          "unit": "\u00b5s",
          "custom": {
-            "axisLabel": "Duration (µs)",
+            "axisLabel": "Duration (\u00b5s)",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -1986,9 +1986,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "µs",
+          "unit": "\u00b5s",
          "custom": {
-            "axisLabel": "Wait Time (µs)",
+            "axisLabel": "Wait Time (\u00b5s)",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -2140,7 +2140,7 @@
    },
    {
      "title": "State Duration Rate (All States)",
-      "description": "###### What this is:\n*Which operating mode the node is accumulating time in right now, one line per state, normalized to seconds per second.*\n\n###### How it's computed:\n*Per-second rate of each state's duration counter, scaled from microseconds to seconds. The five lines sum to about 1.0 because the node is always in exactly one state.*\n\n###### Reading it:\n*The line sitting near 1.0 is the state the node is currently in; the others sit at 0. A handover between two lines marks a state change, and its width is how long that state lasted.*\n\n###### Healthy range:\n*Full near 1.0 with every other line at 0.*\n\n###### Watch for:\n*Time accumulating in Connected or Syncing means the node is catching up rather than serving; repeated handovers mean it is flapping.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* — the node's sync level: Disconnected, Connected, Syncing, Tracking, Full.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Server states](https://xrpl.org/docs/concepts/networks-and-servers/rippled-server-states)",
+      "description": "###### What this is:\n*Which operating mode the node is accumulating time in right now, one line per state, normalized to seconds per second.*\n\n###### How it's computed:\n*Per-second rate of each state's duration counter, scaled from microseconds to seconds. The five lines sum to about 1.0 because the node is always in exactly one state.*\n\n###### Reading it:\n*The line sitting near 1.0 is the state the node is currently in; the others sit at 0. A handover between two lines marks a state change, and its width is how long that state lasted.*\n\n###### Healthy range:\n*Full near 1.0 with every other line at 0.*\n\n###### Watch for:\n*Time accumulating in Connected or Syncing means the node is catching up rather than serving; repeated handovers mean it is flapping.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Server states](https://xrpl.org/docs/concepts/networks-and-servers/rippled-server-states)",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
@@ -2190,7 +2190,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "short",
+          "unit": "suffix: s/s",
          "custom": {
            "axisLabel": "Rate (s/s)",
            "spanNulls": 1800000,
@@ -2230,9 +2230,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ms",
+          "unit": "\u00b5s",
          "custom": {
-            "axisLabel": "Duration (ms)",
+            "axisLabel": "Duration (\u00b5s)",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -2270,9 +2270,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ms",
+          "unit": "\u00b5s",
          "custom": {
-            "axisLabel": "Wait Time (ms)",
+            "axisLabel": "Wait Time (\u00b5s)",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
--- a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json
+++ b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json
@@ -67,9 +67,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "cps",
+          "unit": "suffix: messages/s",
          "custom": {
-            "axisLabel": "Messages",
+            "axisLabel": "Messages / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -195,9 +195,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "cps",
+          "unit": "suffix: messages/s",
          "custom": {
-            "axisLabel": "Count",
+            "axisLabel": "Messages / Sec & Bytes / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -328,9 +328,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "cps",
+          "unit": "suffix: messages/s",
          "custom": {
-            "axisLabel": "Messages",
+            "axisLabel": "Messages / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
--- a/docker/telemetry/grafana/dashboards/peer-network.json
+++ b/docker/telemetry/grafana/dashboards/peer-network.json
@@ -37,7 +37,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: proposals/s",
          "custom": {
            "axisLabel": "Proposals / Sec",
            "spanNulls": 1800000,
@@ -77,7 +77,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: validations/s",
          "custom": {
            "axisLabel": "Validations / Sec",
            "spanNulls": 1800000,
@@ -111,13 +111,13 @@
          "datasource": {
            "type": "prometheus"
          },
-          "expr": "label_replace(sum by (proposal_trusted, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", proposal_trusted=~\"$proposal_trusted\", span_name=\"peer.proposal.receive\"}[$__rate_interval])), \"series\", \"Trusted = $1\", \"proposal_trusted\", \"(.*)\")"
+          "expr": "label_replace(sum by (proposal_trusted, service_instance_id, xrpl_branch, xrpl_node_role) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", proposal_trusted=~\"$proposal_trusted\", span_name=\"peer.proposal.receive\"}[$__rate_interval])), \"series\", \"Trusted = $1\", \"proposal_trusted\", \"(.*)\")"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops"
+          "unit": "short"
        },
        "overrides": []
      }
@@ -144,13 +144,13 @@
          "datasource": {
            "type": "prometheus"
          },
-          "expr": "label_replace(sum by (validation_trusted, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", validation_trusted=~\"$validation_trusted\", span_name=\"peer.validation.receive\"}[$__rate_interval])), \"series\", \"Trusted = $1\", \"validation_trusted\", \"(.*)\")"
+          "expr": "label_replace(sum by (validation_trusted, service_instance_id, xrpl_branch, xrpl_node_role) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", validation_trusted=~\"$validation_trusted\", span_name=\"peer.validation.receive\"}[$__rate_interval])), \"series\", \"Trusted = $1\", \"validation_trusted\", \"(.*)\")"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops"
+          "unit": "short"
        },
        "overrides": []
      }
--- a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json
+++ b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json
@@ -282,7 +282,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: warnings/s",
          "thresholds": {
            "steps": [
              {
@@ -331,7 +331,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: drops/s",
          "thresholds": {
            "steps": [
              {
@@ -380,7 +380,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: calls/s",
          "custom": {
            "axisLabel": "Calls / Sec",
            "spanNulls": 1800000,
@@ -460,7 +460,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: calls/s",
          "custom": {
            "axisLabel": "Calls / Sec",
            "spanNulls": 1800000,
@@ -546,7 +546,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: operations/s",
          "custom": {
            "axisLabel": "Operations / Sec",
            "spanNulls": 1800000,
--- a/docker/telemetry/grafana/dashboards/rpc-performance.json
+++ b/docker/telemetry/grafana/dashboards/rpc-performance.json
@@ -166,7 +166,8 @@
      "options": {
        "tooltip": {
          "mode": "multi",
-          "sort": "desc"
+          "sort": "desc",
+          "maxHeight": 600
        },
        "yAxis": {
          "axisLabel": "Duration (ms)",
@@ -271,7 +272,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: commands/s",
          "custom": {
            "axisLabel": "Commands / Sec",
            "spanNulls": 1800000,
@@ -346,7 +347,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops"
+          "unit": "suffix: messages/s"
        },
        "overrides": []
      },
@@ -385,7 +386,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: requests/s",
          "custom": {
            "axisLabel": "Requests / Sec",
            "spanNulls": 1800000,
@@ -432,7 +433,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: requests/s",
          "custom": {
            "axisLabel": "Requests / Sec",
            "spanNulls": 1800000,
--- a/docker/telemetry/grafana/dashboards/transaction-overview.json
+++ b/docker/telemetry/grafana/dashboards/transaction-overview.json
@@ -36,7 +36,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: transactions/s",
          "thresholds": {
            "steps": [
              {
@@ -137,9 +137,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: transactions/s",
          "custom": {
-            "axisLabel": "TX / Sec",
+            "axisLabel": "Transactions / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -183,9 +183,9 @@
      ],
      "fieldConfig": {
        "defaults": {
-          "unit": "ops",
+          "unit": "suffix: transactions/s",
          "custom": {
-            "axisLabel": "Failed TX / Sec",
+            "axisLabel": "Transactions / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -224,7 +224,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: transactions/s",
          "custom": {
            "axisLabel": "Transactions / Sec",
            "spanNulls": 1800000,
@@ -271,7 +271,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: transactions/s",
          "custom": {
            "axisLabel": "Transactions / Sec",
            "spanNulls": 1800000,
@@ -418,7 +418,9 @@
        "alignValue": "center",
        "rowHeight": 0.85,
        "tooltip": {
-          "mode": "single"
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 600
        },
        "legend": {
          "displayMode": "list",
@@ -463,7 +465,8 @@
          },
          "custom": {
            "fillOpacity": 80,
-            "lineWidth": 0
+            "lineWidth": 0,
+            "spanNulls": 1800000
          }
        },
        "overrides": []
@@ -549,9 +552,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: failures/s",
          "custom": {
-            "axisLabel": "Failed Spans / Sec",
+            "axisLabel": "Failures / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -677,7 +680,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: enqueues/s",
          "custom": {
            "axisLabel": "Enqueues / Sec",
            "spanNulls": 1800000,
@@ -800,7 +803,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: cleanups/s",
          "custom": {
            "axisLabel": "Cleanups / Sec",
            "spanNulls": 1800000,
--- a/docs/telemetry-runbook.md
+++ b/docs/telemetry-runbook.md
@@ -1046,7 +1046,7 @@ Ten dashboards are pre-provisioned in `docker/telemetry/grafana/dashboards/`:
 | ---------------------------------- | -------------- | -------------------------------------------------------------------------------------------- | ----------------------------------- |
 | Transaction Processing Rate        | timeseries     | `rate(span_calls_total{span_name="tx.process"}[5m])` and `tx.receive`                        | `span_name`                         |
 | Transaction Processing Latency     | timeseries     | `histogram_quantile(0.95 / 0.50, ... {span_name="tx.process"})`                              | —                                   |
-| Transaction Path Distribution      | piechart       | `sum by (local) (rate(span_calls_total{span_name="tx.process"}[5m]))`                        | `local`                             |
+| Transaction Path Distribution      | piechart       | `sum by (local) (increase(span_calls_total{span_name="tx.process"}[5m]))`                    | `local`                             |
 | Transaction Receive vs Suppressed  | timeseries     | `rate(span_calls_total{span_name="tx.receive"}[5m])`                                         | —                                   |
 | TX Processing Duration Heatmap     | heatmap        | `tx.process` histogram buckets                                                               | `le`                                |
 | TX Apply Duration per Ledger       | timeseries     | p95/p50 of `tx.apply`                                                                        | —                                   |
@@ -1086,41 +1086,41 @@ Ten dashboards are pre-provisioned in `docker/telemetry/grafana/dashboards/`:

 Requires `trace_peer=1` in the `[telemetry]` config section.

-| Panel                            | Type       | PromQL                         | Labels Used          |
-| -------------------------------- | ---------- | ------------------------------ | -------------------- |
-| Proposal Receive Rate            | timeseries | `peer.proposal.receive` rate   | —                    |
-| Validation Receive Rate          | timeseries | `peer.validation.receive` rate | —                    |
-| Proposals Trusted vs Untrusted   | piechart   | by `proposal_trusted`          | `proposal_trusted`   |
-| Validations Trusted vs Untrusted | piechart   | by `validation_trusted`        | `validation_trusted` |
+| Panel                            | Type       | PromQL                                                                    | Labels Used          |
+| -------------------------------- | ---------- | ------------------------------------------------------------------------- | -------------------- |
+| Proposal Receive Rate            | timeseries | `peer.proposal.receive` rate                                              | —                    |
+| Validation Receive Rate          | timeseries | `peer.validation.receive` rate                                            | —                    |
+| Proposals Trusted vs Untrusted   | piechart   | `increase()` counts in the selected window, split by `proposal_trusted`   | `proposal_trusted`   |
+| Validations Trusted vs Untrusted | piechart   | `increase()` counts in the selected window, split by `validation_trusted` | `validation_trusted` |

 ### Node Health -- System Metrics (`node-health`)

-| Panel                            | Type       | PromQL                                                     | Labels Used      |
-| -------------------------------- | ---------- | ---------------------------------------------------------- | ---------------- |
-| Validated Ledger Age             | stat       | `ledgermaster_validated_ledger_age`                        | —                |
-| Published Ledger Age             | stat       | `ledgermaster_published_ledger_age`                        | —                |
-| Operating Mode (Time Share)      | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))` | —                |
-| Operating Mode Transitions       | timeseries | `state_accounting_*_transitions`                           | —                |
-| I/O Latency                      | timeseries | `histogram_quantile(0.95, ios_latency_bucket)`             | —                |
-| Job Queue Depth                  | timeseries | `job_count`                                                | —                |
-| Ledger Fetch Rate                | stat       | `rate(ledger_fetches[5m])`                                 | —                |
-| Ledger History Mismatches        | stat       | `rate(ledger_history_mismatch[5m])`                        | —                |
-| Key Jobs Execution Time          | timeseries | `acceptledger{quantile="$quantile"}` (+ 10 more key jobs)  | `quantile`       |
-| Key Jobs Dequeue Wait Time       | timeseries | `acceptledger_q{quantile="$quantile"}` (+ 10 more)         | `quantile`       |
-| FullBelowCache Size              | timeseries | `node_family_full_below_cache_size`                        | —                |
-| FullBelowCache Hit Rate          | gauge      | `node_family_full_below_cache_hit_rate`                    | —                |
-| Ledger Publish Gap               | stat       | `Published_Ledger_Age - Validated_Ledger_Age`              | —                |
-| State Duration Rate (All States) | timeseries | `rate(state_accounting_<mode>_duration[5m]) / 1000000`     | —                |
-| All Jobs Execution Time (Detail) | timeseries | `{__name__=~"<all_jobs>", quantile="$quantile"}`           | `quantile`       |
-| All Jobs Dequeue Wait (Detail)   | timeseries | `{__name__=~"<all_jobs>_q", quantile="$quantile"}`         | `quantile`       |
-| Server State                     | stat       | `server_info{metric="server_state"}`                       | `metric`         |
-| Uptime                           | stat       | `server_info{metric="uptime"}`                             | `metric`         |
-| Peer Count                       | stat       | `server_info{metric="peers"}`                              | `metric`         |
-| Validated Ledger Seq             | stat       | `server_info{metric="validated_ledger_seq"}`               | `metric`         |
-| Build Version                    | stat       | `build_info`                                               | `version`        |
-| Complete Ledger Ranges           | table      | `complete_ledgers`                                         | `bound`, `index` |
-| Database Sizes                   | timeseries | `db_metrics{metric=~"db_kb_.*"}`                           | `metric`         |
-| Historical Fetch Rate            | stat       | `db_metrics{metric="historical_perminute"}`                | `metric`         |
+| Panel                            | Type       | PromQL                                                                            | Labels Used      |
+| -------------------------------- | ---------- | --------------------------------------------------------------------------------- | ---------------- |
+| Validated Ledger Age             | stat       | `ledgermaster_validated_ledger_age`                                               | —                |
+| Published Ledger Age             | stat       | `ledgermaster_published_ledger_age`                                               | —                |
+| Operating Mode (Time Share)      | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))`                        | —                |
+| Operating Mode Transitions       | timeseries | `state_accounting_*_transitions`                                                  | —                |
+| I/O Latency                      | timeseries | `histogram_quantile(0.95, ios_latency_bucket)`                                    | —                |
+| Job Queue Depth                  | timeseries | `job_count`                                                                       | —                |
+| Ledger Fetch Rate                | stat       | `rate(ledger_fetches[5m])`                                                        | —                |
+| Ledger History Mismatches        | stat       | `rate(ledger_history_mismatch[5m])`                                               | —                |
+| Key Jobs Execution Time          | timeseries | `acceptledger{quantile="$quantile"}` (+ 10 more key jobs)                         | `quantile`       |
+| Key Jobs Dequeue Wait Time       | timeseries | `acceptledger_q{quantile="$quantile"}` (+ 10 more)                                | `quantile`       |
+| FullBelowCache Size              | timeseries | `node_family_full_below_cache_size`                                               | —                |
+| FullBelowCache Hit Rate          | gauge      | `node_family_full_below_cache_hit_rate`                                           | —                |
+| Ledger Publish Gap               | stat       | `Published_Ledger_Age - Validated_Ledger_Age`                                     | —                |
+| State Duration Rate (All States) | timeseries | `rate(state_accounting_<mode>_duration[5m]) / 1000000`                            | —                |
+| All Jobs Execution Time (Detail) | timeseries | `histogram_quantile($quantile, rate(job_running_us_bucket[5m])) by job_type` — µs | `quantile`       |
+| All Jobs Dequeue Wait (Detail)   | timeseries | `histogram_quantile($quantile, rate(job_queued_us_bucket[5m])) by job_type` — µs  | `quantile`       |
+| Server State                     | stat       | `server_info{metric="server_state"}`                                              | `metric`         |
+| Uptime                           | stat       | `server_info{metric="uptime"}`                                                    | `metric`         |
+| Peer Count                       | stat       | `server_info{metric="peers"}`                                                     | `metric`         |
+| Validated Ledger Seq             | stat       | `server_info{metric="validated_ledger_seq"}`                                      | `metric`         |
+| Build Version                    | stat       | `build_info`                                                                      | `version`        |
+| Complete Ledger Ranges           | table      | `complete_ledgers`                                                                | `bound`, `index` |
+| Database Sizes                   | timeseries | `db_metrics{metric=~"db_kb_.*"}`                                                  | `metric`         |
+| Historical Fetch Rate            | stat       | `db_metrics{metric="historical_perminute"}`                                       | `metric`         |

 ### Network Traffic -- System Metrics (`network-traffic`)

@@ -1147,8 +1147,8 @@ Requires `trace_peer=1` in the `[telemetry]` config section.
 | RPC Response Time Heatmap | heatmap    | `rpc_time_bucket`                                | —           |
 | Pathfinding Fast Duration | timeseries | `histogram_quantile(0.95, pathfind_fast_bucket)` | —           |
 | Pathfinding Full Duration | timeseries | `histogram_quantile(0.95, pathfind_full_bucket)` | —           |
-| Resource Warnings Rate    | stat       | `rate(warn[5m])`                                 | —           |
-| Resource Drops Rate       | stat       | `rate(drop[5m])`                                 | —           |
+| Resource Warnings Rate    | stat       | `rate(warn_total[$__rate_interval])`             | —           |
+| Resource Drops Rate       | stat       | `rate(drop_total[$__rate_interval])`             | —           |

 ### Span → Metric → Dashboard Summary