Merge branch 'pratik/otel-phase8-log-correlation' into pratik/otel-phase9-metric-gap-fill

# Conflicts: # docker/telemetry/grafana/dashboards/consensus-health.json # docker/telemetry/grafana/dashboards/ledger-operations.json # docker/telemetry/grafana/dashboards/node-health.json # docker/telemetry/grafana/dashboards/peer-network.json # docker/telemetry/grafana/dashboards/rpc-pathfinding.json # docker/telemetry/grafana/dashboards/rpc-performance.json # docker/telemetry/grafana/dashboards/transaction-overview.json # docs/telemetry-runbook.md
2026-08-21 14:20:56 +00:00 · 2026-07-30 19:15:15 +01:00
parent 370130a9f9 09c9dc4ec8
commit b31183cf78
11 changed files with 182 additions and 127 deletions
--- a/docker/telemetry/grafana/dashboards/consensus-health.json
+++ b/docker/telemetry/grafana/dashboards/consensus-health.json
@@ -36,7 +36,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops"
+          "unit": "suffix: validations/s"
        },
        "overrides": []
      },
@@ -70,9 +70,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: closes/s",
          "custom": {
-            "axisLabel": "Events / Sec",
+            "axisLabel": "Closes / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -111,7 +111,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: proposals/s",
          "custom": {
            "axisLabel": "Proposals / Sec",
            "spanNulls": 1800000,
@@ -158,7 +158,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: events/s",
          "custom": {
            "axisLabel": "Events / Sec",
            "spanNulls": 1800000,
@@ -205,7 +205,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: events/s",
          "custom": {
            "axisLabel": "Events / Sec",
            "spanNulls": 1800000,
@@ -524,7 +524,14 @@
        },
        "overrides": []
      },
-      "id": 12
+      "id": 12,
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 600
+        }
+      }
    },
    {
      "title": "Consensus Accept Duration Heatmap",
@@ -539,7 +546,8 @@
      "options": {
        "tooltip": {
          "mode": "multi",
-          "sort": "desc"
+          "sort": "desc",
+          "maxHeight": 600
        },
        "yAxis": {
          "axisLabel": "Duration (ms)",
@@ -584,7 +592,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: rounds/s",
          "custom": {
            "axisLabel": "Rounds / Sec",
            "spanNulls": 1800000,
@@ -595,7 +603,14 @@
        },
        "overrides": []
      },
-      "id": 14
+      "id": 14,
+      "options": {
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 600
+        }
+      }
    },
    {
      "title": "Close Time Vote Bins & Resolution",
@@ -657,7 +672,8 @@
      "options": {
        "tooltip": {
          "mode": "multi",
-          "sort": "desc"
+          "sort": "desc",
+          "maxHeight": 600
        },
        "legend": {
          "displayMode": "table",
@@ -880,7 +896,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: failures/s",
          "custom": {
            "axisLabel": "Failures / Sec",
            "spanNulls": 1800000,
@@ -927,7 +943,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: checks/s",
          "custom": {
            "axisLabel": "Checks / Sec",
            "spanNulls": 1800000,
@@ -968,9 +984,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: changes/s",
          "custom": {
-            "axisLabel": "Mode Changes / Sec",
+            "axisLabel": "Changes / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
--- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json
+++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json
@@ -496,9 +496,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "cps",
+          "unit": "suffix: messages/s",
          "custom": {
-            "axisLabel": "Messages In",
+            "axisLabel": "Messages / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -543,9 +543,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "cps",
+          "unit": "suffix: messages/s",
          "custom": {
-            "axisLabel": "Messages In",
+            "axisLabel": "Messages / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -584,9 +584,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "cps",
+          "unit": "suffix: messages/s",
          "custom": {
-            "axisLabel": "Messages In",
+            "axisLabel": "Messages / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -637,9 +637,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "cps",
+          "unit": "suffix: messages/s",
          "custom": {
-            "axisLabel": "Messages In",
+            "axisLabel": "Messages / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -985,8 +985,9 @@
        "alignValue": "left",
        "rowHeight": 0.9,
        "tooltip": {
-          "mode": "single",
-          "sort": "none"
+          "maxHeight": 600,
+          "mode": "multi",
+          "sort": "desc"
        },
        "legend": {
          "displayMode": "list",
@@ -1012,7 +1013,7 @@
            "fillOpacity": 80,
            "lineWidth": 0,
            "insertNulls": false,
-            "spanNulls": false
+            "spanNulls": 1800000
          },
          "mappings": [
            {
@@ -1393,16 +1394,49 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
-          "unit": "short",
+          "unit": "suffix: reads/s",
          "custom": {
-            "axisLabel": "Reads/s & Depth",
+            "axisLabel": "Reads / Sec & Depth",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
            "pointSize": 3
          }
        },
-        "overrides": []
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "/^Read Queue/"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "short"
+              },
+              {
+                "id": "custom.axisPlacement",
+                "value": "right"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "/^Read Threads Running/"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "short"
+              },
+              {
+                "id": "custom.axisPlacement",
+                "value": "right"
+              }
+            ]
+          }
+        ]
      },
      "id": 24
    },
--- a/docker/telemetry/grafana/dashboards/ledger-operations.json
+++ b/docker/telemetry/grafana/dashboards/ledger-operations.json
@@ -36,7 +36,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops"
+          "unit": "suffix: ledgers/s"
        },
        "overrides": []
      }
@@ -109,7 +109,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops"
+          "unit": "suffix: ledgers/s"
        },
        "overrides": []
      }
@@ -127,7 +127,8 @@
      "options": {
        "tooltip": {
          "mode": "multi",
-          "sort": "desc"
+          "sort": "desc",
+          "maxHeight": 600
        },
        "yAxis": {
          "axisLabel": "Duration (ms)"
@@ -218,9 +219,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: transactions/s",
          "custom": {
-            "axisLabel": "Operations / Sec",
+            "axisLabel": "Transactions / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -258,7 +259,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops"
+          "unit": "suffix: ledgers/s"
        },
        "overrides": []
      }
--- a/docker/telemetry/grafana/dashboards/network-traffic.json
+++ b/docker/telemetry/grafana/dashboards/network-traffic.json
@@ -63,7 +63,7 @@
        "h": 8,
        "w": 12,
        "x": 12,
-        "y": 0
+        "y": 24
      },
      "options": {
        "tooltip": {
@@ -175,9 +175,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "cps",
+          "unit": "suffix: messages/s",
          "custom": {
-            "axisLabel": "Messages",
+            "axisLabel": "Messages / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -227,9 +227,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "cps",
+          "unit": "suffix: messages/s",
          "custom": {
-            "axisLabel": "Messages",
+            "axisLabel": "Messages / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -285,9 +285,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "cps",
+          "unit": "suffix: messages/s",
          "custom": {
-            "axisLabel": "Messages",
+            "axisLabel": "Messages / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -343,9 +343,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "cps",
+          "unit": "suffix: messages/s",
          "custom": {
-            "axisLabel": "Messages",
+            "axisLabel": "Messages / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -363,7 +363,7 @@
        "h": 8,
        "w": 12,
        "x": 12,
-        "y": 24
+        "y": 0
      },
      "options": {
        "tooltip": {
--- a/docker/telemetry/grafana/dashboards/node-health.json
+++ b/docker/telemetry/grafana/dashboards/node-health.json
@@ -419,7 +419,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: fetches/s",
          "custom": {}
        },
        "overrides": []
@@ -453,7 +453,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: mismatches/s",
          "thresholds": {
            "steps": [
              {
@@ -1886,9 +1886,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "µs",
+          "unit": "\u00b5s",
          "custom": {
-            "axisLabel": "Duration (µs)",
+            "axisLabel": "Duration (\u00b5s)",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -1986,9 +1986,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "µs",
+          "unit": "\u00b5s",
          "custom": {
-            "axisLabel": "Wait Time (µs)",
+            "axisLabel": "Wait Time (\u00b5s)",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -2140,7 +2140,7 @@
    },
    {
      "title": "State Duration Rate (All States)",
-      "description": "###### What this is:\n*Which operating mode the node is accumulating time in right now, one line per state, normalized to seconds per second.*\n\n###### How it's computed:\n*Per-second rate of each state's duration counter, scaled from microseconds to seconds. The five lines sum to about 1.0 because the node is always in exactly one state.*\n\n###### Reading it:\n*The line sitting near 1.0 is the state the node is currently in; the others sit at 0. A handover between two lines marks a state change, and its width is how long that state lasted.*\n\n###### Healthy range:\n*Full near 1.0 with every other line at 0.*\n\n###### Watch for:\n*Time accumulating in Connected or Syncing means the node is catching up rather than serving; repeated handovers mean it is flapping.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* — the node's sync level: Disconnected, Connected, Syncing, Tracking, Full.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Server states](https://xrpl.org/docs/concepts/networks-and-servers/rippled-server-states)",
+      "description": "###### What this is:\n*Which operating mode the node is accumulating time in right now, one line per state, normalized to seconds per second.*\n\n###### How it's computed:\n*Per-second rate of each state's duration counter, scaled from microseconds to seconds. The five lines sum to about 1.0 because the node is always in exactly one state.*\n\n###### Reading it:\n*The line sitting near 1.0 is the state the node is currently in; the others sit at 0. A handover between two lines marks a state change, and its width is how long that state lasted.*\n\n###### Healthy range:\n*Full near 1.0 with every other line at 0.*\n\n###### Watch for:\n*Time accumulating in Connected or Syncing means the node is catching up rather than serving; repeated handovers mean it is flapping.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Server states](https://xrpl.org/docs/concepts/networks-and-servers/rippled-server-states)",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
@@ -2190,7 +2190,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "short",
+          "unit": "suffix: s/s",
          "custom": {
            "axisLabel": "Rate (s/s)",
            "spanNulls": 1800000,
@@ -2230,9 +2230,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ms",
+          "unit": "\u00b5s",
          "custom": {
-            "axisLabel": "Duration (ms)",
+            "axisLabel": "Duration (\u00b5s)",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -2270,9 +2270,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ms",
+          "unit": "\u00b5s",
          "custom": {
-            "axisLabel": "Wait Time (ms)",
+            "axisLabel": "Wait Time (\u00b5s)",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
--- a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json
+++ b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json
@@ -67,9 +67,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "cps",
+          "unit": "suffix: messages/s",
          "custom": {
-            "axisLabel": "Messages",
+            "axisLabel": "Messages / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -195,9 +195,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "cps",
+          "unit": "suffix: messages/s",
          "custom": {
-            "axisLabel": "Count",
+            "axisLabel": "Messages / Sec & Bytes / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -328,9 +328,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "cps",
+          "unit": "suffix: messages/s",
          "custom": {
-            "axisLabel": "Messages",
+            "axisLabel": "Messages / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
--- a/docker/telemetry/grafana/dashboards/peer-network.json
+++ b/docker/telemetry/grafana/dashboards/peer-network.json
@@ -37,7 +37,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: proposals/s",
          "custom": {
            "axisLabel": "Proposals / Sec",
            "spanNulls": 1800000,
@@ -77,7 +77,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: validations/s",
          "custom": {
            "axisLabel": "Validations / Sec",
            "spanNulls": 1800000,
@@ -111,13 +111,13 @@
          "datasource": {
            "type": "prometheus"
          },
-          "expr": "label_replace(sum by (proposal_trusted, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", proposal_trusted=~\"$proposal_trusted\", span_name=\"peer.proposal.receive\"}[$__rate_interval])), \"series\", \"Trusted = $1\", \"proposal_trusted\", \"(.*)\")"
+          "expr": "label_replace(sum by (proposal_trusted, service_instance_id, xrpl_branch, xrpl_node_role) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", proposal_trusted=~\"$proposal_trusted\", span_name=\"peer.proposal.receive\"}[$__rate_interval])), \"series\", \"Trusted = $1\", \"proposal_trusted\", \"(.*)\")"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops"
+          "unit": "short"
        },
        "overrides": []
      }
@@ -144,13 +144,13 @@
          "datasource": {
            "type": "prometheus"
          },
-          "expr": "label_replace(sum by (validation_trusted, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", validation_trusted=~\"$validation_trusted\", span_name=\"peer.validation.receive\"}[$__rate_interval])), \"series\", \"Trusted = $1\", \"validation_trusted\", \"(.*)\")"
+          "expr": "label_replace(sum by (validation_trusted, service_instance_id, xrpl_branch, xrpl_node_role) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", validation_trusted=~\"$validation_trusted\", span_name=\"peer.validation.receive\"}[$__rate_interval])), \"series\", \"Trusted = $1\", \"validation_trusted\", \"(.*)\")"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops"
+          "unit": "short"
        },
        "overrides": []
      }
--- a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json
+++ b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json
@@ -282,7 +282,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: warnings/s",
          "thresholds": {
            "steps": [
              {
@@ -331,7 +331,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: drops/s",
          "thresholds": {
            "steps": [
              {
@@ -380,7 +380,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: calls/s",
          "custom": {
            "axisLabel": "Calls / Sec",
            "spanNulls": 1800000,
@@ -460,7 +460,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: calls/s",
          "custom": {
            "axisLabel": "Calls / Sec",
            "spanNulls": 1800000,
@@ -546,7 +546,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: operations/s",
          "custom": {
            "axisLabel": "Operations / Sec",
            "spanNulls": 1800000,
--- a/docker/telemetry/grafana/dashboards/rpc-performance.json
+++ b/docker/telemetry/grafana/dashboards/rpc-performance.json
@@ -166,7 +166,8 @@
      "options": {
        "tooltip": {
          "mode": "multi",
-          "sort": "desc"
+          "sort": "desc",
+          "maxHeight": 600
        },
        "yAxis": {
          "axisLabel": "Duration (ms)",
@@ -271,7 +272,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: commands/s",
          "custom": {
            "axisLabel": "Commands / Sec",
            "spanNulls": 1800000,
@@ -346,7 +347,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops"
+          "unit": "suffix: messages/s"
        },
        "overrides": []
      },
@@ -385,7 +386,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: requests/s",
          "custom": {
            "axisLabel": "Requests / Sec",
            "spanNulls": 1800000,
@@ -432,7 +433,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: requests/s",
          "custom": {
            "axisLabel": "Requests / Sec",
            "spanNulls": 1800000,
--- a/docker/telemetry/grafana/dashboards/transaction-overview.json
+++ b/docker/telemetry/grafana/dashboards/transaction-overview.json
@@ -36,7 +36,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: transactions/s",
          "thresholds": {
            "steps": [
              {
@@ -137,9 +137,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: transactions/s",
          "custom": {
-            "axisLabel": "TX / Sec",
+            "axisLabel": "Transactions / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -183,9 +183,9 @@
      ],
      "fieldConfig": {
        "defaults": {
-          "unit": "ops",
+          "unit": "suffix: transactions/s",
          "custom": {
-            "axisLabel": "Failed TX / Sec",
+            "axisLabel": "Transactions / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -224,7 +224,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: transactions/s",
          "custom": {
            "axisLabel": "Transactions / Sec",
            "spanNulls": 1800000,
@@ -271,7 +271,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: transactions/s",
          "custom": {
            "axisLabel": "Transactions / Sec",
            "spanNulls": 1800000,
@@ -418,7 +418,9 @@
        "alignValue": "center",
        "rowHeight": 0.85,
        "tooltip": {
-          "mode": "single"
+          "mode": "multi",
+          "sort": "desc",
+          "maxHeight": 600
        },
        "legend": {
          "displayMode": "list",
@@ -463,7 +465,8 @@
          },
          "custom": {
            "fillOpacity": 80,
-            "lineWidth": 0
+            "lineWidth": 0,
+            "spanNulls": 1800000
          }
        },
        "overrides": []
@@ -549,9 +552,9 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: failures/s",
          "custom": {
-            "axisLabel": "Failed Spans / Sec",
+            "axisLabel": "Failures / Sec",
            "spanNulls": 1800000,
            "insertNulls": false,
            "showPoints": "auto",
@@ -677,7 +680,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: enqueues/s",
          "custom": {
            "axisLabel": "Enqueues / Sec",
            "spanNulls": 1800000,
@@ -800,7 +803,7 @@
      "fieldConfig": {
        "defaults": {
          "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
-          "unit": "ops",
+          "unit": "suffix: cleanups/s",
          "custom": {
            "axisLabel": "Cleanups / Sec",
            "spanNulls": 1800000,