diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index 42916eb34a..2f01f834d9 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -496,9 +496,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages In", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -543,9 +543,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages In", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -584,9 +584,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages In", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -637,9 +637,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages In", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -985,8 +985,9 @@ "alignValue": "left", "rowHeight": 0.9, "tooltip": { - "mode": "single", - "sort": "none" + "maxHeight": 600, + "mode": "multi", + "sort": "desc" }, "legend": { "displayMode": "list", @@ -1012,7 +1013,7 @@ "fillOpacity": 80, "lineWidth": 0, "insertNulls": false, - "spanNulls": false + "spanNulls": 1800000 }, "mappings": [ { @@ -1117,16 +1118,33 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "reqps", + "unit": "suffix: ledgers/s", "custom": { - "axisLabel": "Rate (per second)", + "axisLabel": "Ledgers / Sec & Fetches / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^Ledger Fetches/" + }, + "properties": [ + { + "id": "unit", + "value": "suffix: fetches/s" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] }, "id": 19 }, @@ -1376,16 +1394,49 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "short", + "unit": "suffix: reads/s", "custom": { - "axisLabel": "Reads/s & Depth", + "axisLabel": "Reads / Sec & Depth", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^Read Queue/" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^Read Threads Running/" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] }, "id": 24 }, diff --git a/docker/telemetry/grafana/dashboards/network-traffic.json b/docker/telemetry/grafana/dashboards/network-traffic.json index 7323a32bef..724f05f652 100644 --- a/docker/telemetry/grafana/dashboards/network-traffic.json +++ b/docker/telemetry/grafana/dashboards/network-traffic.json @@ -63,7 +63,7 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 24 }, "options": { "tooltip": { @@ -175,9 +175,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -227,9 +227,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -285,9 +285,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -343,9 +343,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -363,7 +363,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 0 }, "options": { "tooltip": { diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index a21bb2d574..44ea4cc5cb 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -113,10 +113,11 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 24 }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -163,7 +164,7 @@ "unit": "percentunit", "custom": { "axisLabel": "Time in mode", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3, @@ -183,10 +184,11 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 24 }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -233,7 +235,7 @@ "unit": "short", "custom": { "axisLabel": "Transitions", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -254,6 +256,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -279,7 +282,7 @@ "unit": "ms", "custom": { "axisLabel": "Latency (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -300,6 +303,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -318,7 +322,7 @@ "unit": "short", "custom": { "axisLabel": "Jobs", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -335,7 +339,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 8 }, "options": { "tooltip": { @@ -354,7 +358,7 @@ ], "fieldConfig": { "defaults": { - "unit": "ops" + "unit": "suffix: fetches/s" }, "overrides": [] } @@ -367,7 +371,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 8 }, "options": { "tooltip": { @@ -386,7 +390,7 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: mismatches/s", "thresholds": { "steps": [ { @@ -427,6 +431,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -512,10 +517,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ms", + "unit": "µs", "custom": { - "axisLabel": "Duration (ms)", - "spanNulls": true, + "axisLabel": "Duration (µs)", + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -536,6 +541,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -621,10 +627,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ms", + "unit": "µs", "custom": { - "axisLabel": "Wait Time (ms)", - "spanNulls": true, + "axisLabel": "Wait Time (µs)", + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -645,6 +651,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -663,7 +670,7 @@ "unit": "short", "custom": { "axisLabel": "Entries", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -782,6 +789,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -804,10 +812,10 @@ ], "fieldConfig": { "defaults": { - "unit": "short", + "unit": "suffix: s/s", "custom": { "axisLabel": "Rate (s/s)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -828,6 +836,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -843,10 +852,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ms", + "unit": "µs", "custom": { - "axisLabel": "Duration (ms)", - "spanNulls": true, + "axisLabel": "Duration (µs)", + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -867,6 +876,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -882,10 +892,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ms", + "unit": "µs", "custom": { - "axisLabel": "Wait Time (ms)", - "spanNulls": true, + "axisLabel": "Wait Time (µs)", + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 diff --git a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json index f86a75b68c..2b1ffc8ea1 100644 --- a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json +++ b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json @@ -67,9 +67,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -195,9 +195,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Count", + "axisLabel": "Messages / Sec & Bytes / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", @@ -328,9 +328,9 @@ "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "cps", + "unit": "suffix: messages/s", "custom": { - "axisLabel": "Messages", + "axisLabel": "Messages / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", diff --git a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json index 28f9a51ad5..bae50fa772 100644 --- a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json +++ b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json @@ -54,6 +54,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -81,7 +82,7 @@ "unit": "ms", "custom": { "axisLabel": "Latency (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -98,10 +99,11 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 24 }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -129,7 +131,7 @@ "unit": "decbytes", "custom": { "axisLabel": "Size (Bytes)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -146,10 +148,11 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 24 }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -193,7 +196,7 @@ "unit": "ms", "custom": { "axisLabel": "Latency (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -214,6 +217,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -241,7 +245,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -262,6 +266,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -289,7 +294,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -306,7 +311,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 8 }, "options": { "tooltip": { @@ -326,7 +331,7 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: warnings/s", "thresholds": { "steps": [ { @@ -355,7 +360,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 8 }, "options": { "tooltip": { @@ -375,7 +380,7 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: drops/s", "thresholds": { "steps": [ { @@ -408,6 +413,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -424,10 +430,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: calls/s", "custom": { "axisLabel": "Calls / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -448,6 +454,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -467,7 +474,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -488,6 +495,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -504,10 +512,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: calls/s", "custom": { "axisLabel": "Calls / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -528,6 +536,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -555,7 +564,7 @@ "unit": "ms", "custom": { "axisLabel": "Duration (ms)", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 @@ -576,6 +585,7 @@ }, "options": { "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "desc" } @@ -600,10 +610,10 @@ ], "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "suffix: operations/s", "custom": { "axisLabel": "Operations / Sec", - "spanNulls": true, + "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", "pointSize": 3 diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 57a5c7ecba..22aded49fc 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -718,24 +718,24 @@ Requires `trace_peer=1` in the `[telemetry]` config section. ### Node Health -- System Metrics (`node-health`) -| Panel | Type | PromQL | Labels Used | -| -------------------------------------- | ---------- | ---------------------------------------------------------- | ----------- | -| Validated Ledger Age | stat | `ledgermaster_validated_ledger_age` | — | -| Published Ledger Age | stat | `ledgermaster_published_ledger_age` | — | -| Operating Mode (Time Share) | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))` | — | -| Operating Mode Transitions | timeseries | `state_accounting_*_transitions` | — | -| I/O Latency | timeseries | `histogram_quantile(0.95, ios_latency_bucket)` | — | -| Job Queue Depth | timeseries | `job_count` | — | -| Ledger Fetch Rate | stat | `rate(ledger_fetches[5m])` | — | -| Ledger History Mismatches | stat | `rate(ledger_history_mismatch[5m])` | — | -| Key Jobs Execution Time | timeseries | `acceptledger{quantile="$quantile"}` (+ 10 more key jobs) | `quantile` | -| Key Jobs Dequeue Wait Time | timeseries | `acceptledger_q{quantile="$quantile"}` (+ 10 more) | `quantile` | -| FullBelowCache Size | timeseries | `node_family_full_below_cache_size` | — | -| FullBelowCache Hit Rate | gauge | `node_family_full_below_cache_hit_rate` | — | -| Ledger Publish Gap | stat | `Published_Ledger_Age - Validated_Ledger_Age` | — | -| State Duration Rate (Full vs Tracking) | timeseries | `rate(state_accounting_full_duration[5m]) / 1000000` | — | -| All Jobs Execution Time (Detail) | timeseries | `{__name__=~"", quantile="$quantile"}` | `quantile` | -| All Jobs Dequeue Wait (Detail) | timeseries | `{__name__=~"_q", quantile="$quantile"}` | `quantile` | +| Panel | Type | PromQL | Labels Used | +| -------------------------------------- | ---------- | --------------------------------------------------------------------------------- | ----------- | +| Validated Ledger Age | stat | `ledgermaster_validated_ledger_age` | — | +| Published Ledger Age | stat | `ledgermaster_published_ledger_age` | — | +| Operating Mode (Time Share) | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))` | — | +| Operating Mode Transitions | timeseries | `state_accounting_*_transitions` | — | +| I/O Latency | timeseries | `histogram_quantile(0.95, ios_latency_bucket)` | — | +| Job Queue Depth | timeseries | `job_count` | — | +| Ledger Fetch Rate | stat | `rate(ledger_fetches_total[$__rate_interval])` | — | +| Ledger History Mismatches | stat | `rate(ledger_history_mismatch_total[$__rate_interval])` | — | +| Key Jobs Execution Time | timeseries | `acceptledger{quantile="$quantile"}` (+ 10 more key jobs) | `quantile` | +| Key Jobs Dequeue Wait Time | timeseries | `acceptledger_q{quantile="$quantile"}` (+ 10 more) | `quantile` | +| FullBelowCache Size | timeseries | `node_family_full_below_cache_size` | — | +| FullBelowCache Hit Rate | gauge | `node_family_full_below_cache_hit_rate` | — | +| Ledger Publish Gap | stat | `Published_Ledger_Age - Validated_Ledger_Age` | — | +| State Duration Rate (Full vs Tracking) | timeseries | `rate(state_accounting_full_duration[5m]) / 1000000` | — | +| All Jobs Execution Time (Detail) | timeseries | `histogram_quantile($quantile, rate(job_running_us_bucket[5m])) by job_type` — µs | `quantile` | +| All Jobs Dequeue Wait (Detail) | timeseries | `histogram_quantile($quantile, rate(job_queued_us_bucket[5m])) by job_type` — µs | `quantile` | ### Network Traffic -- System Metrics (`network-traffic`) @@ -762,8 +762,8 @@ Requires `trace_peer=1` in the `[telemetry]` config section. | RPC Response Time Heatmap | heatmap | `rpc_time_bucket` | — | | Pathfinding Fast Duration | timeseries | `histogram_quantile(0.95, pathfind_fast_bucket)` | — | | Pathfinding Full Duration | timeseries | `histogram_quantile(0.95, pathfind_full_bucket)` | — | -| Resource Warnings Rate | stat | `rate(warn[5m])` | — | -| Resource Drops Rate | stat | `rate(drop[5m])` | — | +| Resource Warnings Rate | stat | `rate(warn_total[$__rate_interval])` | — | +| Resource Drops Rate | stat | `rate(drop_total[$__rate_interval])` | — | ### Span → Metric → Dashboard Summary