diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index 35e525d27f..b5231ae7d5 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -274,7 +274,7 @@ }, { "title": "Operating Mode Transitions", - "description": "**What:** Transitions into each operating mode, per interval.\n**How it's computed:** increase() over the per-mode transition counters, so each point is the number of transitions in that bucket and the series stays correct across an xrpld restart (the counters reset to 0).\n**Reading it:** Few transitions is good; a stable node rarely leaves Full. Brief flaps are visible here even when they are too short to appear on Operating Mode (State Timeline), which can only sample state once per scrape.\n**Healthy range:** workload-dependent; low and infrequent transitions.\n**Watch for:** Frequent transitions out of Full, or any into Disconnected/Syncing (flapping).\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp NetworkOPsImp::Stats ctor", + "description": "**What:** Transitions into each operating mode, per interval.\n**How it's computed:** round(increase(...[$__interval])) over the per-mode transition counters. $__interval tiles the buckets exactly, so each bar is the transitions in that bucket and the legend Total is the true count; $__rate_interval would overlap each bucket by one scrape and inflate it (measured +5% at a 36h range, +26% zoomed in). round() removes increase()'s extrapolation, which otherwise reports fractional counts. The series stays correct across an xrpld restart (the counters reset to 0).\n**Reading it:** Few transitions is good; a stable node rarely leaves Full. Brief flaps are visible here even when they are too short to appear on Operating Mode (State Timeline), which can only sample state once per scrape.\n**Healthy range:** workload-dependent; low and infrequent transitions.\n**Watch for:** Frequent transitions out of Full, or any into Disconnected/Syncing (flapping).\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp NetworkOPsImp::Stats ctor", "type": "timeseries", "gridPos": { "h": 10, @@ -287,6 +287,12 @@ "maxHeight": 600, "mode": "multi", "sort": "desc" + }, + "legend": { + "calcs": ["sum", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true } }, "targets": [ @@ -294,35 +300,35 @@ "datasource": { "type": "prometheus" }, - "expr": "increase(state_accounting_full_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", + "expr": "round(increase(state_accounting_full_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__interval]))", "legendFormat": "Full [{{service_instance_id}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "increase(state_accounting_tracking_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", + "expr": "round(increase(state_accounting_tracking_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__interval]))", "legendFormat": "Tracking [{{service_instance_id}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "increase(state_accounting_syncing_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", + "expr": "round(increase(state_accounting_syncing_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__interval]))", "legendFormat": "Syncing [{{service_instance_id}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "increase(state_accounting_connected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", + "expr": "round(increase(state_accounting_connected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__interval]))", "legendFormat": "Connected [{{service_instance_id}}]" }, { "datasource": { "type": "prometheus" }, - "expr": "increase(state_accounting_disconnected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", + "expr": "round(increase(state_accounting_disconnected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__interval]))", "legendFormat": "Disconnected [{{service_instance_id}}]" } ], @@ -335,14 +341,17 @@ "insertNulls": false, "showPoints": "auto", "pointSize": 5, - "lineWidth": 1, - "fillOpacity": 0, - "gradientMode": "none" - } + "lineWidth": 0, + "fillOpacity": 70, + "gradientMode": "none", + "drawStyle": "bars" + }, + "decimals": 0 }, "overrides": [] }, - "id": 6 + "id": 6, + "interval": "1m" }, { "title": "I/O Latency", diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 59a6519e25..2ee2be6851 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -725,24 +725,24 @@ Requires `trace_peer=1` in the `[telemetry]` config section. ### Node Health -- System Metrics (`node-health`) -| Panel | Type | PromQL | Labels Used | -| -------------------------------------- | ---------- | --------------------------------------------------------------------------------- | ----------- | -| Validated Ledger Age | stat | `ledgermaster_validated_ledger_age` | — | -| Published Ledger Age | stat | `ledgermaster_published_ledger_age` | — | -| Operating Mode (Time Share) | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))` | — | -| Operating Mode Transitions | timeseries | `increase(state_accounting_*_transitions[$__rate_interval])` | — | -| I/O Latency | timeseries | `histogram_quantile(0.95, ios_latency_bucket)` | — | -| Job Queue Depth | timeseries | `jobq_job_count` | — | -| Ledger Fetch Rate | stat | `rate(ledger_fetches_total[$__rate_interval])` | — | -| Ledger History Mismatches | stat | `rate(ledger_history_mismatch_total[$__rate_interval])` | — | -| Key Jobs Execution Time | timeseries | `acceptledger{quantile="$quantile"}` (+ 10 more key jobs) | `quantile` | -| Key Jobs Dequeue Wait Time | timeseries | `acceptledger_q{quantile="$quantile"}` (+ 10 more) | `quantile` | -| FullBelowCache Size | timeseries | `node_family_full_below_cache_size` | — | -| FullBelowCache Hit Rate | gauge | `node_family_full_below_cache_hit_rate` | — | -| Ledger Publish Gap | stat | `Published_Ledger_Age - Validated_Ledger_Age` | — | -| State Duration Rate (Full vs Tracking) | timeseries | `rate(state_accounting_full_duration[5m]) / 1000000` | — | -| All Jobs Execution Time (Detail) | timeseries | `histogram_quantile($quantile, rate(job_running_us_bucket[5m])) by job_type` — µs | `quantile` | -| All Jobs Dequeue Wait (Detail) | timeseries | `histogram_quantile($quantile, rate(job_queued_us_bucket[5m])) by job_type` — µs | `quantile` | +| Panel | Type | PromQL | Labels Used | +| -------------------------------------- | ---------- | ---------------------------------------------------------------------------------- | ----------- | +| Validated Ledger Age | stat | `ledgermaster_validated_ledger_age` | — | +| Published Ledger Age | stat | `ledgermaster_published_ledger_age` | — | +| Operating Mode (Time Share) | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))` | — | +| Operating Mode Transitions | timeseries | `round(increase(state_accounting_*_transitions[$__interval]))` (bars, Min step 1m) | — | +| I/O Latency | timeseries | `histogram_quantile(0.95, ios_latency_bucket)` | — | +| Job Queue Depth | timeseries | `jobq_job_count` | — | +| Ledger Fetch Rate | stat | `rate(ledger_fetches_total[$__rate_interval])` | — | +| Ledger History Mismatches | stat | `rate(ledger_history_mismatch_total[$__rate_interval])` | — | +| Key Jobs Execution Time | timeseries | `acceptledger{quantile="$quantile"}` (+ 10 more key jobs) | `quantile` | +| Key Jobs Dequeue Wait Time | timeseries | `acceptledger_q{quantile="$quantile"}` (+ 10 more) | `quantile` | +| FullBelowCache Size | timeseries | `node_family_full_below_cache_size` | — | +| FullBelowCache Hit Rate | gauge | `node_family_full_below_cache_hit_rate` | — | +| Ledger Publish Gap | stat | `Published_Ledger_Age - Validated_Ledger_Age` | — | +| State Duration Rate (Full vs Tracking) | timeseries | `rate(state_accounting_full_duration[5m]) / 1000000` | — | +| All Jobs Execution Time (Detail) | timeseries | `histogram_quantile($quantile, rate(job_running_us_bucket[5m])) by job_type` — µs | `quantile` | +| All Jobs Dequeue Wait (Detail) | timeseries | `histogram_quantile($quantile, rate(job_queued_us_bucket[5m])) by job_type` — µs | `quantile` | ### Network Traffic -- System Metrics (`network-traffic`)