From 7c7509d01f887904323c836d7edfe3fe9210bf91 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Sat, 25 Jul 2026 09:02:03 +0100 Subject: [PATCH] feat(telemetry): add sync-state diagnostics (WP-A2) Five signals that explain why a node is not advancing toward full, none of which were observable before: - state_changes_total now carries {from,to} mode labels, emitted at setMode using the existing strOperatingMode helper. A bare count could not distinguish a healthy climb from a node flapping between tracking and connected. Removes the now-unused incrementStateChanges wrapper. - sync_state{initial_full_duration_us}: time to first reach full, which StateAccounting already computed but exposed only in server_info. - sync_state{network_ledger_gate}: whether the node is still refusing to build ledgers because it has no network ledger. - sync_state{server_stall_seconds} and server_stall_events_total: how long the main thread has been unresponsive. LoadManager computed this and only logged it, so a stall was invisible until the fatal threshold. The episode rule is a pure function so it can be tested without adding a test-only mutator to LoadManager. - sync_state{ledgers_behind}: how far our validated sequence trails the best sequence any peer advertises, read from already-cached peer ranges so no extra network traffic is added. Also fixes the naming checker: it derived only the first label of a multi-label instrument, so a dashboard querying the second label was wrongly rejected. Note: the clang-tidy hook cannot run in this worktree (no build directory); the remaining pre-commit hooks, the naming check, dashboard schema and harness syntax all pass. Co-Authored-By: Claude Opus 5 (1M context) --- .../scripts/levelization/results/ordering.txt | 1 + .../scripts/otel-naming/check_otel_naming.py | 16 +- .../09-data-collection-reference.md | 26 +- .../dashboards/ledger-sync-health.json | 526 +++++++++++++++++- .../telemetry/workload/expected_metrics.json | 11 +- .../telemetry/workload/validate_telemetry.py | 8 + docs/telemetry-glossary.md | 44 +- docs/telemetry-runbook.md | 57 ++ include/xrpl/server/NetworkOPs.h | 27 + src/tests/libxrpl/telemetry/MetricMacros.cpp | 229 ++++++++ .../libxrpl/telemetry/MetricsRegistry.cpp | 55 +- .../libxrpl/telemetry/SyncStateSignals.cpp | 141 +++++ src/xrpld/app/main/LoadManager.cpp | 26 + src/xrpld/app/main/LoadManager.h | 148 +++++ src/xrpld/app/misc/NetworkOPs.cpp | 69 ++- src/xrpld/telemetry/MetricsRegistry.cpp | 101 +++- src/xrpld/telemetry/MetricsRegistry.h | 82 ++- 17 files changed, 1503 insertions(+), 64 deletions(-) create mode 100644 src/tests/libxrpl/telemetry/SyncStateSignals.cpp diff --git a/.github/scripts/levelization/results/ordering.txt b/.github/scripts/levelization/results/ordering.txt index ab039431ca..f0a7741368 100644 --- a/.github/scripts/levelization/results/ordering.txt +++ b/.github/scripts/levelization/results/ordering.txt @@ -199,6 +199,7 @@ test.unit_test > xrpl.protocol tests.libxrpl > xrpl.basics tests.libxrpl > xrpl.config tests.libxrpl > xrpl.core +tests.libxrpl > xrpld.app tests.libxrpl > xrpld.telemetry tests.libxrpl > xrpl.json tests.libxrpl > xrpl.ledger diff --git a/.github/scripts/otel-naming/check_otel_naming.py b/.github/scripts/otel-naming/check_otel_naming.py index 89d02d3339..2df6c43790 100644 --- a/.github/scripts/otel-naming/check_otel_naming.py +++ b/.github/scripts/otel-naming/check_otel_naming.py @@ -156,7 +156,16 @@ BLOCK_COMMENT = re.compile(r"/\*.*?\*/", re.DOTALL) TRACEQL_SCOPE = re.compile(r"^(?:span|resource|event|link|instrumentation_scope)\.") # An OTel metric label key as emitted in C++: `Add(.., {{"label", ...}})` / # `{{"label", value}}` instrument calls in MetricsRegistry. +# +# Two patterns are needed because a label set is a nested initializer list: +# `{{"a", x}, {"b", y}}`. The FIRST label is preceded by the doubled brace that +# opens both the set and the pair, while every SUBSEQUENT label is preceded by +# `}, {` closing the previous pair and opening the next. Matching only the +# doubled-brace form would derive just the first label of every multi-label +# instrument, silently under-deriving the L6 key set and making Rule D reject a +# dashboard that queries a label the code genuinely emits. METRIC_LABEL = re.compile(r'\{\{\s*"([a-z_][a-z0-9_]*)"\s*,') +METRIC_LABEL_NEXT = re.compile(r'\}\s*,\s*\{\s*"([a-z_][a-z0-9_]*)"\s*,') def strip_comments(text: str) -> str: @@ -799,7 +808,11 @@ def run_rule_c_tempo(root: Path, l1_keys: Set[str], report: Report) -> None: def metric_label_names(root: Path) -> Set[str]: """L6: OTel native-metric label keys emitted by the telemetry code, e.g. `counter->Add(1, {{"job_type", value}})` in MetricsRegistry.cpp. These are - a valid source of dashboard labels distinct from span attributes (L1).""" + a valid source of dashboard labels distinct from span attributes (L1). + + Collects both the first label of a set and every subsequent one, so a + multi-label instrument such as `{{"from", a}, {"to", b}}` contributes ALL + of its keys.""" labels: Set[str] = set() for base in ("src", "include"): for p in (root / base).rglob("*.cpp"): @@ -809,6 +822,7 @@ def metric_label_names(root: Path) -> Set[str]: if "MetricsRegistry" not in p.name and "metric" not in text.lower(): continue labels |= set(METRIC_LABEL.findall(text)) + labels |= set(METRIC_LABEL_NEXT.findall(text)) return labels diff --git a/OpenTelemetryPlan/09-data-collection-reference.md b/OpenTelemetryPlan/09-data-collection-reference.md index 646d6b8c9e..a809b3822b 100644 --- a/OpenTelemetryPlan/09-data-collection-reference.md +++ b/OpenTelemetryPlan/09-data-collection-reference.md @@ -1401,13 +1401,19 @@ signal as it lands. `Type` is the instrument kind (counter / gauge / histogram / span / span attr), `Emit site` the owning source file, and `Panel` the dashboard panel that renders it. -| Signal | Type | Emit site | Panel | Meaning | -| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | -------------------------------------------------------------------- | ----------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `dns_resolve_total` (`outcome` = `resolved` \| `empty`) | counter | `OverlayImpl.cpp` — `OverlayImpl::reportDnsResolve` | DNS Resolve Outcome Rate | Peer hostname resolutions. `empty` means a configured bootstrap or `[ips_fixed]` name returned no address, so that peer is never dialled. | -| `dns_resolve_latency_ms` | histogram | `OverlayImpl.cpp` — `OverlayImpl::reportDnsResolve` | DNS Resolve Latency (p95) | Time to resolve a configured peer hostname. Seconds-scale values mean the resolver is timing out ahead of every dial. | -| `overlay_connect_total` (`outcome` = `connected` \| `tcp_fail` \| `tls_fail` \| `upgrade_fail` \| `timeout`) | counter | `ConnectAttempt.cpp` — `ConnectAttempt::reportOutcome` | Outbound Dial Outcome Rate | Outbound peer connection attempts by terminal outcome. The outcome names the stage that broke: TCP, TLS, HTTP upgrade, or no terminal state in time. | -| `overlay_dial_latency_ms` | histogram | `ConnectAttempt.cpp` — `ConnectAttempt::reportOutcome` | Outbound Dial Latency (p95) | Time from starting an outbound dial to its terminal outcome, successes and failures together. A p95 near the dial timeout means peers accept TCP but never finish the handshake. | -| `handshake_negotiation_fail_total` (`reason`, 14 values incl. `wrong_network`, `invalid_network_id`, `clock_skew`, `self_connection`, `session_verify_failed`) | counter | `Handshake.cpp` — `throwNegotiationFailure` (from `verifyHandshake`) | Handshake Negotiation Failures by Reason | Peer handshakes rejected after TLS while checking network id, clock, keys and addresses. `reason` names the failing check. | -| `unl_fetch_total` (`site` = configured UNL URI; `outcome` = the 9 `ListDisposition` strings `accepted` \| `expired` \| `same_sequence` \| `pending` \| `known_sequence` \| `unsupported_version` \| `untrusted` \| `stale` \| `invalid`, plus `fetch_error` \| `bad_status` \| `parse_error`) | counter | `ValidatorSite.cpp` — `ValidatorSite::reportFetchOutcome` | UNL Fetch Rate by Site & Outcome | Validator-list fetches per site. `accepted` is the only success; `same_sequence` and `known_sequence` are normal no-op refreshes; the three literals are transport or content faults. | -| `unl_quorum` (`metric` = `trusted_keys` \| `quorum`) | observable gauge | `MetricsRegistry.cpp` — `registerUnlQuorumGauge` | UNL Trusted Keys vs Quorum; UNL Quorum Headroom | Trusted UNL key count against the validations a ledger needs. `trusted_keys` at or below `quorum` means the node can never declare a ledger validated. | -| `clock_close_offset_seconds` (`metric` = `offset`) | observable gauge | `MetricsRegistry.cpp` — `registerClockSkewGauge` | Clock Close Offset | Network close time offset from the local clock. Negative means the local clock runs ahead. `server_info` only surfaces `close_time_offset` at 60 s or more, so this gauge sees skew far earlier. | +| Signal | Type | Emit site | Panel | Meaning | +| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------ | -------------------------------------------------------------------- | ----------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `dns_resolve_total` (`outcome` = `resolved` \| `empty`) | counter | `OverlayImpl.cpp` — `OverlayImpl::reportDnsResolve` | DNS Resolve Outcome Rate | Peer hostname resolutions. `empty` means a configured bootstrap or `[ips_fixed]` name returned no address, so that peer is never dialled. | +| `dns_resolve_latency_ms` | histogram | `OverlayImpl.cpp` — `OverlayImpl::reportDnsResolve` | DNS Resolve Latency (p95) | Time to resolve a configured peer hostname. Seconds-scale values mean the resolver is timing out ahead of every dial. | +| `overlay_connect_total` (`outcome` = `connected` \| `tcp_fail` \| `tls_fail` \| `upgrade_fail` \| `timeout`) | counter | `ConnectAttempt.cpp` — `ConnectAttempt::reportOutcome` | Outbound Dial Outcome Rate | Outbound peer connection attempts by terminal outcome. The outcome names the stage that broke: TCP, TLS, HTTP upgrade, or no terminal state in time. | +| `overlay_dial_latency_ms` | histogram | `ConnectAttempt.cpp` — `ConnectAttempt::reportOutcome` | Outbound Dial Latency (p95) | Time from starting an outbound dial to its terminal outcome, successes and failures together. A p95 near the dial timeout means peers accept TCP but never finish the handshake. | +| `handshake_negotiation_fail_total` (`reason`, 14 values incl. `wrong_network`, `invalid_network_id`, `clock_skew`, `self_connection`, `session_verify_failed`) | counter | `Handshake.cpp` — `throwNegotiationFailure` (from `verifyHandshake`) | Handshake Negotiation Failures by Reason | Peer handshakes rejected after TLS while checking network id, clock, keys and addresses. `reason` names the failing check. | +| `unl_fetch_total` (`site` = configured UNL URI; `outcome` = the 9 `ListDisposition` strings `accepted` \| `expired` \| `same_sequence` \| `pending` \| `known_sequence` \| `unsupported_version` \| `untrusted` \| `stale` \| `invalid`, plus `fetch_error` \| `bad_status` \| `parse_error`) | counter | `ValidatorSite.cpp` — `ValidatorSite::reportFetchOutcome` | UNL Fetch Rate by Site & Outcome | Validator-list fetches per site. `accepted` is the only success; `same_sequence` and `known_sequence` are normal no-op refreshes; the three literals are transport or content faults. | +| `unl_quorum` (`metric` = `trusted_keys` \| `quorum`) | observable gauge | `MetricsRegistry.cpp` — `registerUnlQuorumGauge` | UNL Trusted Keys vs Quorum; UNL Quorum Headroom | Trusted UNL key count against the validations a ledger needs. `trusted_keys` at or below `quorum` means the node can never declare a ledger validated. | +| `clock_close_offset_seconds` (`metric` = `offset`) | observable gauge | `MetricsRegistry.cpp` — `registerClockSkewGauge` | Clock Close Offset | Network close time offset from the local clock. Negative means the local clock runs ahead. `server_info` only surfaces `close_time_offset` at 60 s or more, so this gauge sees skew far earlier. | +| `state_changes_total` (`from`, `to` = `disconnected` \| `connected` \| `syncing` \| `tracking` \| `full`) | counter | `NetworkOPs.cpp` — `NetworkOPsImp::setMode` | Mode Transitions by Edge | Operating-mode transitions keyed on the (`from`, `to`) edge. The edge is what separates a clean `disconnected`→`connected`→`syncing`→`tracking`→`full` climb from `full`→`connected` flapping; an unlabelled total cannot tell them apart. | +| `sync_state` (`metric` = `initial_full_duration_us`) | observable gauge | `MetricsRegistry.cpp` — `registerSyncStateGauge` | Time to First FULL | Microseconds from process start to the first `full` transition, sourced from `NetworkOPs::getInitialSyncDurationUs()`. Stays 0 until `full` is reached, so a flat 0 is itself the "never synced" signal; once set it never changes. | +| `sync_state` (`metric` = `network_ledger_gate`) | observable gauge | `MetricsRegistry.cpp` — `registerSyncStateGauge` | Network Ledger Gate | 1 while the node is still waiting to see a full network ledger (`NetworkOPs::isNeedNetworkLedger()`), else 0. A persistent 1 blocks transaction submission and `full`, whatever the rest of the pipeline shows. | +| `sync_state` (`metric` = `server_stall_seconds`) | observable gauge | `MetricsRegistry.cpp` — `registerSyncStateGauge` | Server Stall | Current main-loop stall duration from `LoadManager::getCurrentStallSeconds()`, 0 when healthy. Same duration the load monitor logs as "Server stalled for N seconds", which previously existed only in that log line. | +| `sync_state` (`metric` = `ledgers_behind`) | observable gauge | `MetricsRegistry.cpp` — `registerSyncStateGauge` | Ledgers Behind Network | Peer-reported network tip minus our validated sequence, floored at 0 (`NetworkOPs::getLedgersBehindNetwork()`). Reads each peer's already cached ledger range, so no new network round trip. | +| `server_stall_events_total` | observable counter | `MetricsRegistry.cpp` — `registerStallEventsCounter` | Server Stall Event Rate | Distinct stall episodes since process start, counted once per episode rather than per stalled second. A rising rate is repeated fresh stalls; a flat rate with a large `server_stall_seconds` is one long stall. | diff --git a/docker/telemetry/grafana/dashboards/ledger-sync-health.json b/docker/telemetry/grafana/dashboards/ledger-sync-health.json index a8aa27df67..6c35147775 100644 --- a/docker/telemetry/grafana/dashboards/ledger-sync-health.json +++ b/docker/telemetry/grafana/dashboards/ledger-sync-health.json @@ -32,7 +32,7 @@ } ] }, - "description": "What this shows: Fresh-node ledger-sync diagnostics: pre-quorum bootstrap (Domain 0) and the ledger/tx-set acquire pipeline. — Use it to: Find out why a freshly started node is slow to reach, or never reaches, server_state full.", + "description": "What this shows: Fresh-node ledger-sync diagnostics: pre-quorum bootstrap (Domain 0) and the ledger/tx-set acquire pipeline. \u2014 Use it to: Find out why a freshly started node is slow to reach, or never reaches, server_state full.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -57,7 +57,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate of peer-hostname DNS resolutions, split by outcome (resolved or empty).*\n\n###### How it's computed:\n*Per-second rate of completed resolutions grouped by outcome, per node.*\n\n###### Reading it:\n*Resolved should account for every attempt; the empty line should stay flat at zero.*\n\n###### Healthy range:\n*A short burst of resolved at startup, then flat. Non-zero empty is always a defect.*\n\n###### Watch for:\n*Any empty rate means a configured bootstrap or [ips_fixed] hostname returned no address, so the node never even tries to dial that peer.*\n\n###### Keywords:\n- **DNS resolve** *(per node)* — turning a configured peer hostname into IP addresses before any dial is attempted; `outcome=empty` means the name resolved to nothing.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl::reportDnsResolve`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#dns-resolve)", + "description": "###### What this is:\n*Rate of peer-hostname DNS resolutions, split by outcome (resolved or empty).*\n\n###### How it's computed:\n*Per-second rate of completed resolutions grouped by outcome, per node.*\n\n###### Reading it:\n*Resolved should account for every attempt; the empty line should stay flat at zero.*\n\n###### Healthy range:\n*A short burst of resolved at startup, then flat. Non-zero empty is always a defect.*\n\n###### Watch for:\n*Any empty rate means a configured bootstrap or [ips_fixed] hostname returned no address, so the node never even tries to dial that peer.*\n\n###### Keywords:\n- **DNS resolve** *(per node)* \u2014 turning a configured peer hostname into IP addresses before any dial is attempted; `outcome=empty` means the name resolved to nothing.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl::reportDnsResolve`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#dns-resolve)", "fieldConfig": { "defaults": { "color": { @@ -160,7 +160,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Time taken to resolve a configured peer hostname, at the 95th percentile.*\n\n###### How it's computed:\n*Resolution duration samples aggregated to their 95th percentile per node.*\n\n###### Reading it:\n*Lower is better; it is the delay before the node can start dialling peers.*\n\n###### Healthy range:\n*Tens of milliseconds against a healthy resolver.*\n\n###### Watch for:\n*Seconds-scale latency means the resolver is timing out and every bootstrap attempt is paying that delay before the first dial.*\n\n###### Keywords:\n- **DNS resolve** *(per node)* — turning a configured peer hostname into IP addresses before any dial is attempted; slow resolution delays the whole bootstrap.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl::reportDnsResolve`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#dns-resolve)", + "description": "###### What this is:\n*Time taken to resolve a configured peer hostname, at the 95th percentile.*\n\n###### How it's computed:\n*Resolution duration samples aggregated to their 95th percentile per node.*\n\n###### Reading it:\n*Lower is better; it is the delay before the node can start dialling peers.*\n\n###### Healthy range:\n*Tens of milliseconds against a healthy resolver.*\n\n###### Watch for:\n*Seconds-scale latency means the resolver is timing out and every bootstrap attempt is paying that delay before the first dial.*\n\n###### Keywords:\n- **DNS resolve** *(per node)* \u2014 turning a configured peer hostname into IP addresses before any dial is attempted; slow resolution delays the whole bootstrap.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl::reportDnsResolve`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#dns-resolve)", "fieldConfig": { "defaults": { "color": { @@ -263,7 +263,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate of outbound peer connection attempts, split by terminal outcome.*\n\n###### How it's computed:\n*Per-second rate of finished dials grouped by outcome, per node. Filter the outcome set with the Dial Outcome variable.*\n\n###### Reading it:\n*Connected should dominate. The failure lines name the stage that broke: tcp_fail (no route or refused), tls_fail (TLS handshake), upgrade_fail (HTTP upgrade or protocol negotiation), timeout (no terminal state in time).*\n\n###### Healthy range:\n*Connected rising to the configured peer count, then flat with failures near zero.*\n\n###### Watch for:\n*All attempts landing on one failure outcome and no connected line — the node has no outbound peers and can never sync.*\n\n###### Keywords:\n- **Outbound dial latency** *(per node)* — an outbound peer connection attempt from TCP connect through TLS to protocol upgrade; each attempt ends in exactly one outcome.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ConnectAttempt.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/ConnectAttempt.cpp)\n\n###### Function:\n`ConnectAttempt::reportOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", + "description": "###### What this is:\n*Rate of outbound peer connection attempts, split by terminal outcome.*\n\n###### How it's computed:\n*Per-second rate of finished dials grouped by outcome, per node. Filter the outcome set with the Dial Outcome variable.*\n\n###### Reading it:\n*Connected should dominate. The failure lines name the stage that broke: tcp_fail (no route or refused), tls_fail (TLS handshake), upgrade_fail (HTTP upgrade or protocol negotiation), timeout (no terminal state in time).*\n\n###### Healthy range:\n*Connected rising to the configured peer count, then flat with failures near zero.*\n\n###### Watch for:\n*All attempts landing on one failure outcome and no connected line \u2014 the node has no outbound peers and can never sync.*\n\n###### Keywords:\n- **Outbound dial latency** *(per node)* \u2014 an outbound peer connection attempt from TCP connect through TLS to protocol upgrade; each attempt ends in exactly one outcome.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ConnectAttempt.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/ConnectAttempt.cpp)\n\n###### Function:\n`ConnectAttempt::reportOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", "fieldConfig": { "defaults": { "color": { @@ -366,7 +366,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Time from starting an outbound peer dial to its terminal outcome, at the 95th percentile.*\n\n###### How it's computed:\n*Dial duration samples aggregated to their 95th percentile per node.*\n\n###### Reading it:\n*Lower is better. The series covers successes and failures together, so a rising p95 usually means attempts are ending in timeout rather than being refused fast.*\n\n###### Healthy range:\n*Tens to low hundreds of milliseconds on a local or same-region peer.*\n\n###### Watch for:\n*A p95 pinned near the dial timeout, which means peers accept the TCP connection but never complete the handshake.*\n\n###### Keywords:\n- **Outbound dial latency** *(per node)* — elapsed time of an outbound peer connection attempt, measured to whichever outcome ends it.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ConnectAttempt.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/ConnectAttempt.cpp)\n\n###### Function:\n`ConnectAttempt::reportOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", + "description": "###### What this is:\n*Time from starting an outbound peer dial to its terminal outcome, at the 95th percentile.*\n\n###### How it's computed:\n*Dial duration samples aggregated to their 95th percentile per node.*\n\n###### Reading it:\n*Lower is better. The series covers successes and failures together, so a rising p95 usually means attempts are ending in timeout rather than being refused fast.*\n\n###### Healthy range:\n*Tens to low hundreds of milliseconds on a local or same-region peer.*\n\n###### Watch for:\n*A p95 pinned near the dial timeout, which means peers accept the TCP connection but never complete the handshake.*\n\n###### Keywords:\n- **Outbound dial latency** *(per node)* \u2014 elapsed time of an outbound peer connection attempt, measured to whichever outcome ends it.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ConnectAttempt.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/ConnectAttempt.cpp)\n\n###### Function:\n`ConnectAttempt::reportOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", "fieldConfig": { "defaults": { "color": { @@ -469,7 +469,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate of peer handshakes rejected during protocol negotiation, split by reason.*\n\n###### How it's computed:\n*Per-second rate of rejected handshakes grouped by reason, per node. Filter the reason set with the Handshake Reason variable.*\n\n###### Reading it:\n*Flat at zero is healthy. The reason names the exact check that rejected the peer, so one dominant reason is the fault to fix.*\n\n###### Healthy range:\n*Zero, or a low background rate of self_connection and remote_ip_mismatch on a NAT'd host.*\n\n###### Watch for:\n*wrong_network or invalid_network_id — the node is configured for a different network than its peers and will never reach a quorum. clock_skew points at the local clock; session_verify_failed and bad_public_key at a misbehaving peer.*\n\n###### Keywords:\n- **Handshake negotiation failure** *(per node)* — a peer connection rejected after TLS while checking network id, clock, keys and addresses; the reason label names the failing check.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Handshake.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/Handshake.cpp)\n\n###### Function:\n`throwNegotiationFailure`\n\n###### References:\n[Peer protocol on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#handshake-negotiation-failure)", + "description": "###### What this is:\n*Rate of peer handshakes rejected during protocol negotiation, split by reason.*\n\n###### How it's computed:\n*Per-second rate of rejected handshakes grouped by reason, per node. Filter the reason set with the Handshake Reason variable.*\n\n###### Reading it:\n*Flat at zero is healthy. The reason names the exact check that rejected the peer, so one dominant reason is the fault to fix.*\n\n###### Healthy range:\n*Zero, or a low background rate of self_connection and remote_ip_mismatch on a NAT'd host.*\n\n###### Watch for:\n*wrong_network or invalid_network_id \u2014 the node is configured for a different network than its peers and will never reach a quorum. clock_skew points at the local clock; session_verify_failed and bad_public_key at a misbehaving peer.*\n\n###### Keywords:\n- **Handshake negotiation failure** *(per node)* \u2014 a peer connection rejected after TLS while checking network id, clock, keys and addresses; the reason label names the failing check.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Handshake.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/Handshake.cpp)\n\n###### Function:\n`throwNegotiationFailure`\n\n###### References:\n[Peer protocol on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#handshake-negotiation-failure)", "fieldConfig": { "defaults": { "color": { @@ -572,7 +572,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate of validator-list fetches from each configured UNL site, split by outcome.*\n\n###### How it's computed:\n*Per-second rate of fetch attempts grouped by site and outcome, per node. Filter with the UNL Site and UNL Fetch Outcome variables.*\n\n###### Reading it:\n*accepted is the only success value. same_sequence and known_sequence are normal no-op refreshes of a list the node already holds. fetch_error, bad_status and parse_error are transport or content faults; expired, stale, untrusted, invalid and unsupported_version mean the list was retrieved but rejected.*\n\n###### Healthy range:\n*A first accepted per site at startup, then a steady low rate of same_sequence refreshes.*\n\n###### Watch for:\n*A site with only fetch_error or bad_status is unreachable. Only expired or invalid means the site is reachable but its list is unusable, so no trusted keys are loaded from it.*\n\n###### Keywords:\n- **UNL fetch outcome** *(per node)* — the result of retrieving and applying a validator list from a configured site; `accepted` is the only success.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ValidatorSite.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/ValidatorSite.cpp)\n\n###### Function:\n`ValidatorSite::reportFetchOutcome`\n\n###### References:\n[UNL (Unique Node List)](https://xrpl.org/docs/concepts/consensus-protocol/unl) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-fetch-outcome)", + "description": "###### What this is:\n*Rate of validator-list fetches from each configured UNL site, split by outcome.*\n\n###### How it's computed:\n*Per-second rate of fetch attempts grouped by site and outcome, per node. Filter with the UNL Site and UNL Fetch Outcome variables.*\n\n###### Reading it:\n*accepted is the only success value. same_sequence and known_sequence are normal no-op refreshes of a list the node already holds. fetch_error, bad_status and parse_error are transport or content faults; expired, stale, untrusted, invalid and unsupported_version mean the list was retrieved but rejected.*\n\n###### Healthy range:\n*A first accepted per site at startup, then a steady low rate of same_sequence refreshes.*\n\n###### Watch for:\n*A site with only fetch_error or bad_status is unreachable. Only expired or invalid means the site is reachable but its list is unusable, so no trusted keys are loaded from it.*\n\n###### Keywords:\n- **UNL fetch outcome** *(per node)* \u2014 the result of retrieving and applying a validator list from a configured site; `accepted` is the only success.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ValidatorSite.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/ValidatorSite.cpp)\n\n###### Function:\n`ValidatorSite::reportFetchOutcome`\n\n###### References:\n[UNL (Unique Node List)](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-fetch-outcome)", "fieldConfig": { "defaults": { "color": { @@ -675,7 +675,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Trusted validator keys currently in effect, plotted against the number of agreeing validations a ledger needs.*\n\n###### How it's computed:\n*Two series read from the same gauge: trusted_keys (usable UNL size) and quorum (validations required to declare a ledger validated).*\n\n###### Reading it:\n*Trusted Keys must sit above Quorum. Where the lines cross, or where Trusted Keys is zero, the node cannot reach a quorum and will never validate a ledger no matter how healthy the rest of the pipeline looks.*\n\n###### Healthy range:\n*Trusted Keys comfortably above Quorum and both flat.*\n\n###### Watch for:\n*Trusted Keys at zero (no usable UNL loaded) or below Quorum. Steps in Quorum track validator-list changes; steps down in Trusted Keys mean keys were dropped.*\n\n###### Keywords:\n- **UNL quorum headroom** *(per node)* — trusted UNL key count minus the required quorum; at or below zero the node can never declare a ledger validated.\n- **UNL (Unique Node List)** *(per node)* — the list of validators a node trusts not to collude; the basis for its consensus and quorum.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerUnlQuorumGauge`\n\n###### References:\n[UNL (Unique Node List)](https://xrpl.org/docs/concepts/consensus-protocol/unl) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-quorum-headroom)", + "description": "###### What this is:\n*Trusted validator keys currently in effect, plotted against the number of agreeing validations a ledger needs.*\n\n###### How it's computed:\n*Two series read from the same gauge: trusted_keys (usable UNL size) and quorum (validations required to declare a ledger validated).*\n\n###### Reading it:\n*Trusted Keys must sit above Quorum. Where the lines cross, or where Trusted Keys is zero, the node cannot reach a quorum and will never validate a ledger no matter how healthy the rest of the pipeline looks.*\n\n###### Healthy range:\n*Trusted Keys comfortably above Quorum and both flat.*\n\n###### Watch for:\n*Trusted Keys at zero (no usable UNL loaded) or below Quorum. Steps in Quorum track validator-list changes; steps down in Trusted Keys mean keys were dropped.*\n\n###### Keywords:\n- **UNL quorum headroom** *(per node)* \u2014 trusted UNL key count minus the required quorum; at or below zero the node can never declare a ledger validated.\n- **UNL (Unique Node List)** *(per node)* \u2014 the list of validators a node trusts not to collude; the basis for its consensus and quorum.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerUnlQuorumGauge`\n\n###### References:\n[UNL (Unique Node List)](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-quorum-headroom)", "fieldConfig": { "defaults": { "color": { @@ -786,7 +786,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Spare trusted validator keys above the required quorum — the single number that says whether this node can ever validate.*\n\n###### How it's computed:\n*Trusted key count minus the required quorum, matched per node.*\n\n###### Reading it:\n*Positive is healthy. Zero or negative (red) means the trusted UNL is too small to ever satisfy quorum, so the node will stay short of a validated ledger.*\n\n###### Healthy range:\n*Positive; the exact figure depends on UNL size and the configured quorum.*\n\n###### Watch for:\n*Zero or below. Pair it with UNL Fetch Rate by Site & Outcome: a site stuck on fetch_error or expired is the usual cause of a UNL too small to meet quorum.*\n\n###### Keywords:\n- **UNL quorum headroom** *(per node)* — trusted UNL key count minus the required quorum; at or below zero the node can never declare a ledger validated.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerUnlQuorumGauge`\n\n###### References:\n[Validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-quorum-headroom)", + "description": "###### What this is:\n*Spare trusted validator keys above the required quorum \u2014 the single number that says whether this node can ever validate.*\n\n###### How it's computed:\n*Trusted key count minus the required quorum, matched per node.*\n\n###### Reading it:\n*Positive is healthy. Zero or negative (red) means the trusted UNL is too small to ever satisfy quorum, so the node will stay short of a validated ledger.*\n\n###### Healthy range:\n*Positive; the exact figure depends on UNL size and the configured quorum.*\n\n###### Watch for:\n*Zero or below. Pair it with UNL Fetch Rate by Site & Outcome: a site stuck on fetch_error or expired is the usual cause of a UNL too small to meet quorum.*\n\n###### Keywords:\n- **UNL quorum headroom** *(per node)* \u2014 trusted UNL key count minus the required quorum; at or below zero the node can never declare a ledger validated.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerUnlQuorumGauge`\n\n###### References:\n[Validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-quorum-headroom)", "fieldConfig": { "defaults": { "color": { @@ -853,7 +853,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*How far the network's agreed close time sits from this node's own clock.*\n\n###### How it's computed:\n*Signed offset in seconds, plus its magnitude so a threshold band applies in either direction.*\n\n###### Reading it:\n*Both series should hug zero. A negative signed value means the local clock runs ahead of the network, positive means it lags. The magnitude is what matters: the threshold lines sit at 1s (suspicious) and 60s.*\n\n###### Healthy range:\n*Magnitude under 1 second.*\n\n###### Watch for:\n*A magnitude above 1 second that does not decay, which delays consensus participation. Note that server_info only surfaces close_time_offset once the magnitude reaches 60 seconds, so this panel sees skew long before the API does; a persistent offset is a local NTP fault, not a network one.*\n\n###### Keywords:\n- **Clock close offset** *(per node)* — the difference between the network's agreed close time and this node's clock; a persistent offset delays consensus participation.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerClockSkewGauge`\n\n###### References:\n[Ledger close times on xrpl.org](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#clock-close-offset)", + "description": "###### What this is:\n*How far the network's agreed close time sits from this node's own clock.*\n\n###### How it's computed:\n*Signed offset in seconds, plus its magnitude so a threshold band applies in either direction.*\n\n###### Reading it:\n*Both series should hug zero. A negative signed value means the local clock runs ahead of the network, positive means it lags. The magnitude is what matters: the threshold lines sit at 1s (suspicious) and 60s.*\n\n###### Healthy range:\n*Magnitude under 1 second.*\n\n###### Watch for:\n*A magnitude above 1 second that does not decay, which delays consensus participation. Note that server_info only surfaces close_time_offset once the magnitude reaches 60 seconds, so this panel sees skew long before the API does; a persistent offset is a local NTP fault, not a network one.*\n\n###### Keywords:\n- **Clock close offset** *(per node)* \u2014 the difference between the network's agreed close time and this node's clock; a persistent offset delays consensus participation.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerClockSkewGauge`\n\n###### References:\n[Ledger close times on xrpl.org](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#clock-close-offset)", "fieldConfig": { "defaults": { "color": { @@ -975,6 +975,472 @@ "y": 61 }, "panels": [] + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Whether the node is still waiting to see a full network ledger before it will participate.*\n\n###### How it's computed:\n*sync_state series network_ledger_gate: 1 while the gate is closed, 0 once it opens.*\n\n###### Reading it:\n*0 (green) is healthy. A persistent 1 (red) means the node has never seen a complete network ledger, so it refuses transactions and can never reach full no matter how healthy the rest of the pipeline looks.*\n\n###### Healthy range:\n*0 within the first few minutes of startup.*\n\n###### Watch for:\n*A 1 that never clears. Pair it with the Bootstrap row \u2014 no peers or no quorum is the usual cause.*\n\n###### Keywords:\n- **Network ledger gate** *(per node)* \u2014 the startup guard that holds a node back until it has seen a full ledger from the network.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerSyncStateGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#network-ledger-gate)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 0, + "y": 62 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sync_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"network_ledger_gate\"}, \"series\", \"Gate Closed\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Network Ledger Gate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How long the node took, from process start, to reach the full server state for the first time.*\n\n###### How it's computed:\n*sync_state series initial_full_duration_us, converted from microseconds to seconds.*\n\n###### Reading it:\n*A value appears only once the node has actually synced. Zero (red) means it has never reached full \u2014 that is the signal, not missing data.*\n\n###### Healthy range:\n*Seconds to a few minutes on a warm node; longer on a fresh one that must acquire history.*\n\n###### Watch for:\n*A flat zero. The value never changes after the first full transition, so it either fills in or the node never synced.*\n\n###### Keywords:\n- **Time to first FULL** *(per node)* \u2014 elapsed time from process start until the node first reached the full server state.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerSyncStateGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#time-to-first-full)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 8, + "y": 62 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sync_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"initial_full_duration_us\"} / 1e6, \"series\", \"Time to FULL\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Time to First FULL", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How many seconds the server's main loop has currently been unresponsive.*\n\n###### How it's computed:\n*sync_state series server_stall_seconds, the same duration the load monitor logs as \"Server stalled for N seconds\".*\n\n###### Reading it:\n*0 (green) is healthy. Any non-zero value means the main loop missed its heartbeat for at least the 10 second reporting threshold.*\n\n###### Healthy range:\n*0.*\n\n###### Watch for:\n*Any sustained non-zero value. A stall points at main-loop overload rather than sync data starvation, so check job-queue depth and disk latency next, not peer supply.*\n\n###### Keywords:\n- **Server stall** *(per node)* \u2014 the main loop failing to check in with the load monitor, measured in seconds of unresponsiveness.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LoadManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/LoadManager.cpp)\n\n###### Function:\n`LoadManager::updateStallState`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#server-stall)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 16, + "y": 62 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sync_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"server_stall_seconds\"}, \"series\", \"Stalled\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Server Stall", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How far this node's validated ledger trails the highest ledger any connected peer reports holding.*\n\n###### How it's computed:\n*sync_state series ledgers_behind: the peer-reported network tip minus this node's validated sequence, floored at zero.*\n\n###### Reading it:\n*Trending to zero is healthy convergence. Flat or rising means the node is not catching up. Zero also covers \"no peer has reported a newer ledger\", which on a node with no peers is the same thing.*\n\n###### Healthy range:\n*0 to 1 on a synced node.*\n\n###### Watch for:\n*A plateau or a climb during initial sync: the node is acquiring slower than the network advances, so it will never converge. Correlate with the acquire and job-queue panels.*\n\n###### Keywords:\n- **Ledgers behind network** *(per node)* \u2014 the gap between the peer-reported network tip and this node's own validated ledger sequence.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerSyncStateGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledgers-behind-network)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Ledgers", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 50 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 74 + }, + "id": 15, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sync_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"ledgers_behind\"}, \"series\", \"Ledgers Behind\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Ledgers Behind Network", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How often the server enters a NEW stall episode, as opposed to how long one stall lasts.*\n\n###### How it's computed:\n*Per-second rate of server_stall_events_total, which counts once per stall episode rather than once per stalled second.*\n\n###### Reading it:\n*Flat at zero is healthy. Read it beside the Server Stall stat: a rising rate means repeated fresh stalls, while a flat rate with a large stall value means one long unresolved stall.*\n\n###### Healthy range:\n*0.*\n\n###### Watch for:\n*Any repeating rate. Recurring short stalls and one long stall have different causes, and this panel is what separates them.*\n\n###### Keywords:\n- **Server stall** *(per node)* \u2014 the main loop failing to check in with the load monitor, measured in seconds of unresponsiveness.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LoadManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/LoadManager.cpp)\n\n###### Function:\n`LoadManager::updateStallState`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#server-stall)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Episodes / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 74 + }, + "id": 16, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(server_stall_events_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Stall Episodes\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Server Stall Event Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Which edges of the sync state machine the node actually traversed over the dashboard window, counted per from-to pair.*\n\n###### How it's computed:\n*increase(state_changes_total) over the selected range, summed by the from and to labels.*\n\n###### Reading it:\n*A clean fresh sync shows single traversals along disconnected to connected to syncing to tracking to full. Repeated counts on the full-to-connected edge paired with connected-to-full is flapping.*\n\n###### Healthy range:\n*One traversal per climb edge and nothing on the reverse edges.*\n\n###### Watch for:\n*High counts on a reverse edge such as full to connected: the node reaches full and keeps losing it, which an unlabelled state-change total cannot distinguish from a clean climb.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 how fully the node is participating, in ascending order: disconnected, connected, syncing, tracking, full.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::setMode`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "${__field.labels.series}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 86 + }, + "id": 17, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(sum by (from, to) (increase(state_changes_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", from=~\"$mode_from\", to=~\"$mode_to\"}[$__range])), \"series\", \"$1\", \"from\", \"(.*)\")", + "refId": "A" + } + ], + "title": "Mode Transitions by Edge", + "type": "bargauge" } ], "schemaVersion": 39, @@ -1213,6 +1679,46 @@ "multi": true, "refresh": 2, "sort": 1 + }, + { + "name": "mode_from", + "label": "Mode From", + "description": "Filter mode transitions by the state being left [disconnected / connected / syncing / tracking / full]", + "type": "query", + "query": "label_values(state_changes_total, from)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "mode_to", + "label": "Mode To", + "description": "Filter mode transitions by the state being entered [disconnected / connected / syncing / tracking / full]", + "type": "query", + "query": "label_values(state_changes_total, to)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 } ] }, diff --git a/docker/telemetry/workload/expected_metrics.json b/docker/telemetry/workload/expected_metrics.json index 8f8bbb712a..6fb4d2ef81 100644 --- a/docker/telemetry/workload/expected_metrics.json +++ b/docker/telemetry/workload/expected_metrics.json @@ -138,9 +138,16 @@ "unl_fetch_total", "unl_quorum{metric=\"trusted_keys\"}", "unl_quorum{metric=\"quorum\"}", - "clock_close_offset_seconds{metric=\"offset\"}" + "clock_close_offset_seconds{metric=\"offset\"}", + "sync_state{metric=\"initial_full_duration_us\"}", + "sync_state{metric=\"network_ledger_gate\"}", + "sync_state{metric=\"server_stall_seconds\"}", + "sync_state{metric=\"ledgers_behind\"}", + "server_stall_events_total", + "state_changes_total{from!=\"\",to!=\"\"}" ], - "_conditional_note": "handshake_negotiation_fail_total and unl_fetch_total are conditional under the local harness: the first only exists once a handshake is rejected, and the second needs a [validator_list_sites] entry (run-full-validation.sh generates a static [validators] file instead). The validator has no per-metric optional flag, so if either reports 0 series in a harness run, move it out of this group rather than weakening the check." + "_conditional_note": "handshake_negotiation_fail_total and unl_fetch_total are conditional under the local harness: the first only exists once a handshake is rejected, and the second needs a [validator_list_sites] entry (run-full-validation.sh generates a static [validators] file instead). The validator has no per-metric optional flag, so if either reports 0 series in a harness run, move it out of this group rather than weakening the check.", + "_sync_state_note": "The four sync_state sub-series are unconditional: the gauge observes all four on every collection tick, so each is present as a series even when its value is 0 (a node that never reached FULL reports initial_full_duration_us=0, and a healthy node reports server_stall_seconds=0). The check asserts series presence, not a non-zero value, which is exactly right here — a zero is a meaningful reading for these signals, and absence is the regression. server_stall_events_total is likewise always present because the observable counter reports the tally (0 or more) every tick. state_changes_total is asserted here with a from!=\"\",to!=\"\" selector rather than bare (parity_counters already asserts the bare name): the selector is what proves the WP-A2 {from,to} label dimension actually reached Prometheus, so a regression to the old unlabelled counter fails this check instead of silently passing on the bare name. It needs at least one real mode transition, which any node reaching connected/syncing produces during startup." }, "grafana_dashboards": { "description": "All Grafana dashboards that must render data (UIDs as provisioned on disk under docker/telemetry/grafana/dashboards/).", diff --git a/docker/telemetry/workload/validate_telemetry.py b/docker/telemetry/workload/validate_telemetry.py index 959064d15c..9b7854d17b 100644 --- a/docker/telemetry/workload/validate_telemetry.py +++ b/docker/telemetry/workload/validate_telemetry.py @@ -590,6 +590,14 @@ async def validate_metrics( "handshake_", "unl_", "clock_close_offset", + # Sync-state signals. sync_state carries the gate, + # stall-seconds, ledgers-behind and time-to-first-FULL + # sub-series; state_changes_total is now labelled with + # the {from,to} transition edge, and + # server_stall_events_total is the stall episode count. + "sync_state", + "state_changes_total", + "server_stall_events", ) ) ] diff --git a/docs/telemetry-glossary.md b/docs/telemetry-glossary.md index 5ed921e2c2..6651bca073 100644 --- a/docs/telemetry-glossary.md +++ b/docs/telemetry-glossary.md @@ -555,15 +555,35 @@ Acquiring a ledger means requesting it and its contents from peers when the node **Scope:** per node — measured on and specific to this individual server. + + +### Ledgers behind network + +How many ledgers this node's validated sequence trails the network's. The network figure is the highest ledger sequence any connected peer reports holding, so the gap is what the node still has to close to reach the tip. Trending down to zero is healthy convergence; flat or rising means the node acquires slower than the network advances and will not converge on its own. Because the value is floored at zero and the target comes from peer reports, a node with no peers — or whose peers have not reported a range yet — also reads zero, so a zero is only "at the tip" once there are peers. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Time to first FULL](#time-to-first-full) · [Ledger acquire (inbound fetch)](#ledger-acquire-inbound-fetch) · [Validated ledger on xrpl.org](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) + + + +### Network ledger gate + +The startup guard that holds a node back until it has seen a complete ledger from the network. While the gate is closed the node refuses submitted transactions and cannot reach the full state, no matter how healthy the rest of the sync pipeline looks. It normally opens within the first minutes of startup; a gate that stays closed means the node never obtained a full network ledger, which is a peering or quorum fault upstream rather than a sync-pipeline one. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Operating mode / server state](#operating-mode-server-state) · [UNL quorum headroom](#unl-quorum-headroom) · [Operating mode / server state on xrpl.org](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) + ### Operating mode / server state -The server state describes how fully the node is participating, in ascending order: disconnected, connected, syncing, tracking, full (caught up), and for validators validating and proposing. A healthy non-validator sits in Full; frequent transitions out of Full indicate instability. +The server state describes how fully the node is participating, in ascending order: disconnected, connected, syncing, tracking, full (caught up), and for validators validating and proposing. A healthy non-validator sits in Full; frequent transitions out of Full indicate instability. Transitions are recorded as a from-to edge rather than a bare count, which is what distinguishes a clean one-way climb to Full from flapping in and out of it. **Scope:** per node — measured on and specific to this individual server. -**See also:** [Operating mode / server state on xrpl.org](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) +**See also:** [Time to first FULL](#time-to-first-full) · [Network ledger gate](#network-ledger-gate) · [Operating mode / server state on xrpl.org](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) @@ -575,6 +595,26 @@ The elapsed time of one outbound peer connection attempt, from starting the TCP **See also:** [DNS resolve](#dns-resolve) · [Handshake negotiation failure](#handshake-negotiation-failure) · [Peer protocol on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) + + +### Server stall + +The server's main loop failing to check in with the load monitor, measured as seconds of unresponsiveness. A stall means main-loop overload, not a shortage of sync data, so the cause is downstream work such as job-queue backlog or slow disk rather than peer supply. Two readings mean different things: a large duration with a flat episode count is one long unresolved stall, while a small duration with a rising episode count is repeated short stalls the server keeps recovering from. Episodes are counted once per stall, not once per stalled second, which is what keeps those two cases distinguishable. A stall that persists long enough is treated as unrecoverable and deliberately ends the process. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Ledgers behind network](#ledgers-behind-network) · [Consensus stall](#consensus-stall) + + + +### Time to first FULL + +The elapsed time from process start until the node first reached the full server state. It is a one-shot measurement: it is set on the first transition to full and never changes afterwards, so it has no trend to read. That leaves exactly two meaningful readings — a duration, meaning the node synced and this is how long it took, or zero, meaning it has never reached full at all. The zero is the diagnostic signal rather than absent data, and it is the starting point for working through the rest of the sync pipeline. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Operating mode / server state](#operating-mode-server-state) · [Network ledger gate](#network-ledger-gate) · [Ledgers behind network](#ledgers-behind-network) + ### UNL fetch outcome diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index fd4ce9a58a..3eb88f53b3 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -2157,6 +2157,63 @@ first one that is wrong and fix it before reading further panels. If all five steps are clean the bootstrap stage is healthy, and the problem is in the **Sync pipeline** row instead. +#### Sync pipeline — ordered diagnosis + +Once bootstrap is clean, work the Sync pipeline row in this order. As above, +each step gates the next: stop at the first one that is wrong. + +1. **Did the node ever sync at all?** + Panel _Time to First FULL_ (`sync_state`, `metric=initial_full_duration_us`, + shown in seconds). This is a one-shot measurement: it fills in the moment the + node first reaches `full` and never changes again. So there are exactly two + readings that matter — a value, meaning the node synced and this is how long + it took, or a flat zero (red), meaning it has **never** reached `full`. A + flat zero is the signal, not missing data; everything below explains why. + Do not read a rising or falling trend into this panel — it cannot have one. + +2. **Is the node gated on the network ledger?** + Panel _Network Ledger Gate_ (`sync_state`, `metric=network_ledger_gate`). A + persistent 1 (red) means the node has never seen a complete ledger from the + network, so it refuses submitted transactions and cannot reach `full` however + healthy the acquire pipeline looks. This normally clears within the first + minutes of startup; a 1 that never clears sends you back to the Bootstrap row + (no peers, or no quorum) rather than deeper into the pipeline. + +3. **Is the server stalling rather than starving?** + Panels _Server Stall_ (`sync_state`, `metric=server_stall_seconds`) and + _Server Stall Event Rate_ (`server_stall_events_total`). A non-zero stall + means the main loop missed its heartbeat for at least 10 seconds, which is + main-loop **overload**, not sync data starvation — so the fix is in job-queue + depth and disk latency, not peer supply. Read the two panels together, as + they separate two different faults that look identical in a log: + - Large stall seconds with a **flat** event rate — one long unresolved + stall. Note that past 600 seconds the server deliberately fails with a + logic error, so a stall approaching that is about to end the process. + - Small stall seconds with a **rising** event rate — repeated short stalls; + the server keeps recovering and re-stalling, which points at periodic work + (sweeps, large writes) rather than a single stuck operation. + +4. **How far behind is it, and is it converging?** + Panel _Ledgers Behind Network_ (`sync_state`, `metric=ledgers_behind`). The + target is the highest ledger any connected peer reports holding, so this is + the gap the node must close. Trending down to 0 is healthy convergence; flat + or rising means the node acquires slower than the network advances and will + never converge on its own. One caveat when reading a zero: the value is + floored at 0 and the target comes from peer reports, so a node with no peers + — or whose peers have reported no range yet — also reads 0. Confirm against + the peer count before treating a 0 here as "at the tip". + +5. **Which state edges is it actually taking?** + Panel _Mode Transitions by Edge_ (`state_changes_total`, grouped by `from` + and `to`). A clean fresh sync traverses each climb edge + (`disconnected`→`connected`→`syncing`→`tracking`→`full`) roughly once. + Repeated counts on a reverse edge such as `full`→`connected`, paired with + `connected`→`full`, is flapping: the node keeps reaching `full` and losing + it. Flapping with a healthy step 4 points back at step 3 (stalls) or at the + Bootstrap row's clock and quorum panels, since those are what drop a node out + of `full` once it has arrived. Use the _Mode From_ and _Mode To_ template + variables to isolate one edge. + ## Performance Tuning | Scenario | Recommendation | diff --git a/include/xrpl/server/NetworkOPs.h b/include/xrpl/server/NetworkOPs.h index 4839b5f1f3..22e9d00776 100644 --- a/include/xrpl/server/NetworkOPs.h +++ b/include/xrpl/server/NetworkOPs.h @@ -111,10 +111,37 @@ public: */ [[nodiscard]] virtual std::chrono::microseconds getServerStateDurationUs() const = 0; + /** + * Microseconds from process start until the node first reached FULL. + * + * Zero while the node has not completed initial sync yet, so a value that + * stays at zero is itself the signal that sync never finished. Once set it + * never changes: this is the one-shot time-to-first-FULL, not the time in + * the current state. Same value as `initial_sync_duration_us` in + * server_info, exposed as a lightweight accessor so metrics can read it + * without building the full server_info JSON on every collection tick. + * + * @return Microseconds to first FULL, or 0 if not yet reached. + */ + [[nodiscard]] virtual std::uint64_t + getInitialSyncDurationUs() const = 0; [[nodiscard]] virtual std::string strOperatingMode(OperatingMode const mode, bool const admin = false) const = 0; [[nodiscard]] virtual std::string strOperatingMode(bool const admin = false) const = 0; + /** + * How far this node's validated ledger trails the network's. + * + * The network target is the highest ledger sequence any connected peer + * reports holding; the result is that target minus our own validated + * sequence, floored at zero. Zero therefore means either "at the tip" or + * "no peer has told us about a newer ledger", which on a node with no + * peers is the same thing. + * + * @return Ledgers behind the network, 0 when at or ahead of the tip. + */ + [[nodiscard]] virtual std::uint32_t + getLedgersBehindNetwork() const = 0; //-------------------------------------------------------------------------- // diff --git a/src/tests/libxrpl/telemetry/MetricMacros.cpp b/src/tests/libxrpl/telemetry/MetricMacros.cpp index 9c92deb22e..b617c0c41a 100644 --- a/src/tests/libxrpl/telemetry/MetricMacros.cpp +++ b/src/tests/libxrpl/telemetry/MetricMacros.cpp @@ -1017,4 +1017,233 @@ TEST(MetricMacros, sync_diagnostics_metrics_emit_nothing_when_registry_disabled) EXPECT_EQ(app.registry().meterCalls(), 0); } +// ----------------------------------------------------------------- +// Sync-state diagnostics (WP-A2). +// +// Asserts the EXACT values and label shapes of the five sync-state signals: +// state_changes_total{from,to} NetworkOPsImp::setMode +// sync_state{metric} MetricsRegistry::registerSyncStateGauge +// initial_full_duration_us +// network_ledger_gate +// server_stall_seconds +// ledgers_behind +// server_stall_events_total MetricsRegistry::registerStallEventsCounter +// +// The counter goes through the same macro production uses. The two observable +// instruments are registered directly on the SDK meter, mirroring the +// production callback shape, because the real MetricsRegistry's enabled path +// cannot be linked into this standalone binary (see the file header). +// ----------------------------------------------------------------- + +// state_changes_total is keyed on the (from, to) PAIR, so a transition edge is +// its own series. This is the whole point of the label: an unlabelled total +// cannot tell a clean tracking->connected->full climb from full->connected +// flapping, because both produce the same count. +TEST(MetricMacros, state_changes_total_keys_series_on_from_to_pair) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + // Mirrors the production call site: one macro invocation, the label values + // supplied by strOperatingMode() on the previous and new mode. + auto const transition = [&app](char const* from, char const* to) { + XRPL_METRIC_COUNTER_INC_LABELED( + app, + "state_changes_total", + "Total operating mode changes", + {{"from", std::string(from)}, {"to", std::string(to)}}); + }; + + // A clean climb: disconnected -> connected -> syncing -> full, once each. + transition("disconnected", "connected"); + transition("connected", "syncing"); + transition("syncing", "full"); + // Then flapping: full -> connected twice more, and connected -> full twice. + transition("full", "connected"); + transition("full", "connected"); + transition("connected", "full"); + transition("connected", "full"); + + auto const data = provider.collect(); + + // Six distinct (from, to) pairs -> exactly six series. + ASSERT_EQ(data.at("state_changes_total").size(), 6u); + + // The climb edges, each traversed exactly once. + EXPECT_EQ( + counterValue(data, "state_changes_total", attrs("from", "disconnected", "to", "connected")), + 1); + EXPECT_EQ( + counterValue(data, "state_changes_total", attrs("from", "connected", "to", "syncing")), 1); + EXPECT_EQ(counterValue(data, "state_changes_total", attrs("from", "syncing", "to", "full")), 1); + + // The flap edges carry their own exact counts and do not merge into the + // climb edges above: full->connected is 2, not folded into connected->full. + EXPECT_EQ( + counterValue(data, "state_changes_total", attrs("from", "full", "to", "connected")), 2); + EXPECT_EQ( + counterValue(data, "state_changes_total", attrs("from", "connected", "to", "full")), 2); + + // Direction matters: connected->syncing exists, syncing->connected does not, + // proving the pair is ordered rather than an unordered edge set. + EXPECT_EQ( + data.at("state_changes_total").count(attrs("from", "syncing", "to", "connected")), 0u); + + // NEGATIVE: a mode pair never emitted has no series at all. + EXPECT_EQ(data.at("state_changes_total").count(attrs("from", "tracking", "to", "full")), 0u); + + // Every series key carries exactly the two expected label names and nothing + // else -- no stray dimension inflating the cardinality. + for (auto const& [labels, point] : data.at("state_changes_total")) + { + ASSERT_EQ(labels.size(), 2u); + EXPECT_EQ(labels.count("from"), 1u); + EXPECT_EQ(labels.count("to"), 1u); + } +} + +// sync_state fans four independent signals out of ONE callback under the +// `metric` label, mirroring MetricsRegistry::registerSyncStateGauge(). The +// values chosen are the diagnostically interesting combination: never reached +// FULL (0 duration) while the gate is still closed, the loop is stalled, and +// the node trails the network. +TEST(MetricMacros, sync_state_gauge_observes_exact_stuck_node_values) +{ + CollectingProvider const provider; + + // The live values the callback reports, owned by the test exactly as the + // real registry reads them from NetworkOPs/LoadManager on each tick. + struct Observed + { + std::int64_t initialFullDurationUs; + std::int64_t networkLedgerGate; + std::int64_t serverStallSeconds; + std::int64_t ledgersBehind; + }; + // A node that never synced: no FULL yet, gate closed, 42 s stalled, 150 + // ledgers behind (network tip 250 vs our validated 100). + Observed observed{0, 1, 42, 150}; + + // Keep the instrument alive for the whole test: destroying the handle + // deregisters the callback, which is why the real registry holds a member. + auto gauge = + provider.meter()->CreateInt64ObservableGauge("sync_state", "Sync-pipeline health signals"); + gauge->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto const* self = static_cast(state); + // Same Observe() form the production callback uses. + auto observe = [&](char const* name, std::int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + observe("initial_full_duration_us", self->initialFullDurationUs); + observe("network_ledger_gate", self->networkLedgerGate); + observe("server_stall_seconds", self->serverStallSeconds); + observe("ledgers_behind", self->ledgersBehind); + }, + &observed); + + auto const data = provider.collect(); + + // Exactly four series, one per `metric` value -- the four signals must not + // collapse into a single series. + ASSERT_EQ(data.at("sync_state").size(), 4u); + + // Zero is a REAL observed value here, not a missing series: it is the + // "never reached FULL" signal, so the series must exist and read 0. + ASSERT_EQ(data.at("sync_state").count(attrs("metric", "initial_full_duration_us")), 1u); + EXPECT_EQ(gaugeValue(data, "sync_state", attrs("metric", "initial_full_duration_us")), 0); + + EXPECT_EQ(gaugeValue(data, "sync_state", attrs("metric", "network_ledger_gate")), 1); + EXPECT_EQ(gaugeValue(data, "sync_state", attrs("metric", "server_stall_seconds")), 42); + EXPECT_EQ(gaugeValue(data, "sync_state", attrs("metric", "ledgers_behind")), 150); + + // The label key is exactly "metric" and it is the only label present. + auto const& firstKey = data.at("sync_state").begin()->first; + ASSERT_EQ(firstKey.size(), 1u); + EXPECT_EQ(firstKey.begin()->first, "metric"); + + // NEGATIVE: the stall EPISODE count is deliberately NOT a sync_state + // series -- it is a separate cumulative instrument, so querying it here + // must find nothing. + EXPECT_EQ(data.at("sync_state").count(attrs("metric", "server_stall_events")), 0u); + + // A healthy node reports the complementary values through the same + // callback: synced in 12.5 s, gate open, no stall, at the tip. + observed = Observed{12'500'000, 0, 0, 0}; + auto const healthy = provider.collect(); + EXPECT_EQ( + gaugeValue(healthy, "sync_state", attrs("metric", "initial_full_duration_us")), 12'500'000); + EXPECT_EQ(gaugeValue(healthy, "sync_state", attrs("metric", "network_ledger_gate")), 0); + EXPECT_EQ(gaugeValue(healthy, "sync_state", attrs("metric", "server_stall_seconds")), 0); + EXPECT_EQ(gaugeValue(healthy, "sync_state", attrs("metric", "ledgers_behind")), 0); +} + +// server_stall_events_total is a cumulative ObservableCounter, not a gauge +// series. It must aggregate as a Sum (so rate() is meaningful) and be +// unlabelled, mirroring MetricsRegistry::registerStallEventsCounter(). +TEST(MetricMacros, stall_events_counter_observes_exact_cumulative_count) +{ + CollectingProvider const provider; + + // Three stall episodes reported so far by the load-monitor thread. + std::int64_t stallEpisodes = 3; + + auto counter = provider.meter()->CreateInt64ObservableCounter( + "server_stall_events_total", "Total server main-loop stall episodes"); + counter->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto const* value = static_cast(state); + // Production observes this with NO labels. + opentelemetry::nostd::get>>(result) + ->Observe(*value); + }, + &stallEpisodes); + + auto const data = provider.collect(); + + // Exactly one unlabelled series, read through counterValue() -- which + // unwraps a SumPointData, so this also proves the instrument aggregates as + // a counter and not as a last-value gauge. + ASSERT_EQ(data.at("server_stall_events_total").size(), 1u); + EXPECT_TRUE(data.at("server_stall_events_total").begin()->first.empty()); + EXPECT_EQ(counterValue(data, "server_stall_events_total", otel_sdk::PointAttributes{}), 3); + + // Monotonic: a later collection sees the higher total, not a delta. + stallEpisodes = 5; + EXPECT_EQ( + counterValue(provider.collect(), "server_stall_events_total", otel_sdk::PointAttributes{}), + 5); +} + +// RUNTIME-DISABLED no-op proof for the counter half of WP-A2: with the registry +// disabled, the setMode call site emits NOTHING -- no series for +// state_changes_total, and meter() is never consulted, so not even an +// instrument was created. +TEST(MetricMacros, state_changes_total_emits_nothing_when_registry_disabled) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/false, provider.meter()); + + XRPL_METRIC_COUNTER_INC_LABELED( + app, + "state_changes_total", + "Total operating mode changes", + {{"from", std::string("connected")}, {"to", std::string("full")}}); + + auto const data = provider.collect(); + + // Total absence, not a zero-valued series. + EXPECT_EQ(data.count("state_changes_total"), 0u); + EXPECT_EQ(data.size(), 0u); + + // Cause, not just state: the isEnabled() gate short-circuited before the + // macro asked for a meter. + EXPECT_EQ(app.registry().meterCalls(), 0); +} + #endif // XRPL_ENABLE_TELEMETRY diff --git a/src/tests/libxrpl/telemetry/MetricsRegistry.cpp b/src/tests/libxrpl/telemetry/MetricsRegistry.cpp index 36c9e44b0f..c910d1718a 100644 --- a/src/tests/libxrpl/telemetry/MetricsRegistry.cpp +++ b/src/tests/libxrpl/telemetry/MetricsRegistry.cpp @@ -17,14 +17,16 @@ * so the tests are compiled out. * * CONSEQUENCE for the sync-diagnostics gauges (`unl_quorum`, - * `clock_close_offset_seconds`): this file CANNOT assert an observed gauge + * `clock_close_offset_seconds`, `sync_state`, + * `server_stall_events_total`): this file CANNOT assert an observed gauge * value, because on this build the gauges do not exist -- their registration * methods and the OTel instrument members are inside * `#ifdef XRPL_ENABLE_TELEMETRY`, and there is no MeterProvider at all. What * is provable here, and what the tests below assert, is the complementary * half: that nothing is registered and no service is consulted. The exact - * observed values (trusted_keys=5, quorum=4, offset=-3) are asserted in - * MetricMacros.cpp, which is the file compiled when telemetry IS enabled. + * observed values (trusted_keys=5, quorum=4, offset=-3, and the sync_state / + * stall-episode values) are asserted in MetricMacros.cpp, which is the file + * compiled when telemetry IS enabled. */ // When telemetry is globally enabled, MetricsRegistry.cpp requires xrpld @@ -375,11 +377,12 @@ TEST_F(MetricsRegistryTest, destructor_calls_stop) // Sync-diagnostics gauges: compile-time-disabled proof. // // `unl_quorum` reads ValidatorList::trustedKeyCount() and quorum(); -// `clock_close_offset_seconds` reads TimeKeeper::closeOffset(). Both are +// `clock_close_offset_seconds` reads TimeKeeper::closeOffset(); `sync_state` and +// `server_stall_events_total` read NetworkOPs and LoadManager. All are // reached through the ServiceRegistry, and MockServiceRegistry::getValidators() -// / getTimeKeeper() THROW std::logic_error. So "no gauge callback ran" is -// directly observable here: had registerAsyncGauges() run and had a callback -// fired, one of those accessors would have thrown. +// / getTimeKeeper() / getOPs() / getLoadManager() THROW std::logic_error. So "no +// gauge callback ran" is directly observable here: had registerAsyncGauges() run +// and had a callback fired, one of those accessors would have thrown. // // Honest scope note: these tests do NOT assert an observed gauge value. On this // build the gauges are not compiled at all (see the file header), so there is no @@ -415,7 +418,8 @@ TEST_F(MetricsRegistryTest, disabled_lifecycle_never_consults_gauge_services) telemetry::MetricsRegistry registry(false, mockApp_, j_); // start() is where registerAsyncGauges() -- and with it - // registerUnlQuorumGauge() / registerClockSkewGauge() -- would run. + // registerUnlQuorumGauge() / registerClockSkewGauge() / + // registerSyncStateGauge() / registerStallEventsCounter() -- would run. EXPECT_NO_THROW(registry.start("http://localhost:4318/v1/metrics")); // detachCallbacks() is the shutdown hook the real gauges honour. It must be @@ -433,6 +437,12 @@ TEST_F(MetricsRegistryTest, disabled_lifecycle_never_consults_gauge_services) // could mean the mock is permissive rather than that no callback ran. EXPECT_THROW(mockApp_.getValidators(), std::logic_error); EXPECT_THROW(mockApp_.getTimeKeeper(), std::logic_error); + // The two services the WP-A2 sync-state signals read. sync_state needs both + // (NetworkOPs for the gate/duration/ledgers-behind, LoadManager for stall + // seconds) and server_stall_events_total needs the second, so either one + // firing would have thrown above. + EXPECT_THROW(mockApp_.getOPs(), std::logic_error); + EXPECT_THROW(mockApp_.getLoadManager(), std::logic_error); } // Even asking for enabled=true registers no sync-diagnostics gauge on a @@ -450,8 +460,10 @@ TEST_F(MetricsRegistryTest, enabled_flag_alone_registers_no_gauges_when_compiled EXPECT_TRUE(enabledRequest.isEnabled()); // Yet the whole lifecycle stays inert. If registerAsyncGauges() had run and - // registered registerUnlQuorumGauge()/registerClockSkewGauge(), a callback - // would reach getValidators()/getTimeKeeper() and throw std::logic_error. + // registered registerUnlQuorumGauge()/registerClockSkewGauge()/ + // registerSyncStateGauge()/registerStallEventsCounter(), a callback would + // reach getValidators()/getTimeKeeper()/getOPs()/getLoadManager() and throw + // std::logic_error. EXPECT_NO_THROW(enabledRequest.start("http://localhost:4318/v1/metrics")); EXPECT_NO_THROW(enabledRequest.detachCallbacks()); EXPECT_NO_THROW(enabledRequest.stop()); @@ -461,4 +473,27 @@ TEST_F(MetricsRegistryTest, enabled_flag_alone_registers_no_gauges_when_compiled EXPECT_FALSE(disabledRequest.isEnabled()); } +// The `state_changes_total` counter no longer has a registry-owned wrapper +// method: WP-A2 moved it to a labelled call-site macro in +// NetworkOPsImp::setMode so it can carry {from,to}. This compile-time +// assertion is the regression guard -- if someone reintroduces +// incrementStateChanges(), the unlabelled instrument would coexist with the +// labelled one and Prometheus would carry two conflicting versions of the same +// metric name. +TEST_F(MetricsRegistryTest, state_changes_counter_has_no_registry_wrapper) +{ + auto hasIncrementStateChanges = [](T* r) { + return requires { r->incrementStateChanges(); }; + }; + EXPECT_FALSE(hasIncrementStateChanges(static_cast(nullptr))); + + // Positive control: a sibling parity counter that WAS deliberately kept as + // a registry wrapper is still detectable, so the trait above is really + // probing for the method and not vacuously false. + auto hasIncrementLedgersClosed = [](T* r) { + return requires { r->incrementLedgersClosed(); }; + }; + EXPECT_TRUE(hasIncrementLedgersClosed(static_cast(nullptr))); +} + #endif // !XRPL_ENABLE_TELEMETRY diff --git a/src/tests/libxrpl/telemetry/SyncStateSignals.cpp b/src/tests/libxrpl/telemetry/SyncStateSignals.cpp new file mode 100644 index 0000000000..713ac9f132 --- /dev/null +++ b/src/tests/libxrpl/telemetry/SyncStateSignals.cpp @@ -0,0 +1,141 @@ +/** + * @file SyncStateSignals.cpp + * Unit tests for the decision rule behind the sync-state stall signals. + * + * `sync_state{metric="server_stall_seconds"}` and + * `server_stall_events_total` are both derived from one rule: + * LoadManager::evaluateStall(). The registry callbacks that export them only + * read atomics, so the rule is the only place a defect can hide -- an + * off-by-one on the threshold, or an episode counted once per tick instead of + * once per stall, would silently misreport every stall. + * + * The rule is a pure `static constexpr` member, so these tests assert it + * directly: no LoadManager instance, no monitor thread, no sleeping, and no + * test-only mutator added to production code to make it reachable. + * + * Compiled only when XRPL_ENABLE_TELEMETRY is defined, because that is the + * configuration in which the test target has `src/` on its include path and can + * therefore reach . The rule itself is not + * telemetry-conditional; only this file's ability to include the header is. + */ + +#ifdef XRPL_ENABLE_TELEMETRY + +#include + +#include + +#include +#include + +using namespace xrpl; +using namespace std::chrono_literals; + +namespace { + +/** + * The threshold production uses (LoadManager::run's kReportingIntervalSeconds). + */ +constexpr auto kThreshold = 10s; + +} // namespace + +// Below the threshold nothing is reported: a brief scheduling hiccup is not a +// stall, so both outputs stay at their healthy values. Asserts the boundary +// exactly at threshold-1, not merely "some small value". +TEST(SyncStateSignals, sub_threshold_stall_reports_healthy) +{ + auto const zero = LoadManager::evaluateStall(0, 0s, kThreshold); + EXPECT_EQ(zero.seconds, 0U); + EXPECT_FALSE(zero.newEpisode); + + // One second below the threshold is still healthy. + auto const justUnder = LoadManager::evaluateStall(0, 9s, kThreshold); + EXPECT_EQ(justUnder.seconds, 0U); + EXPECT_FALSE(justUnder.newEpisode); +} + +// The threshold is inclusive: exactly 10 s is reportable and starts an episode. +// This is the boundary the log line uses, so gauge and log must agree here. +TEST(SyncStateSignals, threshold_is_inclusive_and_starts_an_episode) +{ + auto const atThreshold = LoadManager::evaluateStall(0, kThreshold, kThreshold); + EXPECT_EQ(atThreshold.seconds, 10U); + EXPECT_TRUE(atThreshold.newEpisode); +} + +// One continuous stall spanning many ticks is ONE episode. The seconds value +// tracks the growing duration while newEpisode stays false after the first +// tick -- this is what makes "one long stall" distinguishable from "repeated +// short stalls" on the dashboard. +TEST(SyncStateSignals, continuous_stall_counts_exactly_one_episode) +{ + // Tick 1: crosses the threshold. + auto const first = LoadManager::evaluateStall(0, 10s, kThreshold); + EXPECT_EQ(first.seconds, 10U); + EXPECT_TRUE(first.newEpisode); + + // Ticks 2..4: still stalled, longer each time, but the SAME episode. + auto const second = LoadManager::evaluateStall(first.seconds, 11s, kThreshold); + EXPECT_EQ(second.seconds, 11U); + EXPECT_FALSE(second.newEpisode); + + auto const third = LoadManager::evaluateStall(second.seconds, 90s, kThreshold); + EXPECT_EQ(third.seconds, 90U); + EXPECT_FALSE(third.newEpisode); + + auto const fourth = LoadManager::evaluateStall(third.seconds, 600s, kThreshold); + EXPECT_EQ(fourth.seconds, 600U); + EXPECT_FALSE(fourth.newEpisode); +} + +// Recovery clears the seconds, and a LATER stall is a NEW episode. Without the +// healthy tick in between this would be indistinguishable from the continuous +// case above, so this is the test that pins the transition semantics. +TEST(SyncStateSignals, stall_after_recovery_is_a_new_episode) +{ + // Stalled, then recovered: seconds drop back to 0 and no episode starts on + // the recovery tick itself. + auto const stalled = LoadManager::evaluateStall(0, 30s, kThreshold); + EXPECT_EQ(stalled.seconds, 30U); + EXPECT_TRUE(stalled.newEpisode); + + auto const recovered = LoadManager::evaluateStall(stalled.seconds, 0s, kThreshold); + EXPECT_EQ(recovered.seconds, 0U); + EXPECT_FALSE(recovered.newEpisode); + + // Stalling again after recovery: a second, distinct episode. + auto const again = LoadManager::evaluateStall(recovered.seconds, 15s, kThreshold); + EXPECT_EQ(again.seconds, 15U); + EXPECT_TRUE(again.newEpisode); +} + +// A stall that decays to a sub-threshold value counts as recovered, so the next +// crossing is a new episode. Edge case: the previous value was non-zero but +// below the threshold, which must be treated as healthy, not as "still stalled". +TEST(SyncStateSignals, decay_below_threshold_is_treated_as_recovered) +{ + // Contrived previous value: below threshold, so it reads as healthy even + // though it is non-zero. (Production never publishes such a value -- it + // stores 0 when healthy -- so this pins the rule, not just the caller.) + auto const crossing = LoadManager::evaluateStall(9, kThreshold, kThreshold); + EXPECT_EQ(crossing.seconds, 10U); + EXPECT_TRUE(crossing.newEpisode); +} + +// Compile-time proof that the rule is genuinely constexpr and side-effect free: +// if it ever grows state or a runtime-only dependency, these fail the build +// rather than the run. +TEST(SyncStateSignals, rule_is_evaluated_at_compile_time) +{ + static_assert(LoadManager::evaluateStall(0, 10s, 10s).newEpisode); + static_assert(LoadManager::evaluateStall(0, 10s, 10s).seconds == 10U); + static_assert(!LoadManager::evaluateStall(0, 9s, 10s).newEpisode); + static_assert(LoadManager::evaluateStall(0, 9s, 10s).seconds == 0U); + static_assert(!LoadManager::evaluateStall(10, 20s, 10s).newEpisode); + + // Keeps the test body non-empty for readers scanning for an assertion. + EXPECT_TRUE(LoadManager::evaluateStall(0, 10s, 10s).newEpisode); +} + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/src/xrpld/app/main/LoadManager.cpp b/src/xrpld/app/main/LoadManager.cpp index 36f82e392b..2859bb5b8b 100644 --- a/src/xrpld/app/main/LoadManager.cpp +++ b/src/xrpld/app/main/LoadManager.cpp @@ -11,7 +11,9 @@ #include #include +#include #include +#include #include #include #include @@ -115,6 +117,13 @@ LoadManager::run() static constexpr auto kStallFatalLogMessageTimeLimit = 90s; static constexpr auto kStallLogicErrorTimeLimit = 600s; + // Publish the stall state for telemetry before acting on it, so the + // gauge still sees the final duration on the tick that logicErrors. + // An unarmed detector reports healthy: its heartbeat is not yet + // meaningful, so a large elapsed time there is not a stall. + updateStallState( + armed ? timeSpentStalled : 0s, std::chrono::seconds(kReportingIntervalSeconds)); + if (armed && (timeSpentStalled >= kReportingIntervalSeconds)) { // Report the stalled condition every reportingIntervalSeconds @@ -171,6 +180,23 @@ LoadManager::run() } } +void +LoadManager::updateStallState( + std::chrono::seconds const stalled, + std::chrono::seconds const reportThreshold) +{ + // Read the previous tick's value before overwriting it: the healthy -> + // reportable transition is what defines a new episode. Only the monitor + // thread writes these, so the read-then-write needs no atomicity as a pair. + auto const state = evaluateStall( + currentStallSeconds_.load(std::memory_order_relaxed), stalled, reportThreshold); + + currentStallSeconds_.store(state.seconds, std::memory_order_relaxed); + + if (state.newEpisode) + stallEventCount_.fetch_add(1, std::memory_order_relaxed); +} + //------------------------------------------------------------------------------ std::unique_ptr diff --git a/src/xrpld/app/main/LoadManager.h b/src/xrpld/app/main/LoadManager.h index 5d3f07e996..db34b40309 100644 --- a/src/xrpld/app/main/LoadManager.h +++ b/src/xrpld/app/main/LoadManager.h @@ -2,8 +2,10 @@ #include +#include #include #include +#include #include #include #include @@ -23,6 +25,25 @@ class Application; * * The warning system is used instead of merely dropping, because hostile * peers can just reconnect anyway. + * + * Besides warning peers, the monitor thread is the only place that knows how + * long the server's main loop has been unresponsive. That duration used to + * exist only inside a log line; it is now also published through the two + * stall accessors below so telemetry can chart it. + * + * +-------------+ heartbeat() +--------------------+ + * | main loop |--------------->| LoadManager | + * +-------------+ | (monitor thread) | + * +---------+----------+ + * stall state | | fee changes + * v v + * currentStallSeconds_ LoadFeeTrack + * stallEventCount_ + * ^ + * | getCurrentStallSeconds() + * | getStallEventCount() + * telemetry::MetricsRegistry + * (sync_state gauge callback) */ class LoadManager { @@ -66,6 +87,43 @@ public: void heartbeat(); + /** + * Seconds the server's main loop has currently been unresponsive. + * + * Zero means healthy: either the heartbeat is current or the stall + * detector is not armed yet. A non-zero value is the same duration the + * monitor thread logs as "Server stalled for N seconds", refreshed on + * its one-second tick. + * + * @return Current stall duration in seconds, 0 when not stalled. + * + * @note Safe to call from any thread, including a telemetry + * observable-gauge callback: one relaxed atomic load, no lock. + */ + [[nodiscard]] std::uint32_t + getCurrentStallSeconds() const + { + return currentStallSeconds_.load(std::memory_order_relaxed); + } + + /** + * Number of distinct stall episodes seen since process start. + * + * Counts once per episode, on the tick the stall is first reported, not + * once per second of stalling. So a rising count means new stalls keep + * happening, which is a different fault from one long stall (that shows + * up as a large getCurrentStallSeconds() with a flat count). + * + * @return Monotonic stall-episode count. + * + * @note Safe to call from any thread; one relaxed atomic load. + */ + [[nodiscard]] std::uint64_t + getStallEventCount() const + { + return stallEventCount_.load(std::memory_order_relaxed); + } + //-------------------------------------------------------------------------- void @@ -74,10 +132,87 @@ public: void stop(); + /** + * What one monitor tick concludes about the stall state. + * + * @see evaluateStall + */ + struct StallState + { + /** + * Stall seconds to publish for this tick; 0 when healthy. + */ + std::uint32_t seconds; + /** + * True only on the tick that begins a new stall episode. + */ + bool newEpisode; + }; + + /** + * Decide the stall state for one monitor tick. + * + * A stall counts as reportable at the same threshold the monitor's log line + * uses, so the gauge and the log never disagree about whether the server is + * stalled. Below the threshold the tick reports healthy (0 seconds), which + * is why a brief scheduling hiccup does not register as a stall. + * + * An episode begins on the healthy -> reportable transition only, so one + * continuous stall increments the episode count exactly once no matter how + * many ticks it spans. That is what lets an operator tell one long stall + * (large `seconds`, flat count) from repeated short ones (rising count). + * + * Pure and side-effect free: it is the whole decision rule for the stall + * signals, kept separate from the atomics it feeds so the rule can be + * asserted directly without a test-only mutator on LoadManager. + * + * @param previousSeconds Value published by the previous tick. + * @param stalled Stall duration measured on this tick. + * @param reportThreshold Duration at which a stall becomes reportable. + * @return The seconds to publish and whether a new episode started. + * + * Example -- a stall crossing the threshold, then persisting: + * @code + * // First tick over the 10 s threshold: publishes 10, starts an episode. + * auto first = LoadManager::evaluateStall(0, 10s, 10s); // {10, true} + * // Still stalled 20 s later: publishes 20, but the SAME episode. + * auto next = LoadManager::evaluateStall(10, 30s, 10s); // {30, false} + * @endcode + * + * Example -- edge case: a sub-threshold blip never becomes an episode. + * @code + * auto blip = LoadManager::evaluateStall(0, 9s, 10s); // {0, false} + * @endcode + */ + [[nodiscard]] static constexpr StallState + evaluateStall( + std::uint32_t const previousSeconds, + std::chrono::seconds const stalled, + std::chrono::seconds const reportThreshold) noexcept + { + bool const reportable = stalled >= reportThreshold; + bool const wasReportable = previousSeconds >= reportThreshold.count(); + return StallState{ + .seconds = reportable ? static_cast(stalled.count()) : 0U, + .newEpisode = reportable && !wasReportable}; + } + private: void run(); + /** + * Publish the stall state read by the telemetry gauge. + * + * Applies evaluateStall() to this tick and stores the result. Called once + * per monitor tick, only from the monitor thread. + * + * @param stalled Stall duration measured on this tick. + * @param reportThreshold Duration at which a stall becomes reportable. + */ + void + updateStallState(std::chrono::seconds stalled, std::chrono::seconds reportThreshold); + private: Application& app_; beast::Journal const journal_; @@ -91,6 +226,19 @@ private: std::chrono::steady_clock::time_point lastHeartbeat_; bool armed_; + /** + * Seconds the main loop has been unresponsive as of the last monitor + * tick; 0 when healthy. Written only by the monitor thread, read by + * telemetry, hence atomic rather than mutex-guarded. + */ + std::atomic currentStallSeconds_{0}; + + /** + * Monotonic count of stall episodes since process start. Written only by + * the monitor thread; never reset. + */ + std::atomic stallEventCount_{0}; + friend std::unique_ptr makeLoadManager(Application& app, beast::Journal journal); }; diff --git a/src/xrpld/app/misc/NetworkOPs.cpp b/src/xrpld/app/misc/NetworkOPs.cpp index b3a2e75c4a..dea39bc36b 100644 --- a/src/xrpld/app/misc/NetworkOPs.cpp +++ b/src/xrpld/app/misc/NetworkOPs.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -292,6 +293,21 @@ class NetworkOPsImp final : public NetworkOPs return std::chrono::duration_cast( std::chrono::steady_clock::now() - start_); } + + /** + * Microseconds from process start to the first FULL transition. This + * is the same quantity reported as `initial_sync_duration_us` in + * json(); reading it alone avoids copying the whole counter array. + * Thread-safe. + * + * @return Time to first FULL, or 0 if FULL was never reached. + */ + std::uint64_t + initialSyncDurationUs() const + { + std::scoped_lock const lock(mutex_); + return initialSyncUs_; + } }; /** @@ -379,12 +395,18 @@ public: std::chrono::microseconds getServerStateDurationUs() const override; + std::uint64_t + getInitialSyncDurationUs() const override; + std::string strOperatingMode(OperatingMode const mode, bool const admin) const override; std::string strOperatingMode(bool const admin = false) const override; + std::uint32_t + getLedgersBehindNetwork() const override; + // // Transaction operations. // @@ -1010,6 +1032,33 @@ NetworkOPsImp::getServerStateDurationUs() const return accounting_.currentStateDurationUs(); } +std::uint64_t +NetworkOPsImp::getInitialSyncDurationUs() const +{ + return accounting_.initialSyncDurationUs(); +} + +std::uint32_t +NetworkOPsImp::getLedgersBehindNetwork() const +{ + // The network tip is the highest ledger sequence any connected peer says it + // holds. Peers report their range in mtSTATUS_CHANGE, which PeerImp caches, + // so this is a read of already-received data: no new network round trip. + std::uint32_t networkTarget = 0; + registry_.get().getOverlay().foreach([&networkTarget](std::shared_ptr const& peer) { + std::uint32_t minSeq = 0; + std::uint32_t maxSeq = 0; + peer->ledgerRange(minSeq, maxSeq); + networkTarget = std::max(networkTarget, maxSeq); + }); + + auto const validated = registry_.get().getLedgerMaster().getValidLedgerIndex(); + + // Floor at zero: we can legitimately be ahead of every peer's reported + // range, and a peer that has reported nothing yet leaves the target at 0. + return networkTarget > validated ? networkTarget - validated : 0; +} + inline std::string NetworkOPsImp::strOperatingMode(bool const admin /* = false */) const { @@ -2661,13 +2710,27 @@ NetworkOPsImp::setMode(OperatingMode om) if (mode_ == om) return; + // Capture the mode we are leaving before overwriting it: the transition + // edge, not just the destination, is what tells flapping apart from a + // clean climb to FULL. + auto const prevMode = mode_.load(); + mode_ = om; accounting_.mode(om); - // Record state change for OTel dashboard parity counter. - if (auto* mr = registry_.get().getMetricsRegistry()) - mr->incrementStateChanges(); + // Record the mode transition labelled with source and destination, so the + // dashboard can chart which edges of the sync state machine are traversed + // (e.g. repeated full->connected flapping vs. a one-way + // tracking->connected->full climb). Only reached on a real mode change, + // never in a hot loop, and there are only five modes so the label + // cardinality is bounded. strOperatingMode(mode, admin=false) supplies the + // names, which keeps them identical to the ones server_info reports. + XRPL_METRIC_COUNTER_INC_LABELED( + registry_.get(), + "state_changes_total", + "Total operating mode changes", + {{"from", strOperatingMode(prevMode, false)}, {"to", strOperatingMode(om, false)}}); JLOG(journal_.info()) << "STATE->" << strOperatingMode(); pubServer(); diff --git a/src/xrpld/telemetry/MetricsRegistry.cpp b/src/xrpld/telemetry/MetricsRegistry.cpp index 82c8ed507c..cc642ecec3 100644 --- a/src/xrpld/telemetry/MetricsRegistry.cpp +++ b/src/xrpld/telemetry/MetricsRegistry.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -255,8 +256,10 @@ MetricsRegistry::initSyncInstruments() "validations_sent_total", "Total validations sent by this node"); validationsCheckedCounter_ = meter_->CreateUInt64Counter( "validations_checked_total", "Total network validations received and checked"); - stateChangesCounter_ = - meter_->CreateUInt64Counter("state_changes_total", "Total operating mode changes"); + // state_changes_total is NOT created here. It is emitted at its call site + // (NetworkOPsImp::setMode) through XRPL_METRIC_COUNTER_INC_LABELED so it + // can carry the {from,to} transition labels; a registry-owned instrument + // would only give an unlabelled total. // jq_trans_overflow_total is observed from Overlay's existing cumulative // atomic (Overlay::getJqTransOverflow()) rather than pushed. The overlay // owns the only increment site (PeerImp), so an ObservableCounter reads the @@ -483,6 +486,8 @@ MetricsRegistry::registerAsyncGauges() registerValidationTotalsCounters(); registerUnlQuorumGauge(); registerClockSkewGauge(); + registerSyncStateGauge(); + registerStallEventsCounter(); } void @@ -1549,6 +1554,89 @@ MetricsRegistry::registerClockSkewGauge() this); } +void +MetricsRegistry::registerSyncStateGauge() +{ + // --- Sync diagnostics: why a fresh node is not FULL yet --- + // Four values that previously lived only in a log line or in server_info + // JSON. All four are cheap reads pulled on the ~10 s reader tick. + syncStateGauge_ = + meter_->CreateInt64ObservableGauge("sync_state", "Sync-pipeline health signals"); + syncStateGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* name, int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + auto& ops = app.getOPs(); + + // Time to first FULL. Zero means the node has not synced yet, + // which is exactly the case this signal exists to expose. + observe( + "initial_full_duration_us", + static_cast(ops.getInitialSyncDurationUs())); + + // 1 = still waiting for a full network ledger. While this is + // set the node refuses transactions and cannot reach FULL. + observe("network_ledger_gate", ops.isNeedNetworkLedger() ? 1 : 0); + + // Current main-loop stall duration; 0 when healthy. + observe( + "server_stall_seconds", + static_cast(app.getLoadManager().getCurrentStallSeconds())); + + // Distance from the network tip, floored at zero by the + // accessor. + observe("ledgers_behind", static_cast(ops.getLedgersBehindNetwork())); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerStallEventsCounter() +{ + // --- Sync diagnostics: stall episode count --- + // Observed rather than pushed: LoadManager's monitor thread already owns + // the cumulative tally, and an ObservableCounter reads it each collection + // cycle without threading a push path through the load-monitor loop. + // Kept out of the sync_state gauge because a cumulative total needs + // counter aggregation for rate() to be meaningful. + stallEventsObservable_ = meter_->CreateInt64ObservableCounter( + "server_stall_events_total", "Total server main-loop stall episodes"); + stallEventsObservable_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + try + { + opentelemetry::nostd::get>>(result) + ->Observe( + static_cast(self->app_.getLoadManager().getStallEventCount())); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + #endif // XRPL_ENABLE_TELEMETRY // ----------------------------------------------------------------- @@ -1582,15 +1670,6 @@ MetricsRegistry::incrementValidationsChecked() #endif } -void -MetricsRegistry::incrementStateChanges() -{ -#ifdef XRPL_ENABLE_TELEMETRY - if (enabled_ && stateChangesCounter_) - stateChangesCounter_->Add(1); -#endif -} - void MetricsRegistry::incrementLedgerHistoryMismatch(std::string_view reason) { diff --git a/src/xrpld/telemetry/MetricsRegistry.h b/src/xrpld/telemetry/MetricsRegistry.h index 9f3125d298..eef0c7ef93 100644 --- a/src/xrpld/telemetry/MetricsRegistry.h +++ b/src/xrpld/telemetry/MetricsRegistry.h @@ -36,7 +36,6 @@ * | +-- ledgers_closed_total * | +-- validations_sent_total * | +-- validations_checked_total - * | +-- state_changes_total * | +-- ledger_history_mismatch_total{reason} * | +-- txq_expired_total * | +-- txq_dropped_total{reason} @@ -61,7 +60,12 @@ * +-- State tracking (mode value, time in state) * +-- Storage detail (NuDB sizes) * +-- Validation agreement (1h/24h pct, counts) + * +-- UNL quorum (trusted keys vs required quorum) + * +-- Clock close offset (local clock skew) + * +-- Sync state (time to first FULL, network-ledger gate, + * | server stall seconds, ledgers behind network) * +-- jq_trans_overflow_total (observed from Overlay) + * +-- server_stall_events_total (observed from LoadManager) * * Control-flow for async gauges: * @@ -370,14 +374,6 @@ public: void incrementValidationsChecked(); - /** - * Increment the state_changes_total counter. - * Called from NetworkOPsImp::setMode() when the server operating mode - * changes (e.g. CONNECTED -> SYNCING -> TRACKING -> FULL). - */ - void - incrementStateChanges(); - /** * Increment the ledger_history_mismatch_total counter for a reason. * Called from LedgerHistory::handleMismatch() once the mismatch has @@ -562,6 +558,18 @@ private: * Observable gauge for the network close-time offset (local clock skew). */ opentelemetry::nostd::shared_ptr clockSkewGauge_; + /** + * Observable gauge for the sync-pipeline state signals (time to first + * FULL, network-ledger gate, server stall, ledgers behind the network). + */ + opentelemetry::nostd::shared_ptr syncStateGauge_; + /** + * ObservableCounter: server_stall_events_total — observed from + * LoadManager::getStallEventCount() (cumulative episode tally owned by the + * load-monitor thread). + */ + opentelemetry::nostd::shared_ptr + stallEventsObservable_; /** * Observable gauge for build version info (label-based, value=1). */ @@ -636,11 +644,6 @@ private: */ opentelemetry::nostd::unique_ptr> validationsCheckedCounter_; - /** - * Counter: state_changes_total — incremented on operating mode transitions. - */ - opentelemetry::nostd::unique_ptr> - stateChangesCounter_; /** * ObservableCounter: jq_trans_overflow_total — observed from * Overlay::getJqTransOverflow() (cumulative overflow tally owned by the overlay). @@ -780,7 +783,56 @@ private: */ void registerClockSkewGauge(); // sync diagnostics: close-time offset -#endif // XRPL_ENABLE_TELEMETRY + + /** + * Register the `sync_state` gauge. + * + * One instrument fanning out four series under the `metric` attribute, + * each answering a different "why is this node not FULL yet?" question + * that previously existed only in a log line or in server_info JSON: + * + * `initial_full_duration_us` — microseconds from process start to the + * first FULL transition (NetworkOPs::getInitialSyncDurationUs()). + * Stays 0 until FULL is reached, so a flat 0 IS the "never synced" + * signal; once set it never changes again. + * `network_ledger_gate` — 1 while the node is still waiting to see a + * full network ledger (NetworkOPs::isNeedNetworkLedger()), else 0. A + * persistent 1 blocks transaction submission and FULL. + * `server_stall_seconds` — current main-loop stall duration + * (LoadManager::getCurrentStallSeconds()), 0 when healthy. + * `ledgers_behind` — network tip minus our validated sequence + * (NetworkOPs::getLedgersBehindNetwork()). + * + * The monotonic stall-episode count is a separate instrument + * (`server_stall_events_total`) because a counter and a gauge cannot share + * one instrument: Prometheus would otherwise see a cumulative total under + * last-value aggregation and `rate()` would be meaningless. + * + * @note Pulled on the OTel reader thread (~10 s tick), never on a hot + * path. Three of the four reads are a lock or atomic load; `ledgers_behind` + * additionally walks the connected-peer list, reading each peer's already + * cached ledger range — bounded by peer count and issuing no network I/O. + */ + void + registerSyncStateGauge(); // sync diagnostics: gate, stall, ledgers behind + + /** + * Register the `server_stall_events_total` observable counter. + * + * Observes LoadManager::getStallEventCount(): how many distinct stall + * episodes the monitor thread has reported since process start. Separate + * from `sync_state` because it is cumulative and monotonic, so it needs + * counter (not last-value) aggregation for `rate()` to mean anything. + * + * Read together with `sync_state{metric="server_stall_seconds"}`: a rising + * event count means repeated fresh stalls, while a flat count with a large + * stall-seconds value means one long unresolved stall. + * + * @note Pulled on the OTel reader thread (~10 s tick); one atomic load. + */ + void + registerStallEventsCounter(); // sync diagnostics: stall episode count +#endif // XRPL_ENABLE_TELEMETRY }; } // namespace telemetry