diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 57f98147d3..fd4ce9a58a 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -2122,9 +2122,14 @@ first one that is wrong and fix it before reading further panels. node already holds. - `fetch_error`, `bad_status`, `parse_error` — transport or content faults; the site is effectively unreachable. - - `expired`, `stale`, `untrusted`, `invalid`, `unsupported_version`, - `pending` — the list arrived but was rejected, so no keys are loaded from - that site. + - `stale`, `untrusted`, `invalid`, `unsupported_version` — the list arrived + but was rejected, so no keys are loaded from that site. + - `expired` — the list was applied and its keys were loaded, but it is past + its validity window, so the expiry sweep drops them again and the + publisher does not count as available. Refresh the publisher blob, and + check the local clock, rather than replacing `validators.txt`. + - `pending` — the list is valid only from a future date and is held for + rotation. Normal, not a fault. Then read _UNL Trusted Keys vs Quorum_ and _UNL Quorum Headroom_ (`unl_quorum`, `metric=trusted_keys` against `metric=quorum`). **This is the "will never validate" check.** If `trusted_keys` is zero, or sits at or @@ -2132,7 +2137,11 @@ first one that is wrong and fix it before reading further panels. too small to ever satisfy quorum: the node can track ledgers but will never declare one validated, and no amount of healthy acquire traffic changes that. A site stuck on `fetch_error` or `expired` in the panel above - is the usual cause. + is the usual cause. A very large `quorum` with a deeply negative headroom + is the distinct "quorum disabled" state: too many publishers are + unavailable, so quorum has been switched off entirely rather than merely + set high. Fix publisher reachability first — the key count is irrelevant + until quorum is enabled again. 5. **Clock — is local time disagreeing with the network?** Panel _Clock Close Offset_ (`clock_close_offset_seconds`, `metric=offset`). diff --git a/src/xrpld/overlay/detail/Handshake.cpp b/src/xrpld/overlay/detail/Handshake.cpp index 45784f7c45..fcd359d158 100644 --- a/src/xrpld/overlay/detail/Handshake.cpp +++ b/src/xrpld/overlay/detail/Handshake.cpp @@ -42,7 +42,6 @@ #include #include #include -#include // VFALCO Shouldn't we have to include the OpenSSL // headers or something for SSL_get_finished? @@ -260,7 +259,7 @@ namespace { * @note Always throws; it never returns to its caller. */ [[noreturn]] void -throwNegotiationFailure(Application& app, char const* reason, std::string message) +throwNegotiationFailure(Application& app, char const* reason, std::string const& message) { XRPL_METRIC_COUNTER_INC_LABELED( app, @@ -268,7 +267,7 @@ throwNegotiationFailure(Application& app, char const* reason, std::string messag "Peer handshake negotiations rejected, by reason", {{"reason", std::string(reason)}}); - throw std::runtime_error(std::move(message)); + throw std::runtime_error(message); } } // namespace diff --git a/src/xrpld/telemetry/MetricsRegistry.cpp b/src/xrpld/telemetry/MetricsRegistry.cpp index 0023ef4034..82c8ed507c 100644 --- a/src/xrpld/telemetry/MetricsRegistry.cpp +++ b/src/xrpld/telemetry/MetricsRegistry.cpp @@ -68,6 +68,7 @@ #include #include #include +#include #include #include #include @@ -1491,7 +1492,19 @@ MetricsRegistry::registerUnlQuorumGauge() observe("trusted_keys", static_cast(validators.trustedKeyCount())); // Validations required for a ledger to be fully validated. - observe("quorum", static_cast(validators.quorum())); + // ValidatorList disables quorum by returning SIZE_MAX when too + // many publishers are unavailable. Casting that straight to + // int64_t would wrap to -1 and make the headroom + // (trusted_keys - quorum) read positive on a node that can + // never validate, so report the disabled state as int64 max + // instead: headroom then goes strongly negative, which is the + // truthful signal. + auto const quorum = validators.quorum(); + observe( + "quorum", + quorum == std::numeric_limits::max() + ? std::numeric_limits::max() + : static_cast(quorum)); } catch (...) // NOLINT(bugprone-empty-catch) {