From e99370d433ea2a85805c42e03614bb404ceb8e96 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Sat, 25 Jul 2026 16:14:45 +0100 Subject: [PATCH] feat(telemetry): add peer-supply, serve and amendment diagnostics (WP-A7) Whether the network can even serve this node, and whether the node is about to be shut out of validation, were both invisible: - peer_ledger_supply: how many connected peers advertise a range covering the sequence being fetched. Peers each track a range from status changes, but nothing aggregated them, so "nobody has what I need" looked identical to "peers are slow". - peerfinder_slot_census: outbound active against capacity, connection attempts, inbound, fixed configured against active, and the bootcache and livecache sizes. All were computed already; only two were exported, read at unrelated instants, so they could not be compared. - peer_disconnect_total{reason,direction} and peer_accept_total{outcome}: every disconnect previously collapsed into one number, so our own backpressure could not be told from topology or network faults. Reasons are a fixed set of literals recorded on the peer and emitted once at close, never data supplied by the remote end. - serve_refused_total{request,reason}: the other half of the sync exchange, when this node declines to serve a peer. - amendment_block: whether an unsupported amendment is expected and how long until it activates. Amendment-blocked is terminal for validation, so the countdown is the only leading indicator. The amendment id is deliberately not a label, since the network can vote an id this build has never heard of; it is already logged. - ledger_jump_total: repeated last-closed-ledger switches, which mean the node is thrashing between chains. Co-Authored-By: Claude Opus 5 (1M context) --- src/xrpld/app/misc/NetworkOPs.cpp | 11 ++ src/xrpld/overlay/Overlay.h | 92 ++++++++++++++ src/xrpld/overlay/detail/OverlayImpl.cpp | 84 +++++++++++++ src/xrpld/overlay/detail/OverlayImpl.h | 50 ++++++++ src/xrpld/overlay/detail/PeerImp.cpp | 84 +++++++++++++ src/xrpld/overlay/detail/PeerImp.h | 65 ++++++++++ src/xrpld/peerfinder/PeerfinderManager.h | 112 ++++++++++++++++++ src/xrpld/peerfinder/detail/Logic.h | 37 ++++++ .../peerfinder/detail/PeerfinderManager.cpp | 6 + 9 files changed, 541 insertions(+) diff --git a/src/xrpld/app/misc/NetworkOPs.cpp b/src/xrpld/app/misc/NetworkOPs.cpp index dea39bc36b..15a1da17d9 100644 --- a/src/xrpld/app/misc/NetworkOPs.cpp +++ b/src/xrpld/app/misc/NetworkOPs.cpp @@ -2164,6 +2164,17 @@ NetworkOPsImp::switchLastClosedLedger(std::shared_ptr const& newLC // set the newLCL as our last closed ledger -- this is abnormal code JLOG(journal_.error()) << "JUMP last closed ledger to " << newLCL->header().hash; + // This node was told the network's last closed ledger is not the one it + // built on, and is discarding its own chain tip to follow. Log-only until + // now, so a node repeatedly thrashing between chains left no time series + // to correlate against the rest of the sync pipeline. Rare event, one + // counter Add, no labels: the ledger hash and sequence would both be + // unbounded as label values, and the log line above already carries them. + XRPL_METRIC_COUNTER_INC( + registry_.get(), + "ledger_jump_total", + "Forced jumps of the last closed ledger to a divergent chain"); + clearNeedNetworkLedger(); // Update fee computations. diff --git a/src/xrpld/overlay/Overlay.h b/src/xrpld/overlay/Overlay.h index 6cc229f5a0..9b44d04bce 100644 --- a/src/xrpld/overlay/Overlay.h +++ b/src/xrpld/overlay/Overlay.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -31,6 +32,62 @@ class context; namespace xrpl { +/** + * How much of the ledger range this node needs its peer set can serve. + * + * Aggregated from the per-peer ranges advertised in mtSTATUS_CHANGE. The + * distinction this exists to draw: a node that is behind because no connected + * peer holds the sequence it wants is a supply problem to be fixed by changing + * the peer set, while a node whose peers all hold that sequence is a + * throughput problem. Without the aggregate the two look identical. + * + * Example usage -- the supply verdict from a telemetry gauge callback: + * @code + * auto const supply = overlay.getPeerLedgerSupply(validatedSeq); + * if (supply.peersReporting > 0 && supply.peersServingNext == 0) + * // peers are connected, but none holds the next ledger needed + * @endcode + * + * Example usage -- edge case: nothing has advertised a range yet: + * @code + * auto const supply = overlay.getPeerLedgerSupply(validatedSeq); + * if (supply.peersReporting == 0) + * // supplyMinSeq / supplyMaxSeq are 0 and mean "unknown", not "empty" + * @endcode + * + * @note A peer that has not yet sent a status change advertises [0, 0]. Such + * peers are excluded from every field, so `peersReporting` is the + * denominator that makes the other counts readable: zero serving out of + * zero reporting is silence, zero out of many is a genuine gap. + */ +struct PeerLedgerSupply +{ + /** + * Connected peers that have advertised a non-empty ledger range. + */ + std::int64_t peersReporting{0}; + + /** + * Reporting peers whose range covers this node's validated sequence. + */ + std::int64_t peersServingValidated{0}; + + /** + * Reporting peers whose range covers validated + 1, the next one needed. + */ + std::int64_t peersServingNext{0}; + + /** + * Lowest sequence any reporting peer offers; 0 when none report. + */ + std::int64_t supplyMinSeq{0}; + + /** + * Highest sequence any reporting peer offers; 0 when none report. + */ + std::int64_t supplyMaxSeq{0}; +}; + /** * Manages the set of connected peers. */ @@ -242,6 +299,41 @@ public: */ [[nodiscard]] virtual json::Value txMetrics() const = 0; + + /** + * Returns how much of the sequence range this node needs its peers can + * actually serve. + * + * Each peer advertises the smallest and largest ledger it holds in + * mtSTATUS_CHANGE, which the connection caches. Those ranges are never + * compared against each other, so "no peer on the network holds the + * ledger I need" is today indistinguishable from "my peers are slow". + * This aggregates them into that answer. + * + * @param validatedSeq This node's validated sequence; the ledger it is + * currently able to serve from. + * @return The supply counts and the sequence window the peer set covers. + * + * @note O(peers), taking the peer-list lock once. Intended for a ~10 s + * telemetry poll, never a per-message path. + */ + [[nodiscard]] virtual PeerLedgerSupply + getPeerLedgerSupply(std::uint32_t validatedSeq) const = 0; + + /** + * Returns PeerFinder slot occupancy and address-cache depth. + * + * Forwarded from PeerFinder, which owns the counts. Exposed on Overlay + * because that is the only handle the rest of the server holds; the + * PeerFinder itself is private to the overlay implementation. + * + * Not `const`: the PeerFinder lock is a plain member, so no method on + * that path can be const. + * + * @return One consistent snapshot of all nine fields. + */ + [[nodiscard]] virtual PeerFinder::SlotCensus + getSlotCensus() = 0; }; } // namespace xrpl diff --git a/src/xrpld/overlay/detail/OverlayImpl.cpp b/src/xrpld/overlay/detail/OverlayImpl.cpp index dd2481fb23..e33baddb0c 100644 --- a/src/xrpld/overlay/detail/OverlayImpl.cpp +++ b/src/xrpld/overlay/detail/OverlayImpl.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -82,6 +83,7 @@ #include #include #include +#include #include #include #include @@ -230,18 +232,26 @@ OverlayImpl::onHandoff( JLOG(journal.debug()) << "Peer connection upgrade from " << remoteEndpoint; + // From here on every exit is a terminal outcome for one inbound peer + // attempt, so each reports exactly once. The two returns above are not + // peer attempts at all (a handled HTTP request, and a request that never + // asked to upgrade), which is why they are not counted. error_code ec; auto const localEndpoint(streamPtr->next_layer().socket().local_endpoint(ec)); if (ec) { JLOG(journal.debug()) << remoteEndpoint << " failed: " << ec.message(); + reportAcceptOutcome("local_endpoint_fail"); return handoff; } auto consumer = resourceManager_.newInboundEndpoint(beast::IPAddressConversion::fromAsio(remoteEndpoint)); if (consumer.disconnect(journal)) + { + reportAcceptOutcome("resource_limit"); return handoff; + } auto const [slot, result] = peerFinder_->newInboundSlot( beast::IPAddressConversion::fromAsio(localEndpoint), @@ -252,6 +262,7 @@ OverlayImpl::onHandoff( // connection refused either IP limit exceeded or self-connect handoff.moved = false; JLOG(journal.debug()) << "Peer " << remoteEndpoint << " refused, " << to_string(result); + reportAcceptOutcome("no_slot"); return handoff; } @@ -266,6 +277,7 @@ OverlayImpl::onHandoff( handoff.moved = false; handoff.response = makeRedirectResponse(slot, request, remoteEndpoint.address()); handoff.keepAlive = beast::rfc2616::isKeepAlive(request); + reportAcceptOutcome("not_peer_request"); return handoff; } } @@ -278,6 +290,7 @@ OverlayImpl::onHandoff( handoff.response = makeErrorResponse( slot, request, remoteEndpoint.address(), "Unable to agree on a protocol version"); handoff.keepAlive = false; + reportAcceptOutcome("protocol_mismatch"); return handoff; } @@ -289,6 +302,7 @@ OverlayImpl::onHandoff( handoff.response = makeErrorResponse(slot, request, remoteEndpoint.address(), "Incorrect security cookie"); handoff.keepAlive = false; + reportAcceptOutcome("bad_cookie"); return handoff; } @@ -318,6 +332,7 @@ OverlayImpl::onHandoff( handoff.moved = false; handoff.response = makeRedirectResponse(slot, request, remoteEndpoint.address()); handoff.keepAlive = false; + reportAcceptOutcome("slot_refused"); return handoff; } } @@ -347,6 +362,10 @@ OverlayImpl::onHandoff( peer->run(); } handoff.moved = true; + + // Only after run() is the peer genuinely accepted. Anything that threw + // above is reported as a handshake error by the catch below instead. + reportAcceptOutcome("accepted"); return handoff; } catch (std::exception const& e) @@ -358,6 +377,7 @@ OverlayImpl::onHandoff( handoff.moved = false; handoff.response = makeErrorResponse(slot, request, remoteEndpoint.address(), e.what()); handoff.keepAlive = false; + reportAcceptOutcome("handshake_error"); return handoff; } } @@ -630,6 +650,70 @@ OverlayImpl::reportDnsResolve(std::chrono::steady_clock::time_point start, bool {{"outcome", std::string(resolved ? "resolved" : "empty")}}); } +void +OverlayImpl::reportAcceptOutcome(char const* outcome) +{ + XRPL_METRIC_COUNTER_INC_LABELED( + app_, + "peer_accept_total", + "Inbound peer connection attempts, by terminal outcome", + {{"outcome", std::string(outcome)}}); +} + +PeerLedgerSupply +OverlayImpl::getPeerLedgerSupply(std::uint32_t validatedSeq) const +{ + PeerLedgerSupply supply; + + // Tracked separately from supply.supplyMinSeq so the "nothing reported + // yet" case stays distinguishable: a peer set that genuinely serves from + // sequence 0 and a peer set that has said nothing must not both read 0. + // Only a peer that reported anything can lower this. + auto lowest = std::numeric_limits::max(); + + // The next sequence this node must acquire. Widened before the increment + // so it cannot wrap to 0 at the top of the sequence space, which would + // silently turn "needs the next ledger" into "needs the genesis ledger". + // On a node with no validated ledger yet this is 1, which is correct. + auto const neededSeq = static_cast(validatedSeq) + 1; + + // getActivePeers() takes the overlay lock, copies the list and releases + // it, so the per-peer reads below hold no overlay lock. + for (auto const& peer : getActivePeers()) + { + std::uint32_t minSeq = 0; + std::uint32_t maxSeq = 0; + peer->ledgerRange(minSeq, maxSeq); + + // A peer that has not sent mtSTATUS_CHANGE yet reports [0, 0]. It + // supplies nothing, so it must not be counted as serving and must not + // pull the reported window down to zero. + if (maxSeq == 0) + continue; + + ++supply.peersReporting; + + if (validatedSeq >= minSeq && validatedSeq <= maxSeq) + ++supply.peersServingValidated; + + if (neededSeq >= static_cast(minSeq) && + neededSeq <= static_cast(maxSeq)) + { + ++supply.peersServingNext; + } + + lowest = std::min(lowest, minSeq); + supply.supplyMaxSeq = std::max(supply.supplyMaxSeq, static_cast(maxSeq)); + } + + // Convert the sentinel explicitly. Casting the unsigned max would produce + // 4294967295, which a dashboard would plot as a real sequence. + if (supply.peersReporting > 0) + supply.supplyMinSeq = static_cast(lowest); + + return supply; +} + void OverlayImpl::stop() { diff --git a/src/xrpld/overlay/detail/OverlayImpl.h b/src/xrpld/overlay/detail/OverlayImpl.h index a9ec637ee9..d6d6ea1cdd 100644 --- a/src/xrpld/overlay/detail/OverlayImpl.h +++ b/src/xrpld/overlay/detail/OverlayImpl.h @@ -426,6 +426,56 @@ public: return txMetrics_.json(); } + /** + * Aggregates the per-peer advertised ledger ranges into supply counts. + * + * Iterates the active peers once, reading each one's cached + * `ledgerRange()`. Peers advertising [0, 0] have not reported yet and are + * skipped entirely, so they cannot drag `supplyMinSeq` to zero and make a + * healthy peer set look like it offers history from genesis. + * + * @param validatedSeq This node's validated sequence. + * @return The supply counts and the covered sequence window. + * + * @note O(peers). `getActivePeers()` copies the peer list under the + * overlay lock and releases it before the loop runs, so no peer's + * `ledgerRange()` lock is ever taken while holding the overlay lock. + * @note Called from the ~10 s telemetry gauge callback only. + */ + [[nodiscard]] PeerLedgerSupply + getPeerLedgerSupply(std::uint32_t validatedSeq) const override; + + /** + * Forwards the PeerFinder slot census. + * + * @return One consistent snapshot of all nine slot/cache fields. + */ + [[nodiscard]] PeerFinder::SlotCensus + getSlotCensus() override + { + return peerFinder_->getSlotCensus(); + } + + /** + * Emits the inbound-accept outcome counter for one handoff attempt. + * + * Counts the terminal outcome of every inbound connection this node is + * offered. The outbound twin (`overlay_connect_total`) already exists in + * ConnectAttempt, so this closes the in/out split: without it, a node + * refusing every inbound connection looks the same as one nobody dials. + * + * @param outcome Stable slug for the terminal outcome. Drawn from a fixed + * set of literals at the call sites, never from peer-supplied data, + * so label cardinality is bounded by the code. + * + * @note Cold path: one call per inbound handoff, which is a + * connection-rate event, not a message-rate one. + * @note No-op when telemetry is compiled out or disabled; the macro + * carries that guard, so this needs no `#ifdef`. + */ + void + reportAcceptOutcome(char const* outcome); + /** * Add tx reduce-relay metrics. */ diff --git a/src/xrpld/overlay/detail/PeerImp.cpp b/src/xrpld/overlay/detail/PeerImp.cpp index a8f5db0508..17d394fc2e 100644 --- a/src/xrpld/overlay/detail/PeerImp.cpp +++ b/src/xrpld/overlay/detail/PeerImp.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -228,7 +229,10 @@ PeerImp::run() closed = parseLedgerHash(iter->value()); if (!closed) + { + self->setDisconnectReason("malformed_handshake"); self->fail("Malformed handshake data (1)"); + } } if (auto const iter = self->headers_.find("Previous-Ledger"); iter != self->headers_.end()) @@ -236,11 +240,17 @@ PeerImp::run() previous = parseLedgerHash(iter->value()); if (!previous) + { + self->setDisconnectReason("malformed_handshake"); self->fail("Malformed handshake data (2)"); + } } if (previous && !closed) + { + self->setDisconnectReason("malformed_handshake"); self->fail("Malformed handshake data (3)"); + } { std::scoped_lock const sl(self->recentLock_); @@ -271,6 +281,9 @@ PeerImp::stop() if (!self->socket_.is_open()) return; + // Overlay-wide shutdown, not a fault with this peer. Distinguished so + // a clean restart does not look like a wave of peer failures. + self->setDisconnectReason("stopping"); self->close(); }); } @@ -397,6 +410,10 @@ PeerImp::charge(Resource::Charge const& fee, std::string const& context) expected, true, std::memory_order_acq_rel)) { self->overlay_.incPeerDisconnectCharges(); + // Set inside the latch, so only the one worker that wins the + // exchange writes it. This is the node's own backpressure, not + // a peer or network fault. + self->setDisconnectReason("charge_resources"); self->fail("charge: Resources"); } } @@ -622,9 +639,34 @@ PeerImp::close() socket_.close(ec); // NOLINT(bugprone-unused-return-value) overlay_.incPeerDisconnect(); + + // Emitted right next to incPeerDisconnect() above, and behind the same + // socket-already-closed early return, so this counter's total tracks that + // existing tally rather than being a second, differently-scoped count. + // What it adds is the split: today every disconnect collapses into one + // number, so our-fault backpressure ("large_sendq", "charge_resources") + // cannot be told apart from a topology or network fault ("not_useful", + // "ping_timeout", "read_error"), and the two need opposite responses. + XRPL_METRIC_COUNTER_INC_LABELED( + app_, + "peer_disconnect_total", + "Peer disconnects, by cause and connection direction", + {{"reason", std::string(disconnectReason_)}, + {"direction", std::string(inbound_ ? "inbound" : "outbound")}}); + JLOG((inbound_ ? journal_.debug() : journal_.info())) << "close: Closed"; } +void +PeerImp::reportServeRefusal(char const* request, char const* reason) +{ + XRPL_METRIC_COUNTER_INC_LABELED( + app_, + "serve_refused_total", + "Peer data requests this node declined to serve, by request kind and cause", + {{"request", std::string(request)}, {"reason", std::string(reason)}}); +} + void PeerImp::fail(std::string const& reason) { @@ -718,12 +760,16 @@ PeerImp::onTimer(error_code const& ec) // This should never happen JLOG(journal_.error()) << "onTimer: " << ec.message(); + setDisconnectReason("timer_error"); close(); return; } if (largeSendq_++ >= Tuning::kSendqIntervals) { + // Our own send queue never drained: this node could not keep up with + // what it owed the peer, so it is local backpressure, not a peer fault. + setDisconnectReason("large_sendq"); fail("Large send queue"); return; } @@ -741,6 +787,9 @@ PeerImp::onTimer(error_code const& ec) (t == Tracking::Unknown && (duration > app_.config().maxUnknownTime))) { overlay_.peerFinder().onFailure(slot_); + // The peer is on a different chain, or we cannot tell: a topology + // signal, not a fault on either side. + setDisconnectReason("not_useful"); fail("Not useful"); return; } @@ -749,6 +798,7 @@ PeerImp::onTimer(error_code const& ec) // Already waiting for PONG if (lastPingSeq_) { + setDisconnectReason("ping_timeout"); fail("Ping Timeout"); return; } @@ -787,6 +837,10 @@ PeerImp::onShutdown(error_code ec) } } + // The TLS shutdown handshake finished. First-wins means the reason set by + // whoever asked for the graceful close is kept; "shutdown" only lands when + // the teardown started here, i.e. a clean close with no earlier cause. + setDisconnectReason("shutdown"); close(); } @@ -802,6 +856,7 @@ PeerImp::doAccept() // the shared value successfully in OverlayImpl if (!sharedValue) { + setDisconnectReason("shared_value"); fail("makeSharedValue: Unexpected failure"); return; } @@ -851,6 +906,7 @@ PeerImp::doAccept() if (ec == boost::asio::error::operation_aborted) return; + setDisconnectReason("write_error"); fail("onWriteResponse", ec); return; } @@ -860,6 +916,7 @@ PeerImp::doAccept() doProtocolStart(); return; } + setDisconnectReason("write_error"); fail("Failed to write header"); return; })); @@ -934,10 +991,14 @@ PeerImp::onReadMessage(error_code ec, std::size_t bytesTransferred) if (ec == boost::asio::error::eof) { JLOG(journal_.info()) << "EOF"; + // The peer closed its side cleanly. Counted apart from a read + // error because it is normal peer churn, not a fault. + setDisconnectReason("graceful"); gracefulClose(); return; } + setDisconnectReason("read_error"); fail("onReadMessage", ec); return; } @@ -967,6 +1028,7 @@ PeerImp::onReadMessage(error_code ec, std::size_t bytesTransferred) if (ec) { + setDisconnectReason("read_error"); fail("onReadMessage", ec); return; } @@ -1003,6 +1065,7 @@ PeerImp::onWriteMessage(error_code ec, std::size_t bytesTransferred) if (ec == boost::asio::error::operation_aborted) return; + setDisconnectReason("write_error"); fail("onWriteMessage", ec); return; } @@ -2536,6 +2599,7 @@ PeerImp::onMessage(std::shared_ptr const& m) if (sendQueue_.size() >= Tuning::kDropSendQueue) { JLOG(pJournal_.debug()) << "GetObject: Large send queue"; + reportServeRefusal("object", "sendq_full"); return; } @@ -2891,6 +2955,10 @@ PeerImp::doFetchPack(std::shared_ptr const& packet) (app_.getJobQueue().getJobCount(JtPack) > 10)) { JLOG(pJournal_.info()) << "Too busy to make fetch pack"; + // A fetch pack is how a syncing peer catches up in bulk, so refusing + // one directly slows that peer's sync. Counted separately from the + // ledger path because the shed threshold is a different one. + reportServeRefusal("fetchpack", "load_shed"); return; } @@ -3407,7 +3475,10 @@ PeerImp::processLedgerRequest(std::shared_ptr const& m) if (itype == protocol::liTS_CANDIDATE) { if (sharedMap = getTxSet(m); !sharedMap) + { + reportServeRefusal("txset", "not_found"); return; + } map = sharedMap.get(); // Fill out the reply @@ -3425,16 +3496,21 @@ PeerImp::processLedgerRequest(std::shared_ptr const& m) if (sendQueue_.size() >= Tuning::kDropSendQueue) { JLOG(pJournal_.debug()) << "processLedgerRequest: Large send queue"; + reportServeRefusal("ledger", "sendq_full"); return; } if (app_.getFeeTrack().isLoadedLocal() && !cluster()) { JLOG(pJournal_.debug()) << "processLedgerRequest: Too busy"; + reportServeRefusal("ledger", "load_shed"); return; } if (ledger = getLedger(m); !ledger) + { + reportServeRefusal("ledger", "not_found"); return; + } // Fill out the reply auto const ledgerHash{ledger->header().hash}; @@ -3465,6 +3541,7 @@ PeerImp::processLedgerRequest(std::shared_ptr const& m) default: // This case should not be possible here JLOG(pJournal_.error()) << "processLedgerRequest: Invalid ledger info type"; + reportServeRefusal("ledger", "bad_type"); return; } } @@ -3472,6 +3549,7 @@ PeerImp::processLedgerRequest(std::shared_ptr const& m) if (map == nullptr) { JLOG(pJournal_.warn()) << "processLedgerRequest: Unable to find map"; + reportServeRefusal("ledger", "no_map"); return; } @@ -3556,7 +3634,13 @@ PeerImp::processLedgerRequest(std::shared_ptr const& m) } if (ledgerData.nodes_size() == 0) + { + // The map was found but produced no nodes to return, so the requester + // gets nothing back and will have to ask someone else. Emitted here, + // after the node loop, rather than inside it -- one call per request. + reportServeRefusal(itype == protocol::liTS_CANDIDATE ? "txset" : "ledger", "empty_reply"); return; + } send(std::make_shared(ledgerData, protocol::mtLEDGER_DATA)); } diff --git a/src/xrpld/overlay/detail/PeerImp.h b/src/xrpld/overlay/detail/PeerImp.h index ea6eccd656..7481939a71 100644 --- a/src/xrpld/overlay/detail/PeerImp.h +++ b/src/xrpld/overlay/detail/PeerImp.h @@ -52,6 +52,7 @@ #include #include #include +#include #include #include #include @@ -186,6 +187,30 @@ private: // post duplicate fail() calls) when several queued requests cross // kDropThreshold before the first fail() lands on the strand. std::atomic chargeDisconnectFired_{false}; + + /** + * Why this connection is being torn down, as a stable slug for the + * `peer_disconnect_total` counter's `reason` label. + * + * Set by the site that decides to disconnect and read once by close(), + * which is the single funnel every teardown passes through. Recording at + * close() rather than at each decision site is what makes the count equal + * the real disconnect count: several sites call fail() and then close(), + * and close() itself already self-guards on the socket being open. + * + * First writer wins, so a reason is never overwritten by a later, less + * specific one on the same teardown. Defaults to "unknown" so a path that + * reaches close() without setting a reason still produces a series rather + * than vanishing. + * + * @note Only ever touched on the strand (or, for the charge path, behind + * the chargeDisconnectFired_ latch), so it needs no lock. + * @note The value is always one of a fixed set of literals in this file -- + * never peer-supplied data -- so the label's cardinality is bounded + * by the code. + */ + char const* disconnectReason_{"unknown"}; + std::shared_ptr const slot_; boost::beast::multi_buffer readBuffer_; http_request_type request_; @@ -470,6 +495,46 @@ public: } private: + /** + * Records why this connection is closing, first writer wins. + * + * @param reason Stable slug from the fixed set used in PeerImp.cpp. + * + * @note Not a metric emit. close() does the single emit per teardown; this + * only carries the cause to it, so a site that decides to disconnect + * and a site that performs it stay separate. + */ + void + setDisconnectReason(char const* reason) noexcept + { + // Only the first cause is kept: fail() sites frequently run before + // close(), and a later generic reason must not mask the real one. + if (disconnectReason_ == std::string_view{"unknown"}) + disconnectReason_ = reason; + } + + /** + * Emits the serve-refusal counter for one request this node would not + * answer. + * + * This is the supply side of the sync exchange: what this node refuses to + * serve its peers. Nothing measured it before, so a node shedding every + * ledger request looked identical to one being asked for nothing. + * + * @param request Which request kind was refused: "ledger", "txset", + * "object" or "fetchpack". + * @param reason Why it was refused, from the fixed slug set in PeerImp.cpp. + * + * @note Cold relative to the message loop: one call per refused request, + * and never inside the per-tree-node loop of processLedgerRequest. + * @note Both labels are code literals, never peer-supplied data, so + * cardinality is bounded at compile time. + * @note No-op when telemetry is compiled out or disabled; the macro + * carries that guard. + */ + void + reportServeRefusal(char const* request, char const* reason); + void close(); diff --git a/src/xrpld/peerfinder/PeerfinderManager.h b/src/xrpld/peerfinder/PeerfinderManager.h index 0530343641..bb5721d321 100644 --- a/src/xrpld/peerfinder/PeerfinderManager.h +++ b/src/xrpld/peerfinder/PeerfinderManager.h @@ -206,6 +206,98 @@ to_string(Result result) noexcept return "unknown"; } +//------------------------------------------------------------------------------ + +/** + * One consistent snapshot of slot occupancy and address-cache depth. + * + * Every field is read under a single acquire of the PeerFinder lock, so the + * nine numbers describe the same instant and can be compared against each + * other. Reading them through separate accessors would not give that: the + * autoconnect logic mutates counts and caches together, so two reads taken a + * moment apart can show a state that never existed. + * + * Dependency: + * + * +-----------------+ reads +--------+ + * | Logic (counts_, |--------->| Counts | + * | fixed_, caches)| +--------+ + * +--------+--------+ + * | fills + * v + * +-----------------+ returned by +---------+ + * | SlotCensus |<--------------| Manager | + * +-----------------+ +---------+ + * + * Example usage -- capacity check from a telemetry gauge callback: + * @code + * auto const census = overlay.getSlotCensus(); + * if (census.outActive < census.outMax && census.connecting > 0) + * // dials are starting but never completing + * @endcode + * + * Example usage -- edge case: a fresh node with nothing to dial: + * @code + * auto const census = overlay.getSlotCensus(); + * if (census.bootcache == 0 && census.livecache == 0) + * // no seed addresses at all; check [ips] and DNS + * @endcode + * + * @note All fields are signed so a caller can hand them straight to a metric + * without an unsigned-to-signed conversion that would turn a sentinel + * into a large positive or a negative value. + * @note Snapshot only. The values are stale the moment the lock is released, + * which is fine for diagnostics but must not be used to make a slot + * decision -- take the lock and re-read for that. + */ +struct SlotCensus +{ + /** + * Outbound slots currently occupied by an active peer. + */ + std::int64_t outActive{0}; + + /** + * Outbound slots configured, i.e. the ceiling on outActive. + */ + std::int64_t outMax{0}; + + /** + * Inbound slots currently occupied by an active peer. + */ + std::int64_t inActive{0}; + + /** + * Inbound slots configured; 0 when inbound connections are disabled. + */ + std::int64_t inMax{0}; + + /** + * Outbound connection attempts in flight, not yet active or failed. + */ + std::int64_t connecting{0}; + + /** + * Fixed peers named in the configuration. + */ + std::int64_t fixedConfigured{0}; + + /** + * Fixed peers currently connected, out of fixedConfigured. + */ + std::int64_t fixedActive{0}; + + /** + * Addresses held for bootstrapping, persisted across restarts. + */ + std::int64_t bootcache{0}; + + /** + * Addresses learned from peers this session, held in memory only. + */ + std::int64_t livecache{0}; +}; + /** * Maintains a set of IP addresses used for getting into the network. */ @@ -250,6 +342,26 @@ public: virtual Config config() = 0; + /** + * Returns one consistent snapshot of slot occupancy and cache depth. + * + * Diagnostics only; nothing in the connection logic reads it. Exposed + * because the counts are otherwise invisible outside the PropertyStream: + * only the two active-peer counts are exported today, so a node that + * cannot dial is indistinguishable from one that has nothing to dial. + * + * Not `const`, and cannot be: the implementation takes the PeerFinder + * lock, which is a plain (non-mutable) member, and every other method on + * this interface is non-const for the same reason. + * + * @return All nine fields, read under a single lock acquire. + * + * @note Cheap: one lock acquire, then integer and container-size reads. + * Intended for a ~10 s telemetry poll, not a per-message path. + */ + virtual SlotCensus + getSlotCensus() = 0; + /** * Add a peer that should always be connected. * This is useful for maintaining a private cluster of peers. diff --git a/src/xrpld/peerfinder/detail/Logic.h b/src/xrpld/peerfinder/detail/Logic.h index a7dbbf850d..9720479fc5 100644 --- a/src/xrpld/peerfinder/detail/Logic.h +++ b/src/xrpld/peerfinder/detail/Logic.h @@ -165,6 +165,43 @@ public: return config_; } + /** + * Builds one consistent snapshot of slot occupancy and cache depth. + * + * Lives here rather than on ManagerImp because this is the only scope + * that can see all four sources at once: `counts_` and `fixed_` are + * private, and `lock` is what makes the nine fields describe a single + * instant instead of nine successive ones. + * + * `fixedConfigured` comes from `fixed_.size()` (peers named in the + * config), not `counts_.fixed()` (slots currently present that happen to + * be fixed). The pair that matters to an operator is "how many did I ask + * for" against "how many do I have", which is `fixed_.size()` against + * `counts_.fixedActive()` -- the same comparison autoconnect() makes when + * it decides to redial a fixed peer. + * + * @return All nine fields, read under a single acquire of `lock`. + * + * @note The container sizes are unsigned; each is converted explicitly so + * no value can wrap into a negative reading. + */ + SlotCensus + getSlotCensus() + { + std::scoped_lock const _(lock); + + return SlotCensus{ + .outActive = counts_.outActive(), + .outMax = counts_.outMax(), + .inActive = counts_.inboundActive(), + .inMax = counts_.inMax(), + .connecting = counts_.connectCount(), + .fixedConfigured = static_cast(fixed_.size()), + .fixedActive = static_cast(counts_.fixedActive()), + .bootcache = static_cast(bootcache.size()), + .livecache = static_cast(livecache.size())}; + } + void addFixedPeer(std::string_view name, beast::IP::Endpoint const& ep) { diff --git a/src/xrpld/peerfinder/detail/PeerfinderManager.cpp b/src/xrpld/peerfinder/detail/PeerfinderManager.cpp index 2727a03013..302f4588a6 100644 --- a/src/xrpld/peerfinder/detail/PeerfinderManager.cpp +++ b/src/xrpld/peerfinder/detail/PeerfinderManager.cpp @@ -98,6 +98,12 @@ public: return logic_.config(); } + SlotCensus + getSlotCensus() override + { + return logic_.getSlotCensus(); + } + void addFixedPeer(std::string_view name, std::vector const& addresses) override {