diff --git a/.github/scripts/levelization/results/loops.txt b/.github/scripts/levelization/results/loops.txt index 0641538e6d..7a9fe50f45 100644 --- a/.github/scripts/levelization/results/loops.txt +++ b/.github/scripts/levelization/results/loops.txt @@ -17,5 +17,5 @@ Loop: xrpld.overlay xrpld.rpc xrpld.rpc ~= xrpld.overlay Loop: xrpld.overlay xrpld.telemetry - xrpld.telemetry ~= xrpld.overlay + xrpld.overlay > xrpld.telemetry diff --git a/OpenTelemetryPlan/09-data-collection-reference.md b/OpenTelemetryPlan/09-data-collection-reference.md index 769e909ff2..646d6b8c9e 100644 --- a/OpenTelemetryPlan/09-data-collection-reference.md +++ b/OpenTelemetryPlan/09-data-collection-reference.md @@ -1401,5 +1401,13 @@ signal as it lands. `Type` is the instrument kind (counter / gauge / histogram / span / span attr), `Emit site` the owning source file, and `Panel` the dashboard panel that renders it. -| Signal | Type | Emit site | Panel | Meaning | -| ------ | ---- | --------- | ----- | ------- | +| Signal | Type | Emit site | Panel | Meaning | +| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | -------------------------------------------------------------------- | ----------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `dns_resolve_total` (`outcome` = `resolved` \| `empty`) | counter | `OverlayImpl.cpp` — `OverlayImpl::reportDnsResolve` | DNS Resolve Outcome Rate | Peer hostname resolutions. `empty` means a configured bootstrap or `[ips_fixed]` name returned no address, so that peer is never dialled. | +| `dns_resolve_latency_ms` | histogram | `OverlayImpl.cpp` — `OverlayImpl::reportDnsResolve` | DNS Resolve Latency (p95) | Time to resolve a configured peer hostname. Seconds-scale values mean the resolver is timing out ahead of every dial. | +| `overlay_connect_total` (`outcome` = `connected` \| `tcp_fail` \| `tls_fail` \| `upgrade_fail` \| `timeout`) | counter | `ConnectAttempt.cpp` — `ConnectAttempt::reportOutcome` | Outbound Dial Outcome Rate | Outbound peer connection attempts by terminal outcome. The outcome names the stage that broke: TCP, TLS, HTTP upgrade, or no terminal state in time. | +| `overlay_dial_latency_ms` | histogram | `ConnectAttempt.cpp` — `ConnectAttempt::reportOutcome` | Outbound Dial Latency (p95) | Time from starting an outbound dial to its terminal outcome, successes and failures together. A p95 near the dial timeout means peers accept TCP but never finish the handshake. | +| `handshake_negotiation_fail_total` (`reason`, 14 values incl. `wrong_network`, `invalid_network_id`, `clock_skew`, `self_connection`, `session_verify_failed`) | counter | `Handshake.cpp` — `throwNegotiationFailure` (from `verifyHandshake`) | Handshake Negotiation Failures by Reason | Peer handshakes rejected after TLS while checking network id, clock, keys and addresses. `reason` names the failing check. | +| `unl_fetch_total` (`site` = configured UNL URI; `outcome` = the 9 `ListDisposition` strings `accepted` \| `expired` \| `same_sequence` \| `pending` \| `known_sequence` \| `unsupported_version` \| `untrusted` \| `stale` \| `invalid`, plus `fetch_error` \| `bad_status` \| `parse_error`) | counter | `ValidatorSite.cpp` — `ValidatorSite::reportFetchOutcome` | UNL Fetch Rate by Site & Outcome | Validator-list fetches per site. `accepted` is the only success; `same_sequence` and `known_sequence` are normal no-op refreshes; the three literals are transport or content faults. | +| `unl_quorum` (`metric` = `trusted_keys` \| `quorum`) | observable gauge | `MetricsRegistry.cpp` — `registerUnlQuorumGauge` | UNL Trusted Keys vs Quorum; UNL Quorum Headroom | Trusted UNL key count against the validations a ledger needs. `trusted_keys` at or below `quorum` means the node can never declare a ledger validated. | +| `clock_close_offset_seconds` (`metric` = `offset`) | observable gauge | `MetricsRegistry.cpp` — `registerClockSkewGauge` | Clock Close Offset | Network close time offset from the local clock. Negative means the local clock runs ahead. `server_info` only surfaces `close_time_offset` at 60 s or more, so this gauge sees skew far earlier. | diff --git a/docker/telemetry/grafana/dashboards/ledger-sync-health.json b/docker/telemetry/grafana/dashboards/ledger-sync-health.json index 33efa14db9..a8aa27df67 100644 --- a/docker/telemetry/grafana/dashboards/ledger-sync-health.json +++ b/docker/telemetry/grafana/dashboards/ledger-sync-health.json @@ -32,7 +32,7 @@ } ] }, - "description": "What this shows: Fresh-node ledger-sync diagnostics: pre-quorum bootstrap (Domain 0) and the ledger/tx-set acquire pipeline. \u2014 Use it to: Find out why a freshly started node is slow to reach, or never reaches, server_state full.", + "description": "What this shows: Fresh-node ledger-sync diagnostics: pre-quorum bootstrap (Domain 0) and the ledger/tx-set acquire pipeline. — Use it to: Find out why a freshly started node is slow to reach, or never reaches, server_state full.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -52,6 +52,917 @@ }, "panels": [] }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Rate of peer-hostname DNS resolutions, split by outcome (resolved or empty).*\n\n###### How it's computed:\n*Per-second rate of completed resolutions grouped by outcome, per node.*\n\n###### Reading it:\n*Resolved should account for every attempt; the empty line should stay flat at zero.*\n\n###### Healthy range:\n*A short burst of resolved at startup, then flat. Non-zero empty is always a defect.*\n\n###### Watch for:\n*Any empty rate means a configured bootstrap or [ips_fixed] hostname returned no address, so the node never even tries to dial that peer.*\n\n###### Keywords:\n- **DNS resolve** *(per node)* — turning a configured peer hostname into IP addresses before any dial is attempted; `outcome=empty` means the name resolved to nothing.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl::reportDnsResolve`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#dns-resolve)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Resolutions / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_node_role) (rate(dns_resolve_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "DNS Resolve Outcome Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Time taken to resolve a configured peer hostname, at the 95th percentile.*\n\n###### How it's computed:\n*Resolution duration samples aggregated to their 95th percentile per node.*\n\n###### Reading it:\n*Lower is better; it is the delay before the node can start dialling peers.*\n\n###### Healthy range:\n*Tens of milliseconds against a healthy resolver.*\n\n###### Watch for:\n*Seconds-scale latency means the resolver is timing out and every bootstrap attempt is paying that delay before the first dial.*\n\n###### Keywords:\n- **DNS resolve** *(per node)* — turning a configured peer hostname into IP addresses before any dial is attempted; slow resolution delays the whole bootstrap.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl::reportDnsResolve`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#dns-resolve)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Latency (ms)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(dns_resolve_latency_ms_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 DNS Resolve\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "DNS Resolve Latency (p95)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Rate of outbound peer connection attempts, split by terminal outcome.*\n\n###### How it's computed:\n*Per-second rate of finished dials grouped by outcome, per node. Filter the outcome set with the Dial Outcome variable.*\n\n###### Reading it:\n*Connected should dominate. The failure lines name the stage that broke: tcp_fail (no route or refused), tls_fail (TLS handshake), upgrade_fail (HTTP upgrade or protocol negotiation), timeout (no terminal state in time).*\n\n###### Healthy range:\n*Connected rising to the configured peer count, then flat with failures near zero.*\n\n###### Watch for:\n*All attempts landing on one failure outcome and no connected line — the node has no outbound peers and can never sync.*\n\n###### Keywords:\n- **Outbound dial latency** *(per node)* — an outbound peer connection attempt from TCP connect through TLS to protocol upgrade; each attempt ends in exactly one outcome.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ConnectAttempt.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/ConnectAttempt.cpp)\n\n###### Function:\n`ConnectAttempt::reportOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Attempts / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 5, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_node_role) (rate(overlay_connect_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$dial_outcome\"}[$__rate_interval])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Outbound Dial Outcome Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Time from starting an outbound peer dial to its terminal outcome, at the 95th percentile.*\n\n###### How it's computed:\n*Dial duration samples aggregated to their 95th percentile per node.*\n\n###### Reading it:\n*Lower is better. The series covers successes and failures together, so a rising p95 usually means attempts are ending in timeout rather than being refused fast.*\n\n###### Healthy range:\n*Tens to low hundreds of milliseconds on a local or same-region peer.*\n\n###### Watch for:\n*A p95 pinned near the dial timeout, which means peers accept the TCP connection but never complete the handshake.*\n\n###### Keywords:\n- **Outbound dial latency** *(per node)* — elapsed time of an outbound peer connection attempt, measured to whichever outcome ends it.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ConnectAttempt.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/ConnectAttempt.cpp)\n\n###### Function:\n`ConnectAttempt::reportOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Latency (ms)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 6, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(overlay_dial_latency_ms_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Outbound Dial\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Outbound Dial Latency (p95)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Rate of peer handshakes rejected during protocol negotiation, split by reason.*\n\n###### How it's computed:\n*Per-second rate of rejected handshakes grouped by reason, per node. Filter the reason set with the Handshake Reason variable.*\n\n###### Reading it:\n*Flat at zero is healthy. The reason names the exact check that rejected the peer, so one dominant reason is the fault to fix.*\n\n###### Healthy range:\n*Zero, or a low background rate of self_connection and remote_ip_mismatch on a NAT'd host.*\n\n###### Watch for:\n*wrong_network or invalid_network_id — the node is configured for a different network than its peers and will never reach a quorum. clock_skew points at the local clock; session_verify_failed and bad_public_key at a misbehaving peer.*\n\n###### Keywords:\n- **Handshake negotiation failure** *(per node)* — a peer connection rejected after TLS while checking network id, clock, keys and addresses; the reason label names the failing check.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Handshake.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/Handshake.cpp)\n\n###### Function:\n`throwNegotiationFailure`\n\n###### References:\n[Peer protocol on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#handshake-negotiation-failure)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Failures / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 7, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (reason, service_instance_id, xrpl_branch, xrpl_node_role) (rate(handshake_negotiation_fail_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", reason=~\"$handshake_reason\"}[$__rate_interval])), \"series\", \"$1\", \"reason\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Handshake Negotiation Failures by Reason", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Rate of validator-list fetches from each configured UNL site, split by outcome.*\n\n###### How it's computed:\n*Per-second rate of fetch attempts grouped by site and outcome, per node. Filter with the UNL Site and UNL Fetch Outcome variables.*\n\n###### Reading it:\n*accepted is the only success value. same_sequence and known_sequence are normal no-op refreshes of a list the node already holds. fetch_error, bad_status and parse_error are transport or content faults; expired, stale, untrusted, invalid and unsupported_version mean the list was retrieved but rejected.*\n\n###### Healthy range:\n*A first accepted per site at startup, then a steady low rate of same_sequence refreshes.*\n\n###### Watch for:\n*A site with only fetch_error or bad_status is unreachable. Only expired or invalid means the site is reachable but its list is unusable, so no trusted keys are loaded from it.*\n\n###### Keywords:\n- **UNL fetch outcome** *(per node)* — the result of retrieving and applying a validator list from a configured site; `accepted` is the only success.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ValidatorSite.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/ValidatorSite.cpp)\n\n###### Function:\n`ValidatorSite::reportFetchOutcome`\n\n###### References:\n[UNL (Unique Node List)](https://xrpl.org/docs/concepts/consensus-protocol/unl) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-fetch-outcome)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Fetches / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 8, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_join(sum by (outcome, site, service_instance_id, xrpl_branch, xrpl_node_role) (rate(unl_fetch_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", site=~\"$unl_site\", outcome=~\"$unl_outcome\"}[$__rate_interval])), \"series\", \" \", \"outcome\", \"site\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "UNL Fetch Rate by Site & Outcome", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Trusted validator keys currently in effect, plotted against the number of agreeing validations a ledger needs.*\n\n###### How it's computed:\n*Two series read from the same gauge: trusted_keys (usable UNL size) and quorum (validations required to declare a ledger validated).*\n\n###### Reading it:\n*Trusted Keys must sit above Quorum. Where the lines cross, or where Trusted Keys is zero, the node cannot reach a quorum and will never validate a ledger no matter how healthy the rest of the pipeline looks.*\n\n###### Healthy range:\n*Trusted Keys comfortably above Quorum and both flat.*\n\n###### Watch for:\n*Trusted Keys at zero (no usable UNL loaded) or below Quorum. Steps in Quorum track validator-list changes; steps down in Trusted Keys mean keys were dropped.*\n\n###### Keywords:\n- **UNL quorum headroom** *(per node)* — trusted UNL key count minus the required quorum; at or below zero the node can never declare a ledger validated.\n- **UNL (Unique Node List)** *(per node)* — the list of validators a node trusts not to collude; the basis for its consensus and quorum.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerUnlQuorumGauge`\n\n###### References:\n[UNL (Unique Node List)](https://xrpl.org/docs/concepts/consensus-protocol/unl) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-quorum-headroom)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Validator Keys", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 9, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(unl_quorum{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"trusted_keys\"}, \"series\", \"Trusted Keys\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(unl_quorum{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"quorum\"}, \"series\", \"Quorum\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "UNL Trusted Keys vs Quorum", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Spare trusted validator keys above the required quorum — the single number that says whether this node can ever validate.*\n\n###### How it's computed:\n*Trusted key count minus the required quorum, matched per node.*\n\n###### Reading it:\n*Positive is healthy. Zero or negative (red) means the trusted UNL is too small to ever satisfy quorum, so the node will stay short of a validated ledger.*\n\n###### Healthy range:\n*Positive; the exact figure depends on UNL size and the configured quorum.*\n\n###### Watch for:\n*Zero or below. Pair it with UNL Fetch Rate by Site & Outcome: a site stuck on fetch_error or expired is the usual cause of a UNL too small to meet quorum.*\n\n###### Keywords:\n- **UNL quorum headroom** *(per node)* — trusted UNL key count minus the required quorum; at or below zero the node can never declare a ledger validated.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerUnlQuorumGauge`\n\n###### References:\n[Validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-quorum-headroom)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "green", + "value": 2 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(unl_quorum{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"trusted_keys\"} - ignoring(metric) unl_quorum{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"quorum\"}, \"series\", \"Quorum Headroom\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "UNL Quorum Headroom", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How far the network's agreed close time sits from this node's own clock.*\n\n###### How it's computed:\n*Signed offset in seconds, plus its magnitude so a threshold band applies in either direction.*\n\n###### Reading it:\n*Both series should hug zero. A negative signed value means the local clock runs ahead of the network, positive means it lags. The magnitude is what matters: the threshold lines sit at 1s (suspicious) and 60s.*\n\n###### Healthy range:\n*Magnitude under 1 second.*\n\n###### Watch for:\n*A magnitude above 1 second that does not decay, which delays consensus participation. Note that server_info only surfaces close_time_offset once the magnitude reaches 60 seconds, so this panel sees skew long before the API does; a persistent offset is a local NTP fault, not a network one.*\n\n###### Keywords:\n- **Clock close offset** *(per node)* — the difference between the network's agreed close time and this node's clock; a persistent offset delays consensus participation.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerClockSkewGauge`\n\n###### References:\n[Ledger close times on xrpl.org](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#clock-close-offset)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "Close Offset (s)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 60 + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 49 + }, + "id": 11, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(clock_close_offset_seconds{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"offset\"}, \"series\", \"Close Offset\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(abs(clock_close_offset_seconds{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"offset\"}), \"series\", \"Offset Magnitude\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "Clock Close Offset", + "type": "timeseries" + }, { "id": 2, "type": "row", @@ -61,7 +972,7 @@ "h": 1, "w": 24, "x": 0, - "y": 1 + "y": 61 }, "panels": [] } @@ -222,6 +1133,86 @@ "multi": true, "refresh": 2, "sort": 1 + }, + { + "name": "dial_outcome", + "label": "Dial Outcome", + "description": "Filter outbound dial attempts by terminal outcome [connected / tcp_fail / tls_fail / upgrade_fail / timeout]", + "type": "query", + "query": "label_values(overlay_connect_total, outcome)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "handshake_reason", + "label": "Handshake Reason", + "description": "Filter handshake negotiation failures by the check that rejected the peer (e.g. wrong_network, clock_skew)", + "type": "query", + "query": "label_values(handshake_negotiation_fail_total, reason)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "unl_site", + "label": "UNL Site", + "description": "Filter validator-list fetches by configured UNL site URI", + "type": "query", + "query": "label_values(unl_fetch_total, site)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "unl_outcome", + "label": "UNL Fetch Outcome", + "description": "Filter validator-list fetches by outcome (accepted is the only success)", + "type": "query", + "query": "label_values(unl_fetch_total, outcome)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 } ] }, diff --git a/docker/telemetry/workload/expected_metrics.json b/docker/telemetry/workload/expected_metrics.json index dc8eb486fa..8f8bbb712a 100644 --- a/docker/telemetry/workload/expected_metrics.json +++ b/docker/telemetry/workload/expected_metrics.json @@ -128,8 +128,19 @@ "metrics": ["storage_detail{metric=\"nudb_bytes\"}"] }, "sync_diagnostics": { - "description": "Fresh-node sync diagnostics (MetricsRegistry native metrics). Bootstrap (Domain 0) and acquire-pipeline signals rendered by the ledger-sync-health dashboard. Empty until the emitting work packages land; names are appended one per signal.", - "metrics": [] + "description": "Fresh-node sync diagnostics (native metrics). Bootstrap (Domain 0) and acquire-pipeline signals rendered by the ledger-sync-health dashboard. Names are appended one per signal. Histograms are listed by their Prometheus _bucket series (the bare instrument name is not a series). The two observable gauges carry an inline metric= selector so the specific sub-series is asserted, matching the parity_* groups.", + "metrics": [ + "dns_resolve_total", + "dns_resolve_latency_ms_bucket", + "overlay_connect_total", + "overlay_dial_latency_ms_bucket", + "handshake_negotiation_fail_total", + "unl_fetch_total", + "unl_quorum{metric=\"trusted_keys\"}", + "unl_quorum{metric=\"quorum\"}", + "clock_close_offset_seconds{metric=\"offset\"}" + ], + "_conditional_note": "handshake_negotiation_fail_total and unl_fetch_total are conditional under the local harness: the first only exists once a handshake is rejected, and the second needs a [validator_list_sites] entry (run-full-validation.sh generates a static [validators] file instead). The validator has no per-metric optional flag, so if either reports 0 series in a harness run, move it out of this group rather than weakening the check." }, "grafana_dashboards": { "description": "All Grafana dashboards that must render data (UIDs as provisioned on disk under docker/telemetry/grafana/dashboards/).", diff --git a/docker/telemetry/workload/validate_telemetry.py b/docker/telemetry/workload/validate_telemetry.py index c5ea8bd815..959064d15c 100644 --- a/docker/telemetry/workload/validate_telemetry.py +++ b/docker/telemetry/workload/validate_telemetry.py @@ -579,6 +579,17 @@ async def validate_metrics( "ledger_economy", "state_tracking", "storage_detail", + # Fresh-node sync diagnostics. Two of these only exist + # under specific conditions (a rejected handshake, a + # configured UNL site), so listing them here is how a + # failed run shows whether the metric was absent or + # merely misnamed. + "dns_resolve", + "overlay_connect", + "overlay_dial", + "handshake_", + "unl_", + "clock_close_offset", ) ) ] diff --git a/docs/telemetry-glossary.md b/docs/telemetry-glossary.md index 0c0d18c4ab..5ed921e2c2 100644 --- a/docs/telemetry-glossary.md +++ b/docs/telemetry-glossary.md @@ -491,6 +491,16 @@ When a node is missing ledgers (at startup, after an outage, or to extend histor **Scope:** per node — measured on and specific to this individual server. + + +### Clock close offset + +The difference between the network's agreed ledger close time and this node's own clock, negative when the local clock runs ahead and positive when it lags. A magnitude above a second that does not decay delays consensus participation and is a local time-sync fault rather than a network one; the server-info API only reports the offset once the magnitude reaches 60 seconds, so the metric sees skew far earlier. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Clock drift](#clock-drift) · [Ledger close times on xrpl.org](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) + ### Complete ledger ranges @@ -499,6 +509,16 @@ A node stores ledger history as one or more contiguous ranges. One continuous ra **Scope:** per node — measured on and specific to this individual server. + + +### DNS resolve + +Turning each configured peer hostname into IP addresses, which happens before any connection is attempted. An empty outcome means the name resolved to nothing, so that peer is never dialled at all; slow resolution delays every dial behind it even when it eventually succeeds. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Outbound dial latency](#outbound-dial-latency) · [Peer protocol on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) + ### Fresh-node sync diagnostics @@ -509,6 +529,16 @@ The set of signals that explain why a freshly-started node is slow to reach, or **See also:** [Diagnosing slow/stuck fresh sync](./telemetry-runbook.md#diagnosing-slowstuck-fresh-sync) (operator flow) · [Server states on xrpl.org](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) + + +### Handshake negotiation failure + +A peer connection rejected after TLS succeeds, while the two sides check network identifier, clock, keys and reported addresses. The rejection reason names the failing check: a wrong network identifier means the node can never reach a quorum with those peers, a clock reason points at local time sync, and key or signature reasons point at the peer. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Outbound dial latency](#outbound-dial-latency) · [Clock close offset](#clock-close-offset) · [Peer protocol on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) + ### Historical fetch rate @@ -535,6 +565,36 @@ The server state describes how fully the node is participating, in ascending ord **See also:** [Operating mode / server state on xrpl.org](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) + + +### Outbound dial latency + +The elapsed time of one outbound peer connection attempt, from starting the TCP connect through TLS to the protocol upgrade, measured to whichever outcome ends it. Every attempt ends in exactly one outcome, so the outcome names the stage that broke; because the timing covers failures too, a value pinned near the dial timeout means peers accept the connection but never finish the handshake. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [DNS resolve](#dns-resolve) · [Handshake negotiation failure](#handshake-negotiation-failure) · [Peer protocol on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) + + + +### UNL fetch outcome + +The result of retrieving a validator list from one configured UNL site and applying it. Accepted is the only success; same-sequence and known-sequence are normal no-op refreshes of a list the node already holds; fetch, status and parse errors are transport or content faults; and expired, stale, untrusted, invalid or unsupported-version mean the list arrived but was rejected, so no trusted keys are loaded from that site. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [UNL quorum headroom](#unl-quorum-headroom) · [UNL (Unique Node List)](#unl-unique-node-list) · [UNL (Unique Node List) on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/unl) + + + +### UNL quorum headroom + +The trusted UNL key count minus the quorum a ledger needs, so it reads as spare validator keys. At or below zero — including a trusted key count of zero, meaning no usable list loaded — the node can track ledgers but can never declare one validated, however healthy the rest of the sync pipeline looks. A UNL site that keeps failing to fetch is the usual cause. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [UNL fetch outcome](#unl-fetch-outcome) · [Validation quorum](#validation-quorum) · [Validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) + ## Peer & Overlay Networking diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 353fea2519..57f98147d3 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -2068,7 +2068,85 @@ Signal index (instrument, emit site, panel): Note: the sync signals are native metrics, which are never sampled — unlike the span-derived (spanmetrics) series, they are always complete. -_(Per-signal diagnosis steps are added as each signal lands.)_ +#### Bootstrap (Domain 0) — ordered diagnosis + +Work the Bootstrap row in this order. Each step gates the next, so stop at the +first one that is wrong and fix it before reading further panels. + +1. **DNS — did the node resolve its configured peers?** + Panel _DNS Resolve Outcome Rate_ (`dns_resolve_total`). Any rate on + `outcome=empty` means a name in `[ips]` or `[ips_fixed]` returned no address, + so that peer is never dialled. Fix the hostname or point at an IP. + If both outcomes are flat at zero the node resolved nothing at all — check + that `[ips]`/`[ips_fixed]` is actually populated. + Then check _DNS Resolve Latency (p95)_ (`dns_resolve_latency_ms`): a + seconds-scale p95 is a resolver timing out, which delays every dial behind + it even when the resolution eventually succeeds. + +2. **Outbound dial — did the connections complete?** + Panel _Outbound Dial Outcome Rate_ (`overlay_connect_total`). The `outcome` + label names the stage that broke: + - `connected` — success; this is the line that must be non-zero. + - `tcp_fail` — no route, refused, or the peer port is closed or firewalled. + - `tls_fail` — the TLS handshake failed. + - `upgrade_fail` — TLS succeeded but the HTTP upgrade or protocol + negotiation was rejected. This is the outcome that pairs with step 3. + - `timeout` — the attempt never reached a terminal state. + If every attempt lands on one failure outcome and `connected` stays at + zero, the node has no outbound peers and cannot sync at all. + _Outbound Dial Latency (p95)_ (`overlay_dial_latency_ms`) covers successes + and failures together, so a p95 pinned near the dial timeout means peers + accept the TCP connection but never finish the handshake. + +3. **Handshake — why were peers rejected?** + Panel _Handshake Negotiation Failures by Reason_ + (`handshake_negotiation_fail_total`). This is where an `upgrade_fail` from + step 2 gets its cause. The `reason` label is the check that rejected the + peer; the ones worth acting on first: + - `wrong_network` or `invalid_network_id` — **most common fresh-node + misconfiguration.** The node is on a different network than its peers, so + it will never reach a quorum however healthy everything else looks. Fix + `[network_id]` and the peer list together. + - `clock_skew` — the peer rejected this node's clock; go to step 5. + - `session_verify_failed`, `bad_public_key`, `no_session_signature` — a + misbehaving or mismatched peer rather than a local fault. + - `self_connection`, `local_ip_mismatch`, `remote_ip_mismatch` — a low + background rate is normal on a NAT'd host. + +4. **UNL — can the node ever form a quorum?** + Panel _UNL Fetch Rate by Site & Outcome_ (`unl_fetch_total`), grouped by + `site` and `outcome`. Read the outcome carefully — `accepted` is the only + success value: + - `accepted` — the list was applied. + - `same_sequence`, `known_sequence` — normal no-op refreshes of a list the + node already holds. + - `fetch_error`, `bad_status`, `parse_error` — transport or content faults; + the site is effectively unreachable. + - `expired`, `stale`, `untrusted`, `invalid`, `unsupported_version`, + `pending` — the list arrived but was rejected, so no keys are loaded from + that site. + Then read _UNL Trusted Keys vs Quorum_ and _UNL Quorum Headroom_ + (`unl_quorum`, `metric=trusted_keys` against `metric=quorum`). **This is + the "will never validate" check.** If `trusted_keys` is zero, or sits at or + below `quorum` (headroom zero or negative, shown red), the trusted UNL is + too small to ever satisfy quorum: the node can track ledgers but will + never declare one validated, and no amount of healthy acquire traffic + changes that. A site stuck on `fetch_error` or `expired` in the panel above + is the usual cause. + +5. **Clock — is local time disagreeing with the network?** + Panel _Clock Close Offset_ (`clock_close_offset_seconds`, `metric=offset`). + The signed series is negative when the local clock runs ahead of the network + and positive when it lags; the magnitude series carries the threshold bands. + A magnitude above 1 second that does not decay is suspicious and delays + consensus participation. Note that `server_info` only reports + `close_time_offset` once the magnitude reaches 60 seconds, so this panel sees + skew long before the API does. A persistent offset is a local NTP fault, not + a network one — and it is also the cause behind a `clock_skew` reason in + step 3. + +If all five steps are clean the bootstrap stage is healthy, and the problem is +in the **Sync pipeline** row instead. ## Performance Tuning diff --git a/src/tests/libxrpl/telemetry/MetricMacros.cpp b/src/tests/libxrpl/telemetry/MetricMacros.cpp index ddcd2af33c..9c92deb22e 100644 --- a/src/tests/libxrpl/telemetry/MetricMacros.cpp +++ b/src/tests/libxrpl/telemetry/MetricMacros.cpp @@ -28,13 +28,23 @@ #include #include #include +#include #include #include +#include +#include +#include +#include +#include #include #include +#include #include +#include +#include #include +#include #include #include #include @@ -167,6 +177,284 @@ private: opentelemetry::nostd::shared_ptr previous_; }; +namespace otel_sdk = opentelemetry::sdk::metrics; + +/** + * One collected time series set for a single metric name: the exact points the + * SDK produced, keyed by their full attribute (label) set. + * + * `PointAttributes` is an ordered map of label name -> owned value, so two + * different label sets are two different keys -- which is precisely what the + * "distinct outcomes must not collapse into one series" assertions check. + */ +using PointsByAttrs = std::map; + +/** + * Everything one Collect() pulled out of the SDK, keyed by metric name. + * A metric name that was never recorded is simply absent from the map, which + * is how the disabled/no-op tests prove nothing was emitted. + */ +using CollectedMetrics = std::map; + +/** + * A MetricReader that collects synchronously, on demand, in-process. + * + * The OTel SDK only hands aggregated points to a MetricReader, and the only + * pull entry point is the base class's `Collect(callback)`. This subclass adds + * nothing but a convenience wrapper that flattens one collection into a + * name -> (labels -> point) map a test can assert exact values against. + * + * Why not the shipped InMemoryMetricExporter? Its symbols live in a separate + * Conan archive (libopentelemetry_exporter_in_memory_metric.a) which is NOT on + * the xrpl_tests link line -- only libopentelemetry_exporter_in_memory.a (the + * SPAN exporter, used by SpanGuardScope.cpp) is. Subclassing MetricReader needs + * only libopentelemetry_metrics.a, which is already linked via the umbrella + * target, so this keeps the tests hermetic with no build-system change. + * + * Inheritance: + * + * +--------------+ + * | MetricReader | (SDK abstract pull interface) + * +------+-------+ + * | + * +------+-----------------+ + * | CollectOnDemandReader | synchronous, in-process collection + * +------------------------+ + * + * Example usage -- assert an exact counter value: + * @code + * CollectingProvider provider; + * FakeApp app; + * wire(app, true, provider.meter()); + * XRPL_METRIC_COUNTER_INC(app, "my_total", "desc"); + * auto data = provider.collect(); + * EXPECT_EQ(counterValue(data, "my_total", {}), 1); + * @endcode + * + * Example usage -- edge case: prove a metric was NOT emitted at all: + * @code + * wire(app, false, provider.meter()); // registry disabled + * XRPL_METRIC_COUNTER_INC(app, "my_total", "desc"); + * EXPECT_EQ(provider.collect().count("my_total"), 0u); + * @endcode + * + * @note Reports kCumulative temporality, so counter totals accumulate across + * repeated collect() calls rather than resetting -- the assertions below + * therefore collect exactly once per test unless stated otherwise. + * @note Test-only and not thread-safe by itself: collect() must not run + * concurrently with the macro calls it measures. Every test here is + * single-threaded, which satisfies that. + */ +class CollectOnDemandReader final : public otel_sdk::MetricReader +{ +public: + /** + * Pull one collection and flatten it to metric name -> labels -> point. + * @return The points produced by this collection. Metrics that were never + * recorded are absent from the returned map. + */ + [[nodiscard]] CollectedMetrics + collect() + { + CollectedMetrics out; + Collect([&out](otel_sdk::ResourceMetrics& resourceMetrics) { + for (auto const& scope : resourceMetrics.scope_metric_data_) + { + for (auto const& metric : scope.metric_data_) + { + for (auto const& point : metric.point_data_attr_) + out[metric.instrument_descriptor.name_][point.attributes] = + point.point_data; + } + } + return true; + }); + return out; + } + + /** + * Cumulative so counter totals are absolute, not per-interval deltas. + */ + [[nodiscard]] otel_sdk::AggregationTemporality + GetAggregationTemporality(otel_sdk::InstrumentType) const noexcept override + { + return otel_sdk::AggregationTemporality::kCumulative; + } + +private: + /** + * Nothing is buffered outside the SDK, so a flush always succeeds. + */ + bool + OnForceFlush(std::chrono::microseconds) noexcept override + { + return true; + } + + /** + * No exporter thread or socket to tear down. + */ + bool + OnShutDown(std::chrono::microseconds) noexcept override + { + return true; + } +}; + +/** + * Installs a bare SDK MeterProvider that has a CollectOnDemandReader attached, + * and restores the previous global provider on scope exit. + * + * This is ScopedBareProvider plus a reader: the plain ScopedBareProvider has no + * reader, so the SDK creates no storage and every recorded value is discarded. + * Attaching a reader is what makes the recorded values observable, which is the + * whole point of the value-asserting tests. + * + * @note The reader must be attached BEFORE any instrument is created, because + * Meter::RegisterSyncMetricStorage() binds storage to the collectors that exist + * at instrument-creation time. The constructor guarantees that ordering. + */ +class CollectingProvider +{ +public: + CollectingProvider() + { + reader_ = std::make_shared(); + // Attach the reader first, then publish the provider globally, so any + // instrument created afterwards gets storage wired to this reader. + sdkProvider_ = opentelemetry::sdk::metrics::MeterProviderFactory::Create(); + sdkProvider_->AddMetricReader(reader_); + previous_ = opentelemetry::metrics::Provider::GetMeterProvider(); + opentelemetry::metrics::Provider::SetMeterProvider( + opentelemetry::nostd::shared_ptr(sdkProvider_)); + } + + ~CollectingProvider() + { + opentelemetry::metrics::Provider::SetMeterProvider(previous_); + } + + CollectingProvider(CollectingProvider const&) = delete; + CollectingProvider& + operator=(CollectingProvider const&) = delete; + + /** + * @return A real meter whose instruments feed this provider's reader. + */ + [[nodiscard]] opentelemetry::nostd::shared_ptr + meter() const + { + return sdkProvider_->GetMeter("xrpld_test", "1.0.0"); + } + + /** + * @return The points from one synchronous collection. + */ + [[nodiscard]] CollectedMetrics + collect() const + { + return reader_->collect(); + } + +private: + /** + * The on-demand reader; owned jointly with the provider's collector. + */ + std::shared_ptr reader_; + + /** + * SDK provider that owns the metric storage the reader collects from. + */ + std::shared_ptr sdkProvider_; + + /** + * Previous global provider, restored on scope exit. + */ + opentelemetry::nostd::shared_ptr previous_; +}; + +/** + * Builds the attribute key for a single-label series, e.g. {"outcome","empty"}. + * @return A PointAttributes usable as a lookup key into PointsByAttrs. + */ +[[nodiscard]] otel_sdk::PointAttributes +attrs(std::string const& key, std::string const& value) +{ + otel_sdk::PointAttributes out; + out.SetAttribute(key, value); + return out; +} + +/** + * Builds the attribute key for a two-label series, e.g. site + outcome. + * @return A PointAttributes carrying exactly the two given labels. + */ +[[nodiscard]] otel_sdk::PointAttributes +attrs( + std::string const& key1, + std::string const& value1, + std::string const& key2, + std::string const& value2) +{ + otel_sdk::PointAttributes out; + out.SetAttribute(key1, value1); + out.SetAttribute(key2, value2); + return out; +} + +/** + * Reads the exact accumulated value of one counter time series. + * + * A uint64 Counter aggregates into a SumPointData holding an int64_t, so the + * value is unwrapped through both variants. Fails the calling test (via + * std::map::at) if the metric or the exact label set is missing -- absence is + * itself a defect for these assertions. + * + * @return The counter total for that exact label set. + */ +[[nodiscard]] std::int64_t +counterValue( + CollectedMetrics const& data, + std::string const& metric, + otel_sdk::PointAttributes const& labels) +{ + auto const& point = data.at(metric).at(labels); + auto const& sum = opentelemetry::nostd::get(point); + return opentelemetry::nostd::get(sum.value_); +} + +/** + * Reads the exact last observed value of one Int64ObservableGauge series. + * An observable gauge aggregates as last-value, holding an int64_t. + * @return The observed value for that exact label set (may be negative). + */ +[[nodiscard]] std::int64_t +gaugeValue( + CollectedMetrics const& data, + std::string const& metric, + otel_sdk::PointAttributes const& labels) +{ + auto const& point = data.at(metric).at(labels); + auto const& last = opentelemetry::nostd::get(point); + return opentelemetry::nostd::get(last.value_); +} + +/** + * Reads the sample count and sum of an unlabeled double Histogram. + * + * The macros record histograms with no labels, so the metric has exactly one + * series whose key is the empty attribute set. + * + * @return {count, sum} for the single unlabeled series. + */ +[[nodiscard]] std::pair +histogramCountAndSum(CollectedMetrics const& data, std::string const& metric) +{ + auto const& point = data.at(metric).at(otel_sdk::PointAttributes{}); + auto const& hist = opentelemetry::nostd::get(point); + return {hist.count_, opentelemetry::nostd::get(hist.sum_)}; +} + /** * Fetches a real (non-noop) meter from whatever provider is globally * installed -- inside a test this is the ScopedBareProvider's SDK provider. @@ -186,6 +474,23 @@ wire(FakeApp& app, bool enabled) app.registry().configure(enabled, bareMeter()); } +/** + * Builds a FakeApp wired to an explicitly supplied meter -- used with a + * CollectingProvider so the recorded values are collectable. + * + * @param app The duck-typed app the macros will be driven through. + * @param enabled What the macros' isEnabled() gate will observe. + * @param meter The meter the macros will create their instruments on. + */ +void +wire( + FakeApp& app, + bool enabled, + opentelemetry::nostd::shared_ptr meter) +{ + app.registry().configure(enabled, std::move(meter)); +} + } // namespace TEST(MetricMacros, counter_inc_creates_once_and_does_not_crash) @@ -335,4 +640,381 @@ TEST(MetricMacros, disabled_registry_is_noop) EXPECT_EQ(app.registry().meterCalls(), 0); } +// ----------------------------------------------------------------- +// Fresh-node sync diagnostics metrics. +// +// The tests below assert the EXACT recorded values and the EXACT label shape of +// the eight sync-diagnostics metrics, using a real SDK MeterProvider with a +// CollectOnDemandReader attached. They drive the same macros production uses, +// through the existing FakeApp, so what is proved is the real emit path: +// dns_resolve_total{outcome} OverlayImpl::reportDnsResolve +// dns_resolve_latency_ms (same) +// overlay_connect_total{outcome} ConnectAttempt::reportOutcome +// overlay_dial_latency_ms (same) +// handshake_negotiation_fail_total{reason} Handshake throwNegotiationFailure +// unl_fetch_total{site,outcome} ValidatorSite::reportFetchOutcome +// unl_quorum{metric} MetricsRegistry::registerUnlQuorumGauge +// clock_close_offset_seconds{metric} MetricsRegistry::registerClockSkewGauge +// +// The two observable gauges are registered directly on the SDK meter, mirroring +// the production callback shape, because the real MetricsRegistry's enabled path +// cannot be linked into this standalone binary (see the file header). +// ----------------------------------------------------------------- + +// dns_resolve_total must tally each outcome separately and dns_resolve_latency_ms +// must record every sample: one "resolved", two "empty", and two known latency +// samples whose count and sum are checked exactly. +TEST(MetricMacros, dns_resolve_records_exact_counts_and_latency) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + // outcome=resolved once; outcome=empty twice. Both go through the single + // production call site, so the label value is the only difference -- exactly + // how OverlayImpl::reportDnsResolve() emits it. + for (bool const resolved : {true, false, false}) + { + XRPL_METRIC_COUNTER_INC_LABELED( + app, + "dns_resolve_total", + "Peer hostname resolutions, by outcome", + {{"outcome", std::string(resolved ? "resolved" : "empty")}}); + } + + // Two latency samples with known values: 1.5 ms + 2.5 ms = 4.0 ms. + for (double const ms : {1.5, 2.5}) + { + XRPL_METRIC_HISTOGRAM_RECORD( + app, + "dns_resolve_latency_ms", + "Time taken to resolve a configured peer hostname, in milliseconds", + ms); + } + + auto const data = provider.collect(); + + // Exactly two series, one per outcome value -- the labeled counter must not + // collapse "resolved" and "empty" into a single series. + ASSERT_EQ(data.at("dns_resolve_total").size(), 2u); + EXPECT_EQ(counterValue(data, "dns_resolve_total", attrs("outcome", "resolved")), 1); + EXPECT_EQ(counterValue(data, "dns_resolve_total", attrs("outcome", "empty")), 2); + + // The label key is exactly "outcome" and it is the ONLY label present. + auto const& firstKey = data.at("dns_resolve_total").begin()->first; + ASSERT_EQ(firstKey.size(), 1u); + EXPECT_EQ(firstKey.begin()->first, "outcome"); + + // Histogram: exactly the two samples recorded, summing to exactly 4.0 ms. + auto const [count, sum] = histogramCountAndSum(data, "dns_resolve_latency_ms"); + EXPECT_EQ(count, 2u); + EXPECT_DOUBLE_EQ(sum, 4.0); + + // Unlabeled histogram: exactly one series, with an empty label set. + ASSERT_EQ(data.at("dns_resolve_latency_ms").size(), 1u); + EXPECT_TRUE(data.at("dns_resolve_latency_ms").begin()->first.empty()); +} + +// overlay_connect_total must keep every terminal outcome in its own series (a +// "timeout" must never be folded into "tcp_fail"), and overlay_dial_latency_ms +// must record each dial's duration. +TEST(MetricMacros, overlay_connect_records_exact_counts_per_outcome) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + // Three distinct outcomes with distinct multiplicities: connected x1, + // tcp_fail x3, timeout x2. + auto const bump = [&app](char const* outcome) { + XRPL_METRIC_COUNTER_INC_LABELED( + app, + "overlay_connect_total", + "Outbound peer connection attempts, by terminal outcome", + {{"outcome", std::string(outcome)}}); + }; + bump("connected"); + bump("tcp_fail"); + bump("tcp_fail"); + bump("tcp_fail"); + bump("timeout"); + bump("timeout"); + + // Dial latencies: 10.0 + 20.0 + 30.5 = 60.5 ms across 3 samples. + for (double const ms : {10.0, 20.0, 30.5}) + { + XRPL_METRIC_HISTOGRAM_RECORD( + app, + "overlay_dial_latency_ms", + "Time from starting an outbound peer dial to its terminal outcome, in milliseconds", + ms); + } + + auto const data = provider.collect(); + + // Three distinct outcomes stay three distinct series. + ASSERT_EQ(data.at("overlay_connect_total").size(), 3u); + EXPECT_EQ(counterValue(data, "overlay_connect_total", attrs("outcome", "connected")), 1); + EXPECT_EQ(counterValue(data, "overlay_connect_total", attrs("outcome", "tcp_fail")), 3); + EXPECT_EQ(counterValue(data, "overlay_connect_total", attrs("outcome", "timeout")), 2); + + // NEGATIVE: an outcome that was never emitted has no series at all, so the + // counts above are not an artifact of some catch-all series. + EXPECT_EQ(data.at("overlay_connect_total").count(attrs("outcome", "tls_fail")), 0u); + + auto const [count, sum] = histogramCountAndSum(data, "overlay_dial_latency_ms"); + EXPECT_EQ(count, 3u); + EXPECT_DOUBLE_EQ(sum, 60.5); +} + +// handshake_negotiation_fail_total must keep each rejection reason distinct; +// throwNegotiationFailure() routes every branch through one call site, so the +// `reason` label is the only thing separating them. +TEST(MetricMacros, handshake_negotiation_fail_keeps_reasons_distinct) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + // wrong_network twice, self_connection once, clock_skew once. + for (char const* reason : {"wrong_network", "wrong_network", "self_connection", "clock_skew"}) + { + XRPL_METRIC_COUNTER_INC_LABELED( + app, + "handshake_negotiation_fail_total", + "Peer handshake negotiations rejected, by reason", + {{"reason", std::string(reason)}}); + } + + auto const data = provider.collect(); + + ASSERT_EQ(data.at("handshake_negotiation_fail_total").size(), 3u); + EXPECT_EQ( + counterValue(data, "handshake_negotiation_fail_total", attrs("reason", "wrong_network")), + 2); + EXPECT_EQ( + counterValue(data, "handshake_negotiation_fail_total", attrs("reason", "self_connection")), + 1); + EXPECT_EQ( + counterValue(data, "handshake_negotiation_fail_total", attrs("reason", "clock_skew")), 1); + + // The label key is exactly "reason", and nothing else rides along. + auto const& firstKey = data.at("handshake_negotiation_fail_total").begin()->first; + ASSERT_EQ(firstKey.size(), 1u); + EXPECT_EQ(firstKey.begin()->first, "reason"); + + // NEGATIVE: a reason from the production set that was not emitted here has + // no series, proving reasons are not being merged. + EXPECT_EQ( + data.at("handshake_negotiation_fail_total").count(attrs("reason", "bad_public_key")), 0u); +} + +// unl_fetch_total carries TWO labels, so the series identity is the (site, +// outcome) PAIR. This is the highest-value signal of the eight: if the pair did +// not form the key, one failing list site would be masked by a healthy one. +TEST(MetricMacros, unl_fetch_total_keys_series_on_site_and_outcome_pair) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + auto const bump = [&app](char const* site, char const* outcome) { + XRPL_METRIC_COUNTER_INC_LABELED( + app, + "unl_fetch_total", + "Validator list fetch attempts, by site and outcome", + {{"site", std::string(site)}, {"outcome", std::string(outcome)}}); + }; + + constexpr char const* kSiteA = "https://a.example.com/vl.json"; + constexpr char const* kSiteB = "https://b.example.com/vl.json"; + + // Same outcome, two DIFFERENT sites -> must be two series. + bump(kSiteA, "accepted"); + bump(kSiteB, "accepted"); + bump(kSiteB, "accepted"); + // Same site, a SECOND outcome -> must be its own series. + bump(kSiteA, "fetch_error"); + bump(kSiteA, "fetch_error"); + bump(kSiteA, "fetch_error"); + + auto const data = provider.collect(); + + // Three distinct (site, outcome) pairs -> exactly three series. + ASSERT_EQ(data.at("unl_fetch_total").size(), 3u); + + // Two different sites with the SAME outcome are distinct, with their own + // exact counts -- site B's two successes do not inflate site A's one. + EXPECT_EQ( + counterValue(data, "unl_fetch_total", attrs("site", kSiteA, "outcome", "accepted")), 1); + EXPECT_EQ( + counterValue(data, "unl_fetch_total", attrs("site", kSiteB, "outcome", "accepted")), 2); + + // The SAME site with two outcomes is also distinct: A's 3 fetch_errors do + // not merge into A's 1 accepted. + EXPECT_EQ( + counterValue(data, "unl_fetch_total", attrs("site", kSiteA, "outcome", "fetch_error")), 3); + + // NEGATIVE: a pair never emitted (site B erroring) has no series, so the + // per-site failure signal is genuinely per-site. + EXPECT_EQ( + data.at("unl_fetch_total").count(attrs("site", kSiteB, "outcome", "fetch_error")), 0u); + + // Every series key carries exactly the two expected label names. + for (auto const& [labels, point] : data.at("unl_fetch_total")) + { + ASSERT_EQ(labels.size(), 2u); + EXPECT_EQ(labels.count("site"), 1u); + EXPECT_EQ(labels.count("outcome"), 1u); + } +} + +// unl_quorum observes two series from ONE callback, mirroring +// MetricsRegistry::registerUnlQuorumGauge(): trusted_keys and quorum under the +// `metric` label. Registered on the SDK meter directly because that production +// method cannot be linked here (see the file header). +TEST(MetricMacros, unl_quorum_gauge_observes_exact_trusted_keys_and_quorum) +{ + CollectingProvider const provider; + + // Values the callback will report, owned by the test exactly as the real + // registry reads them live from ValidatorList on each collection tick. + struct Observed + { + std::int64_t trustedKeys; + std::int64_t quorum; + }; + Observed observed{5, 4}; + + // Keep the instrument alive for the whole test: destroying the handle + // deregisters the callback (ObservableInstrument's destructor calls + // CleanupCallback), which is why the real registry holds it in a member. + auto gauge = provider.meter()->CreateInt64ObservableGauge( + "unl_quorum", "Trusted UNL key count vs required quorum"); + gauge->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto const* self = static_cast(state); + // Same Observe() form the production callback uses. + auto observe = [&](char const* name, std::int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + observe("trusted_keys", self->trustedKeys); + observe("quorum", self->quorum); + }, + &observed); + + auto const data = provider.collect(); + + // Exactly two series, one per `metric` value, with the exact values. + ASSERT_EQ(data.at("unl_quorum").size(), 2u); + EXPECT_EQ(gaugeValue(data, "unl_quorum", attrs("metric", "trusted_keys")), 5); + EXPECT_EQ(gaugeValue(data, "unl_quorum", attrs("metric", "quorum")), 4); + + // The label key is exactly "metric". + auto const& firstKey = data.at("unl_quorum").begin()->first; + ASSERT_EQ(firstKey.size(), 1u); + EXPECT_EQ(firstKey.begin()->first, "metric"); +} + +// clock_close_offset_seconds must carry a NEGATIVE offset through unchanged -- +// that is the real-world case (local clock ahead of the network) and the reason +// the production gauge is an Int64ObservableGauge rather than an unsigned +// counter. Mirrors MetricsRegistry::registerClockSkewGauge(). +TEST(MetricMacros, clock_skew_gauge_observes_exact_negative_offset) +{ + CollectingProvider const provider; + + // -3 s: this node's clock runs 3 seconds ahead of network close time. + std::int64_t offsetSeconds = -3; + + auto gauge = provider.meter()->CreateInt64ObservableGauge( + "clock_close_offset_seconds", "Network close time offset from the local clock, in seconds"); + gauge->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto const* value = static_cast(state); + auto observe = [&](char const* name, std::int64_t v) { + opentelemetry::nostd::get>>(result) + ->Observe(v, {{"metric", name}}); + }; + observe("offset", *value); + }, + &offsetSeconds); + + auto const data = provider.collect(); + + // Exactly one series, and the negative value survived the round trip: not + // clamped to 0, not reinterpreted as a large unsigned value. + ASSERT_EQ(data.at("clock_close_offset_seconds").size(), 1u); + EXPECT_EQ(gaugeValue(data, "clock_close_offset_seconds", attrs("metric", "offset")), -3); + + auto const& firstKey = data.at("clock_close_offset_seconds").begin()->first; + ASSERT_EQ(firstKey.size(), 1u); + EXPECT_EQ(firstKey.begin()->first, "metric"); +} + +// RUNTIME-DISABLED no-op proof: with the registry disabled, every one of the +// four sync-diagnostics counter/histogram families emits NOTHING -- no series +// exists for any of those metric names, and meter() is never consulted, so not +// even an instrument was created. This is the runtime counterpart to the +// compile-time no-op proof in MetricsRegistry.cpp. +TEST(MetricMacros, sync_diagnostics_metrics_emit_nothing_when_registry_disabled) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/false, provider.meter()); + + XRPL_METRIC_COUNTER_INC_LABELED( + app, + "dns_resolve_total", + "Peer hostname resolutions, by outcome", + {{"outcome", std::string("resolved")}}); + XRPL_METRIC_HISTOGRAM_RECORD( + app, + "dns_resolve_latency_ms", + "Time taken to resolve a configured peer hostname, in milliseconds", + 1.5); + XRPL_METRIC_COUNTER_INC_LABELED( + app, + "overlay_connect_total", + "Outbound peer connection attempts, by terminal outcome", + {{"outcome", std::string("connected")}}); + XRPL_METRIC_HISTOGRAM_RECORD( + app, + "overlay_dial_latency_ms", + "Time from starting an outbound peer dial to its terminal outcome, in milliseconds", + 10.0); + XRPL_METRIC_COUNTER_INC_LABELED( + app, + "handshake_negotiation_fail_total", + "Peer handshake negotiations rejected, by reason", + {{"reason", std::string("wrong_network")}}); + XRPL_METRIC_COUNTER_INC_LABELED( + app, + "unl_fetch_total", + "Validator list fetch attempts, by site and outcome", + {{"site", std::string("https://a.example.com/vl.json")}, + {"outcome", std::string("accepted")}}); + + auto const data = provider.collect(); + + // No series at all for any of the six metric names -- not a zero-valued + // series, but total absence: the instruments were never even created. + EXPECT_EQ(data.count("dns_resolve_total"), 0u); + EXPECT_EQ(data.count("dns_resolve_latency_ms"), 0u); + EXPECT_EQ(data.count("overlay_connect_total"), 0u); + EXPECT_EQ(data.count("overlay_dial_latency_ms"), 0u); + EXPECT_EQ(data.count("handshake_negotiation_fail_total"), 0u); + EXPECT_EQ(data.count("unl_fetch_total"), 0u); + + // Nothing else leaked in either: the collection is completely empty. + EXPECT_EQ(data.size(), 0u); + + // Cause, not just state: the isEnabled() gate short-circuited before the + // macro ever asked for a meter. + EXPECT_EQ(app.registry().meterCalls(), 0); +} + #endif // XRPL_ENABLE_TELEMETRY diff --git a/src/tests/libxrpl/telemetry/MetricsRegistry.cpp b/src/tests/libxrpl/telemetry/MetricsRegistry.cpp index a1000d4cae..36c9e44b0f 100644 --- a/src/tests/libxrpl/telemetry/MetricsRegistry.cpp +++ b/src/tests/libxrpl/telemetry/MetricsRegistry.cpp @@ -7,11 +7,24 @@ * - Synchronous instrument recording methods do not crash when disabled. * - Double stop() is safe. * - Destructor handles cleanup without crash. + * - Compile-time-disabled proof for the sync-diagnostics gauges: the whole + * async-gauge registration surface is compiled away, and a full disabled + * lifecycle never touches any ServiceRegistry service. * * NOTE: These tests only exercise the no-op path (telemetry disabled). * When XRPL_ENABLE_TELEMETRY is defined, MetricsRegistry.cpp pulls in * xrpld symbols that cannot be linked into this standalone test binary, * so the tests are compiled out. + * + * CONSEQUENCE for the sync-diagnostics gauges (`unl_quorum`, + * `clock_close_offset_seconds`): this file CANNOT assert an observed gauge + * value, because on this build the gauges do not exist -- their registration + * methods and the OTel instrument members are inside + * `#ifdef XRPL_ENABLE_TELEMETRY`, and there is no MeterProvider at all. What + * is provable here, and what the tests below assert, is the complementary + * half: that nothing is registered and no service is consulted. The exact + * observed values (trusted_keys=5, quorum=4, offset=-3) are asserted in + * MetricMacros.cpp, which is the file compiled when telemetry IS enabled. */ // When telemetry is globally enabled, MetricsRegistry.cpp requires xrpld @@ -32,6 +45,8 @@ #include #include #include +#include +#include using namespace xrpl; @@ -356,4 +371,94 @@ TEST_F(MetricsRegistryTest, destructor_calls_stop) // If we get here without crash, the destructor handled stop. } +// ----------------------------------------------------------------- +// Sync-diagnostics gauges: compile-time-disabled proof. +// +// `unl_quorum` reads ValidatorList::trustedKeyCount() and quorum(); +// `clock_close_offset_seconds` reads TimeKeeper::closeOffset(). Both are +// reached through the ServiceRegistry, and MockServiceRegistry::getValidators() +// / getTimeKeeper() THROW std::logic_error. So "no gauge callback ran" is +// directly observable here: had registerAsyncGauges() run and had a callback +// fired, one of those accessors would have thrown. +// +// Honest scope note: these tests do NOT assert an observed gauge value. On this +// build the gauges are not compiled at all (see the file header), so there is no +// value to read -- inventing one would be fiction. The value assertions live in +// MetricMacros.cpp. What is asserted here is the other half of the contract: +// registration is absent and no service is consulted. +// ----------------------------------------------------------------- + +// The observable-gauge registration surface is compiled OUT when telemetry is +// disabled: `meter()` -- the only accessor the gauges and the XRPL_METRIC_* +// macros use to reach the OTel SDK -- does not exist as a member at all. This is +// a compile-time assertion, so it fails the build (not the run) if the accessor +// ever escapes its #ifdef and drags the SDK into a telemetry-off build. +TEST_F(MetricsRegistryTest, disabled_build_exposes_no_meter_accessor) +{ + // Detects `registry.meter()` being callable. Under #ifndef + // XRPL_ENABLE_TELEMETRY it must not be, so the trait is false. + auto hasMeter = [](T* r) { return requires { r->meter(); }; }; + EXPECT_FALSE(hasMeter(static_cast(nullptr))); + + // The enable flag is still queryable and reports exactly false -- the class + // is a no-op, not an absent type. + telemetry::MetricsRegistry const registry(false, mockApp_, j_); + EXPECT_FALSE(registry.isEnabled()); +} + +// A full disabled lifecycle registers no gauge and therefore consults NO +// ServiceRegistry service. Asserting the cause, not just the absence of a crash: +// every MockServiceRegistry accessor a sync-diagnostics gauge would need throws, +// so reaching the end without an exception proves no callback ran. +TEST_F(MetricsRegistryTest, disabled_lifecycle_never_consults_gauge_services) +{ + telemetry::MetricsRegistry registry(false, mockApp_, j_); + + // start() is where registerAsyncGauges() -- and with it + // registerUnlQuorumGauge() / registerClockSkewGauge() -- would run. + EXPECT_NO_THROW(registry.start("http://localhost:4318/v1/metrics")); + + // detachCallbacks() is the shutdown hook the real gauges honour. It must be + // safe and idempotent even though there is nothing to detach. + EXPECT_NO_THROW(registry.detachCallbacks()); + EXPECT_NO_THROW(registry.detachCallbacks()); + + // Still disabled after a full start(): start() must not flip the flag. + EXPECT_FALSE(registry.isEnabled()); + + EXPECT_NO_THROW(registry.stop()); + + // Positive control: the mock DOES throw when a gauge-backing service is + // actually requested. Without this, "nothing threw" would be vacuous -- it + // could mean the mock is permissive rather than that no callback ran. + EXPECT_THROW(mockApp_.getValidators(), std::logic_error); + EXPECT_THROW(mockApp_.getTimeKeeper(), std::logic_error); +} + +// Even asking for enabled=true registers no sync-diagnostics gauge on a +// telemetry-off build. isEnabled() faithfully echoes the constructor argument +// (the flag lives outside the #ifdef), so the flag alone does NOT prove the +// gauges are inert -- the mock does: a full start()/stop() cycle with +// enabled=true still consults no service, so no callback was ever registered. +// Asserts the exact flag value on BOTH construction paths. +TEST_F(MetricsRegistryTest, enabled_flag_alone_registers_no_gauges_when_compiled_out) +{ + telemetry::MetricsRegistry enabledRequest(true, mockApp_, j_); + + // The flag is echoed back exactly, true not false: it is a plain member, + // not gated on XRPL_ENABLE_TELEMETRY. + EXPECT_TRUE(enabledRequest.isEnabled()); + + // Yet the whole lifecycle stays inert. If registerAsyncGauges() had run and + // registered registerUnlQuorumGauge()/registerClockSkewGauge(), a callback + // would reach getValidators()/getTimeKeeper() and throw std::logic_error. + EXPECT_NO_THROW(enabledRequest.start("http://localhost:4318/v1/metrics")); + EXPECT_NO_THROW(enabledRequest.detachCallbacks()); + EXPECT_NO_THROW(enabledRequest.stop()); + + // Contrast: enabled=false reports exactly false. + telemetry::MetricsRegistry const disabledRequest(false, mockApp_, j_); + EXPECT_FALSE(disabledRequest.isEnabled()); +} + #endif // !XRPL_ENABLE_TELEMETRY diff --git a/src/xrpld/app/misc/ValidatorList.h b/src/xrpld/app/misc/ValidatorList.h index 3f9039eab8..849ee36c6d 100644 --- a/src/xrpld/app/misc/ValidatorList.h +++ b/src/xrpld/app/misc/ValidatorList.h @@ -713,6 +713,27 @@ public: hash_set getTrustedMasterKeys() const; + /** + * get the number of trusted master public keys + * + * Takes the shared (read) lock and returns the size of the trusted + * master key set. O(1): unlike getTrustedMasterKeys() it copies + * nothing, so it is cheap enough to poll periodically. + * + * A count below quorum() means the node can never fully validate a + * ledger, which is the difference between "still syncing" and + * "will never sync". + * + * @par Thread Safety + * + * May be called concurrently, including from a metrics callback + * thread; it only takes the same shared lock as the other getters. + * + * @return the number of trusted master public keys + */ + std::size_t + trustedKeyCount() const; + /** * get the validator list threshold * @return the threshold diff --git a/src/xrpld/app/misc/ValidatorSite.h b/src/xrpld/app/misc/ValidatorSite.h index 7e7ad098eb..ea3bdd4433 100644 --- a/src/xrpld/app/misc/ValidatorSite.h +++ b/src/xrpld/app/misc/ValidatorSite.h @@ -18,6 +18,7 @@ #include #include #include +#include #include namespace xrpl { @@ -270,6 +271,36 @@ private: */ bool missingSite(std::scoped_lock const&); + + /** + * Record the terminal outcome of one validator list fetch attempt. + * + * Increments the `unl_fetch_total` counter with a `site` label (the + * URI as configured, so the series is stable across redirects) and an + * `outcome` label. Called exactly once per completed fetch attempt. + * Redirects are not terminal, so no outcome is recorded for them. + * + * A node with no reachable list site never gains trusted keys and can + * never reach quorum, so this counter is the signal that separates + * "misconfigured or unreachable list site" from other sync stalls. + * + * The single call to the metric macro lives here rather than at each + * outcome site because the macro caches its instrument in a + * function-local static, one per call site. + * + * @param siteIdx Index into sites_ of the site that was fetched. + * @param outcome Outcome label, e.g. "accepted", "fetch_error". + * @param sitesLock Proof that sitesMutex_ is held; sites_ is read here. + * + * @note Fetches are periodic (minutes apart), not a hot path, so the + * cost of building the label set per call is irrelevant. No-op when + * telemetry is compiled out or disabled. + */ + void + reportFetchOutcome( + std::size_t siteIdx, + std::string_view outcome, + std::scoped_lock const& sitesLock); }; } // namespace xrpl diff --git a/src/xrpld/app/misc/detail/ValidatorList.cpp b/src/xrpld/app/misc/detail/ValidatorList.cpp index a9e7156158..0d0b9f5241 100644 --- a/src/xrpld/app/misc/detail/ValidatorList.cpp +++ b/src/xrpld/app/misc/detail/ValidatorList.cpp @@ -2078,6 +2078,13 @@ ValidatorList::getTrustedMasterKeys() const return trustedMasterKeys_; } +std::size_t +ValidatorList::trustedKeyCount() const +{ + std::shared_lock const readLock{mutex_}; + return trustedMasterKeys_.size(); +} + std::size_t ValidatorList::getListThreshold() const { diff --git a/src/xrpld/app/misc/detail/ValidatorSite.cpp b/src/xrpld/app/misc/detail/ValidatorSite.cpp index 73ec3e0d6f..c40b00042c 100644 --- a/src/xrpld/app/misc/detail/ValidatorSite.cpp +++ b/src/xrpld/app/misc/detail/ValidatorSite.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -37,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -380,6 +382,24 @@ ValidatorSite::onTimer(std::size_t siteIdx, error_code const& ec) } } +void +ValidatorSite::reportFetchOutcome( + std::size_t siteIdx, + std::string_view outcome, + std::scoped_lock const& sitesLock) +{ + // loadedResource is set in Site's constructor and never reassigned, so + // it is always non-null. Unlike activeResource (reset once a fetch + // completes) it also keeps the URI exactly as configured, which keeps + // the time series stable when a site redirects. + XRPL_METRIC_COUNTER_INC_LABELED( + app_, + "unl_fetch_total", + "Validator list fetch attempts, by site and outcome", + {{"site", std::string(sites_[siteIdx].loadedResource->uri)}, + {"outcome", std::string(outcome)}}); +} + void ValidatorSite::parseJsonResponse( std::string const& res, @@ -496,6 +516,9 @@ ValidatorSite::parseJsonResponse( sites_[siteIdx].refreshInterval = refresh; sites_[siteIdx].nextRefresh = clock_type::now() + sites_[siteIdx].refreshInterval; } + + // Last on purpose: if anything above throws, the caller's catch counts it. + reportFetchOutcome(siteIdx, to_string(applyResult.bestDisposition()), sitesLock); } std::shared_ptr @@ -552,7 +575,7 @@ ValidatorSite::onSiteFetch( sites_[siteIdx].lastRequestEndpoint = endpoint; JLOG(j_.debug()) << "Got completion for " << sites_[siteIdx].activeResource->uri << " " << endpoint; - auto onError = [&](std::string const& errMsg, bool retry) { + auto onError = [&](std::string const& errMsg, bool retry, std::string_view outcome) { sites_[siteIdx].lastRefreshStatus.emplace( Site::Status{ .refreshed = clock_type::now(), @@ -560,6 +583,7 @@ ValidatorSite::onSiteFetch( .message = errMsg}); if (retry) sites_[siteIdx].nextRefresh = clock_type::now() + kErrorRetryInterval; + reportFetchOutcome(siteIdx, outcome, lockSites); // See if there's a copy saved locally from last time we // saw the list. @@ -569,7 +593,7 @@ ValidatorSite::onSiteFetch( { JLOG(j_.warn()) << "Problem retrieving from " << sites_[siteIdx].activeResource->uri << " " << endpoint << " " << ec.value() << ":" << ec.message(); - onError("fetch error", true); + onError("fetch error", true, "fetch_error"); } else { @@ -605,14 +629,14 @@ ValidatorSite::onSiteFetch( JLOG(j_.warn()) << "Request for validator list at " << sites_[siteIdx].activeResource->uri << " " << endpoint << " returned bad status: " << res.result_int(); - onError("bad result code", true); + onError("bad result code", true, "bad_status"); } } } catch (std::exception const& ex) { JLOG(j_.error()) << "Exception in " << __func__ << ": " << ex.what(); - onError(ex.what(), false); + onError(ex.what(), false, "parse_error"); } } sites_[siteIdx].activeResource.reset(); @@ -633,12 +657,15 @@ ValidatorSite::onTextFetch( { std::scoped_lock const lockSites{sitesMutex_}; { + // Both failures share one catch, so the label is set where detected. + std::string_view outcome = "parse_error"; try { if (ec) { JLOG(j_.warn()) << "Problem retrieving from " << sites_[siteIdx].activeResource->uri << " " << ec.value() << ": " << ec.message(); + outcome = "fetch_error"; throw std::runtime_error{"fetch error"}; } @@ -654,6 +681,7 @@ ValidatorSite::onTextFetch( .refreshed = clock_type::now(), .disposition = ListDisposition::Invalid, .message = ex.what()}); + reportFetchOutcome(siteIdx, outcome, lockSites); } sites_[siteIdx].activeResource.reset(); } diff --git a/src/xrpld/overlay/detail/ConnectAttempt.cpp b/src/xrpld/overlay/detail/ConnectAttempt.cpp index 064b4ecd3e..9225388d4a 100644 --- a/src/xrpld/overlay/detail/ConnectAttempt.cpp +++ b/src/xrpld/overlay/detail/ConnectAttempt.cpp @@ -1,3 +1,7 @@ +// cspell:ignore ISTOGRAM +// The all-caps macro name XRPL_METRIC_HISTOGRAM_RECORD trips cspell's +// compound-word splitter, which emits the subword "ISTOGRAM"; ignore it here. + #include #include @@ -9,6 +13,7 @@ #include #include #include +#include #include #include @@ -99,6 +104,12 @@ ConnectAttempt::stop() void ConnectAttempt::run() { + // Start the dial clock before any async operation is initiated. The + // constructor is too early (it only builds the object; the caller decides + // when to dial), and after setTimer() would be too late: setTimer() already + // queues a strand-bound wait that can read dialStart_ via reportOutcome(). + dialStart_ = std::chrono::steady_clock::now(); + setTimer(); stream_.next_layer().async_connect( @@ -107,6 +118,33 @@ ConnectAttempt::run() strand_, [self = shared_from_this()](error_code const& ec) { self->onConnect(ec); })); } +void +ConnectAttempt::reportOutcome(char const* outcome) +{ + if (outcomeReported_) + return; + outcomeReported_ = true; + + // Elapsed time is computed inline, not in a named local, so nothing is + // left unused when the macros compile away. Microseconds are converted to + // fractional milliseconds: `duration` would put a + // comma in a non-variadic macro argument, which the preprocessor splits. + XRPL_METRIC_HISTOGRAM_RECORD( + app_, + "overlay_dial_latency_ms", + "Time from starting an outbound peer dial to its terminal outcome, in milliseconds", + std::chrono::duration_cast( + std::chrono::steady_clock::now() - dialStart_) + .count() / + 1000.0); + + XRPL_METRIC_COUNTER_INC_LABELED( + app_, + "overlay_connect_total", + "Outbound peer connection attempts, by terminal outcome", + {{"outcome", std::string(outcome)}}); +} + //------------------------------------------------------------------------------ void @@ -192,6 +230,7 @@ ConnectAttempt::onTimer(error_code ec) close(); return; } + reportOutcome("timeout"); fail("Timeout"); } @@ -205,6 +244,7 @@ ConnectAttempt::onConnect(error_code ec) if (ec == boost::asio::error::operation_aborted) return; + reportOutcome("tcp_fail"); fail("onConnect", ec); return; } @@ -216,6 +256,7 @@ ConnectAttempt::onConnect(error_code ec) socket_.local_endpoint(ec); if (ec) { + reportOutcome("tcp_fail"); fail("onConnect", ec); return; } @@ -241,6 +282,7 @@ ConnectAttempt::onHandshake(error_code ec) if (ec == boost::asio::error::operation_aborted) return; + reportOutcome("tls_fail"); fail("onHandshake", ec); return; } @@ -248,6 +290,7 @@ ConnectAttempt::onHandshake(error_code ec) auto const localEndpoint = socket_.local_endpoint(ec); if (ec) { + reportOutcome("tls_fail"); fail("onHandshake", ec); return; } @@ -255,6 +298,7 @@ ConnectAttempt::onHandshake(error_code ec) if (!overlay_.peerFinder().onConnected( slot_, beast::IPAddressConversion::fromAsio(localEndpoint))) { + reportOutcome("tls_fail"); fail("Duplicate connection"); return; } @@ -262,6 +306,7 @@ ConnectAttempt::onHandshake(error_code ec) auto const sharedValue = makeSharedValue(*streamPtr_, journal_); if (!sharedValue) { + reportOutcome("tls_fail"); close(); // makeSharedValue logs return; } @@ -303,6 +348,7 @@ ConnectAttempt::onWrite(error_code ec) if (ec == boost::asio::error::operation_aborted) return; + reportOutcome("upgrade_fail"); fail("onWrite", ec); return; } @@ -340,6 +386,7 @@ ConnectAttempt::onRead(error_code ec) return; } + reportOutcome("upgrade_fail"); fail("onRead", ec); return; } @@ -359,6 +406,7 @@ ConnectAttempt::onShutdown(error_code ec) if (ec != boost::asio::error::eof) { + reportOutcome("upgrade_fail"); fail("onShutdown", ec); return; } @@ -406,10 +454,13 @@ ConnectAttempt::processResponse() } } + // The HTTP-503 block above only harvests redirect hints and falls through, + // so this is the first terminal point for a response we cannot upgrade. if (!OverlayImpl::isPeerUpgrade(response_)) { JLOG(journal_.info()) << "Unable to upgrade to peer protocol: " << response_.result() << " (" << response_.reason() << ")"; + reportOutcome("upgrade_fail"); close(); return; } @@ -426,6 +477,7 @@ ConnectAttempt::processResponse() if (!negotiatedProtocol) { + reportOutcome("upgrade_fail"); fail("processResponse: Unable to negotiate protocol version"); return; } @@ -434,6 +486,7 @@ ConnectAttempt::processResponse() auto const sharedValue = makeSharedValue(*streamPtr_, journal_); if (!sharedValue) { + reportOutcome("upgrade_fail"); close(); // makeSharedValue logs return; } @@ -464,6 +517,7 @@ ConnectAttempt::processResponse() overlay_.peerFinder().activate(slot_, publicKey, static_cast(member)); if (result != PeerFinder::Result::Success) { + reportOutcome("upgrade_fail"); fail("Outbound " + std::string(to_string(result))); return; } @@ -481,9 +535,14 @@ ConnectAttempt::processResponse() overlay_); overlay_.addActive(peer); + + // Only after addActive succeeds is the dial genuinely complete. If + // anything above threw, the catch below reports the failure instead. + reportOutcome("connected"); } catch (std::exception const& e) { + reportOutcome("upgrade_fail"); fail(std::string("Handshake failure (") + e.what() + ")"); return; } diff --git a/src/xrpld/overlay/detail/ConnectAttempt.h b/src/xrpld/overlay/detail/ConnectAttempt.h index d7836e3c84..56c29771d0 100644 --- a/src/xrpld/overlay/detail/ConnectAttempt.h +++ b/src/xrpld/overlay/detail/ConnectAttempt.h @@ -52,6 +52,20 @@ private: std::shared_ptr slot_; request_type req_; + /** + * When the dial began, set at the top of run() before any async + * operation is started. Base for the `overlay_dial_latency_ms` + * measurement. + */ + std::chrono::steady_clock::time_point dialStart_; + + /** + * True once this attempt's outcome has been reported, so the first + * (most specific) terminal path wins and each attempt is counted at + * most once. Not atomic on purpose -- see reportOutcome(). + */ + bool outcomeReported_{false}; + public: ConnectAttempt( Application& app, @@ -98,6 +112,57 @@ private: void processResponse(); + /** + * Record how this outbound dial ended, exactly once per attempt. + * + * Every terminal path in the dial state machine funnels here, so the + * emit code and its cached instruments live in one place instead of + * being repeated per branch. The first call wins: later calls return + * immediately, which keeps the reported outcome the most specific one + * (e.g. a "timeout" is not later overwritten by the "tcp_fail" that + * the cancelled socket operation reports). + * + * Emits: + * - `overlay_dial_latency_ms` histogram, no labels + * - `overlay_connect_total` counter, label `outcome` + * + * Dial state machine and where each outcome is reported: + * + * run() ---- dialStart_ = now + * | + * +-- onTimer ................................. "timeout" + * +-- onConnect (connect / local_endpoint) . "tcp_fail" + * +-- onHandshake (TLS / slot / shared value) "tls_fail" + * +-- onWrite / onRead / onShutdown ............. "upgrade_fail" + * +-- processResponse + * +-- bad status / protocol / activate ... "upgrade_fail" + * +-- PeerImp created + addActive ........ "connected" + * + * @param outcome One of "connected", "tcp_fail", "tls_fail", + * "upgrade_fail", "timeout". A string literal, so no allocation + * happens on the caller side. + * + * @note Per-connection path: one dial per outbound peer, so this is not + * a hot loop. + * @note Thread safety: every completion handler that calls this is bound + * through `bind_executor(strand_, ...)`, so all callers of this + * method run on the same strand and `outcomeReported_` needs no + * atomic or mutex. `dialStart_` is written once in run(), before + * any async operation is initiated, so it is safely visible to + * those handlers even though run() itself may execute on the + * calling thread rather than the strand. + * @note Known limitation: an attempt torn down by overlay shutdown + * mid-dial (stop() -> close(), or the operation_aborted early + * returns) is deliberately not counted -- it has no network + * outcome to attribute. + * @note MetricsRegistry is already started when this runs: + * ApplicationImp::setup() calls startTelemetry() before + * ApplicationImp::start() calls overlay_->start(). No-op when + * telemetry is compiled out or disabled at runtime. + */ + void + reportOutcome(char const* outcome); + template static boost::asio::ip::tcp::endpoint parseEndpoint(std::string const& s, boost::system::error_code& ec) diff --git a/src/xrpld/overlay/detail/Handshake.cpp b/src/xrpld/overlay/detail/Handshake.cpp index a860d2d604..45784f7c45 100644 --- a/src/xrpld/overlay/detail/Handshake.cpp +++ b/src/xrpld/overlay/detail/Handshake.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -41,6 +42,7 @@ #include #include #include +#include // VFALCO Shouldn't we have to include the OpenSSL // headers or something for SSL_get_finished? @@ -226,6 +228,51 @@ buildHandshake( } } +namespace { + +/** + * Count one handshake negotiation failure, then throw as before. + * + * verifyHandshake() rejects a peer by throwing, from more than a dozen + * distinct branches. Routing every one of them through this helper keeps a + * single emit site -- which matters beyond avoiding duplicated code, because + * the metric macro caches its instrument in a function-local `static`: one + * call site means one instrument shared by all reasons, distinguished only by + * the `reason` label. + * + * Emits `handshake_negotiation_fail_total`, label `reason`. + * + * The thrown message is passed through unchanged so callers, which log + * `e.what()`, see exactly the text they saw before. The reason label is a + * separate, low-cardinality machine-readable value; it is never derived from + * the message. + * + * @param app Provides the metrics registry. + * @param reason Short snake_case cause, e.g. "wrong_network". A string + * literal from a fixed set, so cardinality stays bounded. + * @param message The exact message to throw, byte-identical to the previous + * behaviour. + * + * @note Per-connection handshake path -- one call per rejected peer, not a + * hot loop. + * @note No-op when telemetry is compiled out or disabled at runtime; the + * macro carries that guard, so there is no `#ifdef` here. + * @note Always throws; it never returns to its caller. + */ +[[noreturn]] void +throwNegotiationFailure(Application& app, char const* reason, std::string message) +{ + XRPL_METRIC_COUNTER_INC_LABELED( + app, + "handshake_negotiation_fail_total", + "Peer handshake negotiations rejected, by reason", + {{"reason", std::string(reason)}}); + + throw std::runtime_error(std::move(message)); +} + +} // namespace + PublicKey verifyHandshake( boost::beast::http::fields const& headers, @@ -238,7 +285,7 @@ verifyHandshake( if (auto const iter = headers.find("Server-Domain"); iter != headers.end()) { if (!isProperlyFormedTomlDomain(iter->value())) - throw std::runtime_error("Invalid server domain"); + throwNegotiationFailure(app, "invalid_server_domain", "Invalid server domain"); } if (auto const iter = headers.find("Network-ID"); iter != headers.end()) @@ -246,15 +293,15 @@ verifyHandshake( std::uint32_t nid = 0; if (!beast::lexicalCastChecked(nid, iter->value())) - throw std::runtime_error("Invalid peer network identifier"); + throwNegotiationFailure(app, "invalid_network_id", "Invalid peer network identifier"); if (networkID && nid != *networkID) - throw std::runtime_error("Peer is on a different network"); + throwNegotiationFailure(app, "wrong_network", "Peer is on a different network"); } if (auto const iter = headers.find("Network-Time"); iter != headers.end()) { - auto const netTime = [str = iter->value()]() -> TimeKeeper::time_point { + auto const netTime = [str = iter->value(), &app]() -> TimeKeeper::time_point { TimeKeeper::duration::rep val = 0; if (beast::lexicalCastChecked(val, str)) @@ -262,7 +309,7 @@ verifyHandshake( // It's not an error for the header field to not be present but if // it is present and it contains junk data, that is an error. - throw std::runtime_error("Invalid peer clock timestamp"); + throwNegotiationFailure(app, "invalid_clock_timestamp", "Invalid peer clock timestamp"); }(); using namespace std::chrono; @@ -282,10 +329,10 @@ verifyHandshake( auto const offset = calculateOffset(netTime, ourTime); if (abs(offset) > tolerance) - throw std::runtime_error("Peer clock is too far off"); + throwNegotiationFailure(app, "clock_skew", "Peer clock is too far off"); } - PublicKey const publicKey = [&headers] { + PublicKey const publicKey = [&headers, &app] { if (auto const iter = headers.find("Public-Key"); iter != headers.end()) { auto pk = parseBase58(TokenType::NodePublic, iter->value()); @@ -293,13 +340,14 @@ verifyHandshake( if (pk) { if (publicKeyType(*pk) != KeyType::Secp256k1) - throw std::runtime_error("Unsupported public key type"); + throwNegotiationFailure( + app, "unsupported_key_type", "Unsupported public key type"); return *pk; } } - throw std::runtime_error("Bad node public key"); + throwNegotiationFailure(app, "bad_public_key", "Bad node public key"); }(); // This check gets two birds with one stone: @@ -312,16 +360,16 @@ verifyHandshake( auto const iter = headers.find("Session-Signature"); if (iter == headers.end()) - throw std::runtime_error("No session signature specified"); + throwNegotiationFailure(app, "no_session_signature", "No session signature specified"); auto sig = base64Decode(iter->value()); if (!verifyDigest(publicKey, sharedValue, makeSlice(sig), false)) - throw std::runtime_error("Failed to verify session"); + throwNegotiationFailure(app, "session_verify_failed", "Failed to verify session"); } if (publicKey == app.nodeIdentity().first) - throw std::runtime_error("Self connection"); + throwNegotiationFailure(app, "self_connection", "Self connection"); if (auto const iter = headers.find("Local-IP"); iter != headers.end()) { @@ -329,11 +377,13 @@ verifyHandshake( auto const localIp = boost::asio::ip::make_address(std::string_view(iter->value()), ec); if (ec) - throw std::runtime_error("Invalid Local-IP"); + throwNegotiationFailure(app, "invalid_local_ip", "Invalid Local-IP"); if (beast::IP::isPublic(remote) && remote != localIp) { - throw std::runtime_error( + throwNegotiationFailure( + app, + "local_ip_mismatch", "Incorrect Local-IP: " + remote.to_string() + " instead of " + localIp.to_string()); } } @@ -344,7 +394,7 @@ verifyHandshake( auto const remoteIp = boost::asio::ip::make_address(std::string_view(iter->value()), ec); if (ec) - throw std::runtime_error("Invalid Remote-IP"); + throwNegotiationFailure(app, "invalid_remote_ip", "Invalid Remote-IP"); if (beast::IP::isPublic(remote) && !beast::IP::isUnspecified(publicIp)) { @@ -352,9 +402,11 @@ verifyHandshake( // from some other IP. if (remoteIp != publicIp) { - throw std::runtime_error( + throwNegotiationFailure( + app, + "remote_ip_mismatch", "Incorrect Remote-IP: " + publicIp.to_string() + " instead of " + - remoteIp.to_string()); + remoteIp.to_string()); } } } diff --git a/src/xrpld/overlay/detail/OverlayImpl.cpp b/src/xrpld/overlay/detail/OverlayImpl.cpp index 6a6a6edace..dd2481fb23 100644 --- a/src/xrpld/overlay/detail/OverlayImpl.cpp +++ b/src/xrpld/overlay/detail/OverlayImpl.cpp @@ -1,3 +1,7 @@ +// cspell:ignore ISTOGRAM +// The all-caps macro name XRPL_METRIC_HISTOGRAM_RECORD trips cspell's +// compound-word splitter, which emits the subword "ISTOGRAM"; ignore it here. + #include #include @@ -15,6 +19,7 @@ #include #include #include +#include #include #include @@ -536,9 +541,16 @@ OverlayImpl::start() bootstrapIps.emplace_back("hub.xrpl-commons.org 51235"); } + // Timestamp the resolve request so the handler can report how long the + // whole batch resolve took for each name it reports back. + auto const bootstrapDnsStart = std::chrono::steady_clock::now(); + resolver_.resolve( bootstrapIps, - [this](std::string const& name, std::vector const& addresses) { + [this, bootstrapDnsStart]( + std::string const& name, std::vector const& addresses) { + reportDnsResolve(bootstrapDnsStart, !addresses.empty()); + std::vector ips; ips.reserve(addresses.size()); for (auto const& addr : addresses) @@ -561,9 +573,14 @@ OverlayImpl::start() // Add the ips_fixed from the xrpld.cfg file if (!app_.config().standalone() && !app_.config().ipsFixed.empty()) { + auto const fixedDnsStart = std::chrono::steady_clock::now(); + resolver_.resolve( app_.config().ipsFixed, - [this](std::string const& name, std::vector const& addresses) { + [this, fixedDnsStart]( + std::string const& name, std::vector const& addresses) { + reportDnsResolve(fixedDnsStart, !addresses.empty()); + std::vector ips; ips.reserve(addresses.size()); @@ -590,6 +607,29 @@ OverlayImpl::start() timer->asyncWait(); } +void +OverlayImpl::reportDnsResolve(std::chrono::steady_clock::time_point start, bool resolved) +{ + // Elapsed time is computed inline, not in a named local, so nothing is + // left unused when the macros compile away. Microseconds are converted to + // fractional milliseconds: `duration` would put a + // comma in a non-variadic macro argument, which the preprocessor splits. + XRPL_METRIC_HISTOGRAM_RECORD( + app_, + "dns_resolve_latency_ms", + "Time taken to resolve a configured peer hostname, in milliseconds", + std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count() / + 1000.0); + + XRPL_METRIC_COUNTER_INC_LABELED( + app_, + "dns_resolve_total", + "Peer hostname resolutions, by outcome", + {{"outcome", std::string(resolved ? "resolved" : "empty")}}); +} + void OverlayImpl::stop() { diff --git a/src/xrpld/overlay/detail/OverlayImpl.h b/src/xrpld/overlay/detail/OverlayImpl.h index 092ac86a6d..a9ec637ee9 100644 --- a/src/xrpld/overlay/detail/OverlayImpl.h +++ b/src/xrpld/overlay/detail/OverlayImpl.h @@ -565,6 +565,41 @@ private: void deleteIdlePeers(); + /** + * Emit the DNS resolve outcome and latency metrics for one resolved name. + * + * Shared by both `resolver_.resolve(...)` completion handlers in + * OverlayImpl::start() (the bootstrap `[ips]` batch and the `[ips_fixed]` + * batch), so the emit code exists exactly once. This matters beyond + * de-duplication: the metric macros cache their instrument in a + * function-local `static`, so keeping a single call site also keeps a + * single shared instrument for both batches. + * + * Emits: + * - `dns_resolve_latency_ms` histogram, no labels + * - `dns_resolve_total` counter, label `outcome` = "resolved" | "empty" + * + * ResolverAsio invokes its handler with the same signature for success and + * failure; on failure the address list is empty. So an empty list is the + * only failure signal available to the caller, and `resolved` must be + * derived from it. + * + * @param start When the resolve request was submitted. The measured latency + * therefore spans the whole batch resolve for that name, not just + * the final DNS round trip. + * @param resolved True when the resolver returned at least one address. + * + * @note Called once per resolved name during overlay startup only + * (bootstrap plus fixed peers) -- this is not a hot loop. + * @note MetricsRegistry is already started when this runs: + * ApplicationImp::setup() calls startTelemetry() before + * ApplicationImp::start() calls overlay_->start(). + * @note No-op when telemetry is compiled out or disabled at runtime; the + * macro carries that guard, so this method needs no `#ifdef`. + */ + void + reportDnsResolve(std::chrono::steady_clock::time_point start, bool resolved); + private: struct TrafficGauges { diff --git a/src/xrpld/telemetry/MetricsRegistry.cpp b/src/xrpld/telemetry/MetricsRegistry.cpp index d2269a8d25..0023ef4034 100644 --- a/src/xrpld/telemetry/MetricsRegistry.cpp +++ b/src/xrpld/telemetry/MetricsRegistry.cpp @@ -480,6 +480,8 @@ MetricsRegistry::registerAsyncGauges() registerStorageDetailGauge(); registerValidationAgreementGauge(); registerValidationTotalsCounters(); + registerUnlQuorumGauge(); + registerClockSkewGauge(); } void @@ -1458,6 +1460,82 @@ MetricsRegistry::registerValidationTotalsCounters() this); } +void +MetricsRegistry::registerUnlQuorumGauge() +{ + // --- Sync diagnostics: trusted UNL size against required quorum --- + // validator_health already exports the quorum on its own; pairing it + // with the trusted-key count in one instrument is what makes the + // "can this node ever validate?" comparison a single query. + unlQuorumGauge_ = meter_->CreateInt64ObservableGauge( + "unl_quorum", "Trusted UNL key count vs required quorum"); + unlQuorumGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* name, int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + auto& validators = app.getValidators(); + + // Trusted master keys currently in effect. Zero means no + // usable UNL: quorum can never be met. + observe("trusted_keys", static_cast(validators.trustedKeyCount())); + + // Validations required for a ledger to be fully validated. + observe("quorum", static_cast(validators.quorum())); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerClockSkewGauge() +{ + // --- Sync diagnostics: network close-time offset --- + // A persistent offset shows the local clock disagrees with the + // network, which delays consensus participation. server_info hides + // this below 60 s, so export it continuously instead. + clockSkewGauge_ = meter_->CreateInt64ObservableGauge( + "clock_close_offset_seconds", "Network close time offset from the local clock, in seconds"); + clockSkewGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* name, int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{"metric", name}}); + }; + + // Negative when the local clock runs ahead of the network. + observe("offset", static_cast(app.getTimeKeeper().closeOffset().count())); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + #endif // XRPL_ENABLE_TELEMETRY // ----------------------------------------------------------------- diff --git a/src/xrpld/telemetry/MetricsRegistry.h b/src/xrpld/telemetry/MetricsRegistry.h index 17f0dc3718..9f3125d298 100644 --- a/src/xrpld/telemetry/MetricsRegistry.h +++ b/src/xrpld/telemetry/MetricsRegistry.h @@ -554,6 +554,14 @@ private: * Observable gauge for server-level health metrics (state, uptime, peers, etc.). */ opentelemetry::nostd::shared_ptr serverInfoGauge_; + /** + * Observable gauge for trusted UNL key count against the required quorum. + */ + opentelemetry::nostd::shared_ptr unlQuorumGauge_; + /** + * Observable gauge for the network close-time offset (local clock skew). + */ + opentelemetry::nostd::shared_ptr clockSkewGauge_; /** * Observable gauge for build version info (label-based, value=1). */ @@ -735,7 +743,44 @@ private: registerValidationAgreementGauge(); // Task 7.15 void registerValidationTotalsCounters(); // gap-fill: lifetime agree/miss _total -#endif // XRPL_ENABLE_TELEMETRY + + /** + * Register the `unl_quorum` gauge. + * + * Observes two series under the `metric` attribute: + * `trusted_keys` (ValidatorList::trustedKeyCount()) and `quorum` + * (ValidatorList::quorum()). Both are cheap accessors — one shared + * lock and one atomic load. + * + * `trusted_keys < quorum` means the node can never fully validate a + * ledger, so it will sit in `syncing` until the UNL is fixed. That + * makes this the first place to look when a node never leaves + * `syncing`. + * + * @note Pulled on the OTel reader thread (~10 s tick); does no work + * on any hot path. + */ + void + registerUnlQuorumGauge(); // sync diagnostics: UNL vs quorum + + /** + * Register the `clock_close_offset_seconds` gauge. + * + * Observes one series, `offset`, from + * TimeKeeper::closeOffset(): the seconds this node's notion of + * network close time is displaced from its own wall clock. + * + * The value MAY BE NEGATIVE, meaning the local clock runs ahead of + * the network. Whole-second resolution is all the signal carries, + * since that is the unit TimeKeeper stores. + * + * @note `server_info` only reports this field once |offset| >= 60 s + * (NetworkOPs), so this gauge is the first continuous export of it. + * Pulled on the OTel reader thread (~10 s tick); one atomic load. + */ + void + registerClockSkewGauge(); // sync diagnostics: close-time offset +#endif // XRPL_ENABLE_TELEMETRY }; } // namespace telemetry