From 96914b9f40bb2b12fc82c2bbc86844178e966502 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Fri, 24 Jul 2026 21:06:05 +0100 Subject: [PATCH] feat(telemetry): scaffold fresh-node sync diagnostics (WP-A0) Adds the anchors the sync-diagnostics signals attach to, with no signals emitted yet: - New "Ledger Sync Health" dashboard (uid ledger-sync-health) with the standard template-variable block copied from an existing board, plus empty "Bootstrap (Domain 0)" and "Sync pipeline" rows. - Signal index section in the data-collection reference, an operator-flow stub in the telemetry runbook, and a glossary anchor. - A sync_diagnostics group in expected_metrics.json and a matching assertion helper in validate_telemetry.py so CI fails when a signal regresses to absent. Also registers the new dashboard uid with the harness so the board is covered by validation. Co-Authored-By: Claude Opus 5 (1M context) --- .../09-data-collection-reference.md | 24 ++ .../dashboards/ledger-sync-health.json | 235 ++++++++++++++++++ .../telemetry/workload/expected_metrics.json | 7 +- .../telemetry/workload/validate_telemetry.py | 85 ++++++- docs/telemetry-glossary.md | 10 + docs/telemetry-runbook.md | 27 ++ 6 files changed, 382 insertions(+), 6 deletions(-) create mode 100644 docker/telemetry/grafana/dashboards/ledger-sync-health.json diff --git a/OpenTelemetryPlan/09-data-collection-reference.md b/OpenTelemetryPlan/09-data-collection-reference.md index f0e3f3721c..769e909ff2 100644 --- a/OpenTelemetryPlan/09-data-collection-reference.md +++ b/OpenTelemetryPlan/09-data-collection-reference.md @@ -1379,3 +1379,27 @@ prefix=xrpld | `trace_consensus` | `1` | `consensus.*` spans | | `trace_ledger` | `1` | `ledger.*` spans | | `trace_peer` | `1` | `peer.*` spans (high volume) | + +--- + +## Fresh-node sync diagnostics + +Signals that explain why a freshly-started node is slow to reach, or never +reaches, a validated ledger (`server_state=full`). Two groups: pre-quorum +bootstrap (DNS, dial, handshake, UNL/quorum, clock skew) and the post-peering +ledger/tx-set acquire pipeline. + +Rendered by the **Ledger Sync Health** dashboard +(uid `ledger-sync-health`, rows `Bootstrap (Domain 0)` and `Sync pipeline`). +Operator flow: [telemetry-runbook.md](../docs/telemetry-runbook.md) +"Diagnosing slow/stuck fresh sync". Terms: +[telemetry-glossary.md](../docs/telemetry-glossary.md) +"Fresh-node sync diagnostics". + +The table below is the single index for these signals; one row is added per +signal as it lands. `Type` is the instrument kind (counter / gauge / histogram / +span / span attr), `Emit site` the owning source file, and `Panel` the dashboard +panel that renders it. + +| Signal | Type | Emit site | Panel | Meaning | +| ------ | ---- | --------- | ----- | ------- | diff --git a/docker/telemetry/grafana/dashboards/ledger-sync-health.json b/docker/telemetry/grafana/dashboards/ledger-sync-health.json new file mode 100644 index 0000000000..33efa14db9 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/ledger-sync-health.json @@ -0,0 +1,235 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "description": "What this shows: Fresh-node ledger-sync diagnostics: pre-quorum bootstrap (Domain 0) and the ledger/tx-set acquire pipeline. \u2014 Use it to: Find out why a freshly started node is slow to reach, or never reaches, server_state full.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "id": 1, + "type": "row", + "title": "Bootstrap (Domain 0)", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "id": 2, + "type": "row", + "title": "Sync pipeline", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "panels": [] + } + ], + "schemaVersion": 39, + "tags": ["ledger", "sync", "diagnostics"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by xrpld node (service.instance.id)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Ledger Sync Health", + "uid": "ledger-sync-health", + "refresh": "30s" +} diff --git a/docker/telemetry/workload/expected_metrics.json b/docker/telemetry/workload/expected_metrics.json index 0c1926e878..dc8eb486fa 100644 --- a/docker/telemetry/workload/expected_metrics.json +++ b/docker/telemetry/workload/expected_metrics.json @@ -127,6 +127,10 @@ "description": "External dashboard parity: storage detail metrics (MetricsRegistry).", "metrics": ["storage_detail{metric=\"nudb_bytes\"}"] }, + "sync_diagnostics": { + "description": "Fresh-node sync diagnostics (MetricsRegistry native metrics). Bootstrap (Domain 0) and acquire-pipeline signals rendered by the ledger-sync-health dashboard. Empty until the emitting work packages land; names are appended one per signal.", + "metrics": [] + }, "grafana_dashboards": { "description": "All Grafana dashboards that must render data (UIDs as provisioned on disk under docker/telemetry/grafana/dashboards/).", "uids": [ @@ -143,7 +147,8 @@ "network-traffic", "rpc-pathfinding", "overlay-traffic-detail", - "ledger-data-sync" + "ledger-data-sync", + "ledger-sync-health" ] } } diff --git a/docker/telemetry/workload/validate_telemetry.py b/docker/telemetry/workload/validate_telemetry.py index 40f7e507a8..c5ea8bd815 100644 --- a/docker/telemetry/workload/validate_telemetry.py +++ b/docker/telemetry/workload/validate_telemetry.py @@ -8,9 +8,11 @@ and Grafana (dashboards) APIs to produce a pass/fail report. Validation categories: 1. Span validation — All 16+ span types present with required attributes 2. Metric validation — SpanMetrics, StatsD, and Phase 9 metrics are non-zero - 3. Log-trace correlation — Loki logs contain trace_id/span_id fields - 4. Dashboard validation — All 14 Grafana dashboards render data - 5. External parity — Span attrs, metric existence, and value sanity for + 3. Sync diagnostics — Fresh-node sync signals (bootstrap + acquire + pipeline) declared in the "sync_diagnostics" group + 4. Log-trace correlation — Loki logs contain trace_id/span_id fields + 5. Dashboard validation — All 15 Grafana dashboards render data + 6. External parity — Span attrs, metric existence, and value sanity for external dashboard parity (validator-health, peer-quality, node-health) @@ -62,6 +64,12 @@ EXPECTED_METRICS_FILE = SCRIPT_DIR / "expected_metrics.json" METRIC_POLL_TIMEOUT_SEC = 45.0 METRIC_POLL_INTERVAL_SEC = 5.0 +# Group key in expected_metrics.json holding the fresh-node sync-diagnostics +# metrics (bootstrap + acquire pipeline). Owned by +# assert_sync_diagnostics_metrics() so those signals get their own report +# category and a single, explicit failure per missing metric. +SYNC_DIAGNOSTICS_GROUP = "sync_diagnostics" + # --------------------------------------------------------------------------- # Data classes @@ -518,6 +526,12 @@ async def _validate_parent_child( # Metric Validation (Prometheus API) # --------------------------------------------------------------------------- +# Top-level keys of expected_metrics.json that validate_metrics() must not walk +# with its generic group loop: "description" is prose, "grafana_dashboards" +# holds dashboard UIDs (checked by validate_dashboards), and +# "sync_diagnostics" has its own validator, assert_sync_diagnostics_metrics(). +SKIPPED_METRIC_GROUPS = ("description", "grafana_dashboards", SYNC_DIAGNOSTICS_GROUP) + async def validate_metrics( session: aiohttp.ClientSession, @@ -580,9 +594,12 @@ async def validate_metrics( with open(EXPECTED_METRICS_FILE) as f: expected = json.load(f) - # Check each metric category. + # Check each metric category. SKIPPED_METRIC_GROUPS keys are either not + # metric groups at all or are owned by a dedicated validator below, so + # skipping them here keeps each group to a single owner (no duplicate + # Prometheus queries and no duplicate report entries). for category_key, category_data in expected.items(): - if category_key in ("description", "grafana_dashboards"): + if category_key in SKIPPED_METRIC_GROUPS: continue metrics = category_data.get("metrics", []) @@ -657,6 +674,63 @@ async def _check_prometheus_metric( ) +async def assert_sync_diagnostics_metrics( + session: aiohttp.ClientSession, + prometheus_url: str, + report: ValidationReport, +) -> None: + """Assert every metric in the 'sync_diagnostics' group is present. + + Fresh-node sync-diagnostics work packages append native metric names to the + "sync_diagnostics" group in expected_metrics.json; this check fails the run + if any listed metric regresses to absent, so a dropped signal cannot pass + CI silently. An empty group is a genuine no-op: nothing is queried and no + check is recorded, which is the state before any signal has landed. + + A missing group key is itself a failure — the key is the anchor the sync + work packages append to, so its absence means the harness lost the contract + rather than that there is nothing to check. + + Args: + session: aiohttp client session. + prometheus_url: Prometheus API base URL. + report: ValidationReport to accumulate results. + """ + logger.info("--- Fresh-Node Sync Diagnostics Metrics ---") + + with open(EXPECTED_METRICS_FILE) as f: + expected = json.load(f) + + group = expected.get(SYNC_DIAGNOSTICS_GROUP) + if group is None: + report.add( + CheckResult( + name=f"metric.{SYNC_DIAGNOSTICS_GROUP}.group_present", + category="metric", + passed=False, + message=( + f"'{SYNC_DIAGNOSTICS_GROUP}' group missing from " + f"{EXPECTED_METRICS_FILE.name}" + ), + ) + ) + return + + metrics = group.get("metrics", []) + if not metrics: + logger.info( + "[SKIP] metric.%s: group is empty (no sync-diagnostics signals " + "declared yet)", + SYNC_DIAGNOSTICS_GROUP, + ) + return + + for metric_name in metrics: + await _check_prometheus_metric( + session, prometheus_url, metric_name, SYNC_DIAGNOSTICS_GROUP, report + ) + + # --------------------------------------------------------------------------- # Log-Trace Correlation Validation (Loki API) # --------------------------------------------------------------------------- @@ -1187,6 +1261,7 @@ async def run_validation( await validate_spans(session, tempo_url, report) await validate_span_durations(session, tempo_url, report) await validate_metrics(session, prometheus_url, report) + await assert_sync_diagnostics_metrics(session, prometheus_url, report) if not skip_loki: await validate_log_trace_correlation(session, loki_url, tempo_url, report) await validate_dashboards(session, grafana_url, report) diff --git a/docs/telemetry-glossary.md b/docs/telemetry-glossary.md index 0bdaf6bc05..0c0d18c4ab 100644 --- a/docs/telemetry-glossary.md +++ b/docs/telemetry-glossary.md @@ -499,6 +499,16 @@ A node stores ledger history as one or more contiguous ranges. One continuous ra **Scope:** per node — measured on and specific to this individual server. + + +### Fresh-node sync diagnostics + +The set of signals that explain why a freshly-started node is slow to reach, or never reaches, a validated ledger. They split into pre-quorum bootstrap (DNS, peer dial, protocol negotiation, UNL fetch and quorum, clock skew) and the post-peering acquire pipeline (sync state, ledger and tx-set acquire, job queue, quorum and publish lag, back-fill, persistence). Rendered by the Ledger Sync Health dashboard; individual terms are defined below as each signal lands. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Diagnosing slow/stuck fresh sync](./telemetry-runbook.md#diagnosing-slowstuck-fresh-sync) (operator flow) · [Server states on xrpl.org](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) + ### Historical fetch rate diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 2d90637053..353fea2519 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -2043,6 +2043,33 @@ The `getKBUsed*()` methods require SQLite databases to exist. If running with - Verify Loki is running: `curl http://localhost:3100/ready` - Check the filelog receiver glob `/var/log/xrpld/*/debug.log` matches your log layout — the log file must sit one subdirectory below the mount root +### Diagnosing slow/stuck fresh sync + +A fresh node that is slow to reach `server_state=full`, or never reaches it, is +diagnosed from the **Ledger Sync Health** dashboard (uid `ledger-sync-health`). + +Walk the dashboard rows top-down — the first row that looks wrong is the stage +that is blocking: + +1. **Bootstrap (Domain 0)** — can the node reach peers and form a quorum at all? + Covers DNS resolution, outbound dial, protocol/network-ID negotiation, UNL + fetch with trusted-key-vs-quorum, and clock skew. A node that never gets past + this row can never validate, no matter how healthy the pipeline looks. +2. **Sync pipeline** — is ledger data actually arriving and being applied? + Covers sync-state transitions, ledger/tx-set acquire progress, job-queue + backlog, quorum and publish lag, back-fill, and persistence latency. + +Signal definitions: +[telemetry-glossary.md](./telemetry-glossary.md) "Fresh-node sync diagnostics". +Signal index (instrument, emit site, panel): +[09-data-collection-reference.md](../OpenTelemetryPlan/09-data-collection-reference.md) +"Fresh-node sync diagnostics". + +Note: the sync signals are native metrics, which are never sampled — unlike the +span-derived (spanmetrics) series, they are always complete. + +_(Per-signal diagnosis steps are added as each signal lands.)_ + ## Performance Tuning | Scenario | Recommendation |