mirror of
https://github.com/XRPLF/rippled.git
synced 2026-08-22 14:50:54 +00:00
A freshly started node most often stalls before it ever peers or reaches quorum, and that whole chain had no telemetry. Adds the six signals that make it observable: - dns_resolve_total / dns_resolve_latency_ms: configured-peer hostname resolution, emitted from OverlayImpl so libxrpl stays independent. - overlay_connect_total / overlay_dial_latency_ms: outbound dial outcome by terminal reason, plus dial duration. - handshake_negotiation_fail_total: protocol and network-id negotiation rejections, labelled by reason, so a misconfigured network is no longer indistinguishable from unreachable peers. - unl_fetch_total and the unl_quorum gauge: validator-list fetch outcome per site and trusted key count against the required quorum. Without these a bad validators.txt leaves the node syncing forever with no signal. - clock_close_offset_seconds: network close-time offset, which server_info hides below 60s but which stalls consensus participation. Panels land in the Bootstrap row of the Ledger Sync Health dashboard, the metrics are asserted by the workload validator, and both the reference and the runbook flow describe them. Levelization baseline regenerated: overlay now includes MetricMacros.h, so the overlay/telemetry pair is reported one-way instead of bidirectional. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
166 lines
6.6 KiB
JSON
166 lines
6.6 KiB
JSON
{
|
|
"description": "Expected metric inventory for xrpld telemetry validation. Metric names have no prefix (the xrpld_ prefix was removed). beast::insight metrics are lowercased by formatName. Sourced from the live Grafana dashboards and MetricsRegistry.cpp.",
|
|
"spanmetrics": {
|
|
"description": "SpanMetrics-derived RED metrics from the OTel Collector spanmetrics connector.",
|
|
"metrics": [
|
|
"span_calls_total",
|
|
"span_duration_milliseconds_bucket",
|
|
"span_duration_milliseconds_count",
|
|
"span_duration_milliseconds_sum"
|
|
],
|
|
"required_labels": [
|
|
"span_name",
|
|
"status_code",
|
|
"service_name",
|
|
"span_kind"
|
|
],
|
|
"dimension_labels": [
|
|
"command",
|
|
"rpc_status",
|
|
"consensus_mode",
|
|
"local",
|
|
"proposal_trusted",
|
|
"validation_trusted",
|
|
"tx_type",
|
|
"ter_result",
|
|
"stage",
|
|
"txq_status",
|
|
"close_time_correct",
|
|
"consensus_state",
|
|
"suppressed"
|
|
],
|
|
"_dimension_labels_note": "Bare label names as configured in otel-collector-config.yaml spanmetrics dimensions. Informational only (not asserted by the validator)."
|
|
},
|
|
"statsd_gauges": {
|
|
"description": "beast::insight gauges exported via OTLP/HTTP to the collector (server=otel).",
|
|
"metrics": [
|
|
"ledgermaster_validated_ledger_age",
|
|
"ledgermaster_published_ledger_age",
|
|
"state_accounting_full_duration",
|
|
"peer_finder_active_inbound_peers",
|
|
"peer_finder_active_outbound_peers",
|
|
"jobq_job_count"
|
|
]
|
|
},
|
|
"statsd_counters": {
|
|
"description": "beast::insight counters exported via OTLP/HTTP. The OTel Prometheus exporter appends _total to monotonic counters.",
|
|
"metrics": ["rpc_requests_total", "ledger_fetches_total"]
|
|
},
|
|
"overlay_traffic": {
|
|
"description": "Overlay traffic metrics (subset — full list has 45+ categories).",
|
|
"metrics": [
|
|
"total_bytes_in",
|
|
"total_bytes_out",
|
|
"total_messages_in",
|
|
"total_messages_out"
|
|
]
|
|
},
|
|
"phase9_nodestore": {
|
|
"description": "Phase 9 NodeStore I/O observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label distinguishing sub-metrics.",
|
|
"metrics": ["nodestore_state"]
|
|
},
|
|
"phase9_cache": {
|
|
"description": "Phase 9 cache hit rate observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.",
|
|
"metrics": ["cache_metrics"]
|
|
},
|
|
"phase9_txq": {
|
|
"description": "Phase 9 transaction queue observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.",
|
|
"metrics": ["txq_metrics"]
|
|
},
|
|
"phase9_rpc_method": {
|
|
"description": "Phase 9 per-RPC-method counters (MetricsRegistry via OTLP).",
|
|
"metrics": ["rpc_method_started_total"]
|
|
},
|
|
"rpc_in_flight": {
|
|
"description": "In-flight RPC gauge via the XRPL_METRIC_UPDOWN_ADD call-site macro (PerfLogImp.cpp, +1 rpcStart / -1 rpcEnd). UpDownCounter: no _total suffix.",
|
|
"metrics": ["rpc_in_flight_requests"]
|
|
},
|
|
"phase9_objects": {
|
|
"description": "Phase 9 counted object instances observable gauge (MetricsRegistry via OTLP).",
|
|
"metrics": ["object_count"]
|
|
},
|
|
"phase9_load": {
|
|
"description": "Phase 9 fee escalation and load factor observable gauge (MetricsRegistry via OTLP).",
|
|
"metrics": ["load_factor_metrics"]
|
|
},
|
|
"parity_validation_agreement": {
|
|
"description": "External dashboard parity: validation agreement percentages (MetricsRegistry).",
|
|
"metrics": [
|
|
"validation_agreement{metric=\"agreement_pct_1h\"}",
|
|
"validation_agreement{metric=\"agreement_pct_24h\"}"
|
|
]
|
|
},
|
|
"parity_validator_health": {
|
|
"description": "External dashboard parity: validator health indicators (MetricsRegistry).",
|
|
"metrics": [
|
|
"validator_health{metric=\"amendment_blocked\"}",
|
|
"validator_health{metric=\"unl_expiry_days\"}"
|
|
]
|
|
},
|
|
"parity_peer_quality": {
|
|
"description": "External dashboard parity: peer quality metrics (MetricsRegistry).",
|
|
"metrics": [
|
|
"peer_quality{metric=\"peer_latency_p90_ms\"}",
|
|
"peer_quality{metric=\"peers_insane_count\"}"
|
|
]
|
|
},
|
|
"parity_ledger_economy": {
|
|
"description": "External dashboard parity: ledger economy metrics (MetricsRegistry).",
|
|
"metrics": [
|
|
"ledger_economy{metric=\"base_fee_xrp\"}",
|
|
"ledger_economy{metric=\"transaction_rate\"}"
|
|
]
|
|
},
|
|
"parity_state_tracking": {
|
|
"description": "External dashboard parity: server state tracking (MetricsRegistry).",
|
|
"metrics": ["state_tracking{metric=\"state_value\"}"]
|
|
},
|
|
"parity_counters": {
|
|
"description": "External dashboard parity: monotonic counters (MetricsRegistry).",
|
|
"metrics": [
|
|
"ledgers_closed_total",
|
|
"validations_sent_total",
|
|
"state_changes_total"
|
|
]
|
|
},
|
|
"parity_storage": {
|
|
"description": "External dashboard parity: storage detail metrics (MetricsRegistry).",
|
|
"metrics": ["storage_detail{metric=\"nudb_bytes\"}"]
|
|
},
|
|
"sync_diagnostics": {
|
|
"description": "Fresh-node sync diagnostics (native metrics). Bootstrap (Domain 0) and acquire-pipeline signals rendered by the ledger-sync-health dashboard. Names are appended one per signal. Histograms are listed by their Prometheus _bucket series (the bare instrument name is not a series). The two observable gauges carry an inline metric= selector so the specific sub-series is asserted, matching the parity_* groups.",
|
|
"metrics": [
|
|
"dns_resolve_total",
|
|
"dns_resolve_latency_ms_bucket",
|
|
"overlay_connect_total",
|
|
"overlay_dial_latency_ms_bucket",
|
|
"handshake_negotiation_fail_total",
|
|
"unl_fetch_total",
|
|
"unl_quorum{metric=\"trusted_keys\"}",
|
|
"unl_quorum{metric=\"quorum\"}",
|
|
"clock_close_offset_seconds{metric=\"offset\"}"
|
|
],
|
|
"_conditional_note": "handshake_negotiation_fail_total and unl_fetch_total are conditional under the local harness: the first only exists once a handshake is rejected, and the second needs a [validator_list_sites] entry (run-full-validation.sh generates a static [validators] file instead). The validator has no per-metric optional flag, so if either reports 0 series in a harness run, move it out of this group rather than weakening the check."
|
|
},
|
|
"grafana_dashboards": {
|
|
"description": "All Grafana dashboards that must render data (UIDs as provisioned on disk under docker/telemetry/grafana/dashboards/).",
|
|
"uids": [
|
|
"rpc-performance",
|
|
"transaction-overview",
|
|
"consensus-health",
|
|
"ledger-operations",
|
|
"peer-network",
|
|
"peer-quality",
|
|
"fee-market",
|
|
"job-queue",
|
|
"validator-health",
|
|
"node-health",
|
|
"network-traffic",
|
|
"rpc-pathfinding",
|
|
"overlay-traffic-detail",
|
|
"ledger-data-sync",
|
|
"ledger-sync-health"
|
|
]
|
|
}
|
|
}
|