mirror of
https://github.com/XRPLF/rippled.git
synced 2026-08-22 06:40:53 +00:00
4115617eb9
Sync-critical job types run at very low concurrency limits (ledgerRequest
and ledgerData allow 3 each), so a node can stall simply because those
jobs are held back behind other work. Nothing exposed that until now:
the existing job metrics are rates and quantiles of jobs that already
moved, or a single queue-wide depth.
- jobq_backlog{metric,job_type}: instantaneous waiting, running and
deferred counts per job type. Deferred is the starvation signal and had
no exposure anywhere; it is set when a type is at its concurrency limit.
- jobq_saturation{metric}: running tasks, worker-thread count and total
waiting, so a slowdown spanning several subsystems can be attributed to
worker-pool exhaustion instead of being diagnosed once per victim.
Both read through two new const accessors on JobQueue that take the
existing mutex once and copy integers, so a single reading is internally
consistent and no per-job cost is added. The job_type label reuses the
same JobTypes name helper the existing job counters use, so the two label
sets join.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
187 lines
11 KiB
JSON
187 lines
11 KiB
JSON
{
|
|
"description": "Expected metric inventory for xrpld telemetry validation. Metric names have no prefix (the xrpld_ prefix was removed). beast::insight metrics are lowercased by formatName. Sourced from the live Grafana dashboards and MetricsRegistry.cpp.",
|
|
"spanmetrics": {
|
|
"description": "SpanMetrics-derived RED metrics from the OTel Collector spanmetrics connector.",
|
|
"metrics": [
|
|
"span_calls_total",
|
|
"span_duration_milliseconds_bucket",
|
|
"span_duration_milliseconds_count",
|
|
"span_duration_milliseconds_sum"
|
|
],
|
|
"required_labels": [
|
|
"span_name",
|
|
"status_code",
|
|
"service_name",
|
|
"span_kind"
|
|
],
|
|
"dimension_labels": [
|
|
"command",
|
|
"rpc_status",
|
|
"consensus_mode",
|
|
"local",
|
|
"proposal_trusted",
|
|
"validation_trusted",
|
|
"tx_type",
|
|
"ter_result",
|
|
"stage",
|
|
"txq_status",
|
|
"close_time_correct",
|
|
"consensus_state",
|
|
"suppressed"
|
|
],
|
|
"_dimension_labels_note": "Bare label names as configured in otel-collector-config.yaml spanmetrics dimensions. Informational only (not asserted by the validator)."
|
|
},
|
|
"statsd_gauges": {
|
|
"description": "beast::insight gauges exported via OTLP/HTTP to the collector (server=otel).",
|
|
"metrics": [
|
|
"ledgermaster_validated_ledger_age",
|
|
"ledgermaster_published_ledger_age",
|
|
"state_accounting_full_duration",
|
|
"peer_finder_active_inbound_peers",
|
|
"peer_finder_active_outbound_peers",
|
|
"jobq_job_count"
|
|
]
|
|
},
|
|
"statsd_counters": {
|
|
"description": "beast::insight counters exported via OTLP/HTTP. The OTel Prometheus exporter appends _total to monotonic counters.",
|
|
"metrics": ["rpc_requests_total", "ledger_fetches_total"]
|
|
},
|
|
"overlay_traffic": {
|
|
"description": "Overlay traffic metrics (subset — full list has 45+ categories).",
|
|
"metrics": [
|
|
"total_bytes_in",
|
|
"total_bytes_out",
|
|
"total_messages_in",
|
|
"total_messages_out"
|
|
]
|
|
},
|
|
"phase9_nodestore": {
|
|
"description": "Phase 9 NodeStore I/O observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label distinguishing sub-metrics.",
|
|
"metrics": ["nodestore_state"]
|
|
},
|
|
"phase9_cache": {
|
|
"description": "Phase 9 cache hit rate observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.",
|
|
"metrics": ["cache_metrics"]
|
|
},
|
|
"phase9_txq": {
|
|
"description": "Phase 9 transaction queue observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.",
|
|
"metrics": ["txq_metrics"]
|
|
},
|
|
"phase9_rpc_method": {
|
|
"description": "Phase 9 per-RPC-method counters (MetricsRegistry via OTLP).",
|
|
"metrics": ["rpc_method_started_total"]
|
|
},
|
|
"rpc_in_flight": {
|
|
"description": "In-flight RPC gauge via the XRPL_METRIC_UPDOWN_ADD call-site macro (PerfLogImp.cpp, +1 rpcStart / -1 rpcEnd). UpDownCounter: no _total suffix.",
|
|
"metrics": ["rpc_in_flight_requests"]
|
|
},
|
|
"phase9_objects": {
|
|
"description": "Phase 9 counted object instances observable gauge (MetricsRegistry via OTLP).",
|
|
"metrics": ["object_count"]
|
|
},
|
|
"phase9_load": {
|
|
"description": "Phase 9 fee escalation and load factor observable gauge (MetricsRegistry via OTLP).",
|
|
"metrics": ["load_factor_metrics"]
|
|
},
|
|
"parity_validation_agreement": {
|
|
"description": "External dashboard parity: validation agreement percentages (MetricsRegistry).",
|
|
"metrics": [
|
|
"validation_agreement{metric=\"agreement_pct_1h\"}",
|
|
"validation_agreement{metric=\"agreement_pct_24h\"}"
|
|
]
|
|
},
|
|
"parity_validator_health": {
|
|
"description": "External dashboard parity: validator health indicators (MetricsRegistry).",
|
|
"metrics": [
|
|
"validator_health{metric=\"amendment_blocked\"}",
|
|
"validator_health{metric=\"unl_expiry_days\"}"
|
|
]
|
|
},
|
|
"parity_peer_quality": {
|
|
"description": "External dashboard parity: peer quality metrics (MetricsRegistry).",
|
|
"metrics": [
|
|
"peer_quality{metric=\"peer_latency_p90_ms\"}",
|
|
"peer_quality{metric=\"peers_insane_count\"}"
|
|
]
|
|
},
|
|
"parity_ledger_economy": {
|
|
"description": "External dashboard parity: ledger economy metrics (MetricsRegistry).",
|
|
"metrics": [
|
|
"ledger_economy{metric=\"base_fee_xrp\"}",
|
|
"ledger_economy{metric=\"transaction_rate\"}"
|
|
]
|
|
},
|
|
"parity_state_tracking": {
|
|
"description": "External dashboard parity: server state tracking (MetricsRegistry).",
|
|
"metrics": ["state_tracking{metric=\"state_value\"}"]
|
|
},
|
|
"parity_counters": {
|
|
"description": "External dashboard parity: monotonic counters (MetricsRegistry).",
|
|
"metrics": [
|
|
"ledgers_closed_total",
|
|
"validations_sent_total",
|
|
"state_changes_total"
|
|
]
|
|
},
|
|
"parity_storage": {
|
|
"description": "External dashboard parity: storage detail metrics (MetricsRegistry).",
|
|
"metrics": ["storage_detail{metric=\"nudb_bytes\"}"]
|
|
},
|
|
"sync_diagnostics": {
|
|
"description": "Fresh-node sync diagnostics (native metrics). Bootstrap (Domain 0) and acquire-pipeline signals rendered by the ledger-sync-health dashboard. Names are appended one per signal. Histograms are listed by their Prometheus _bucket series (the bare instrument name is not a series). The two observable gauges carry an inline metric= selector so the specific sub-series is asserted, matching the parity_* groups.",
|
|
"metrics": [
|
|
"dns_resolve_total",
|
|
"dns_resolve_latency_ms_bucket",
|
|
"overlay_connect_total",
|
|
"overlay_dial_latency_ms_bucket",
|
|
"handshake_negotiation_fail_total",
|
|
"unl_fetch_total",
|
|
"unl_quorum{metric=\"trusted_keys\"}",
|
|
"unl_quorum{metric=\"quorum\"}",
|
|
"clock_close_offset_seconds{metric=\"offset\"}",
|
|
"sync_state{metric=\"initial_full_duration_us\"}",
|
|
"sync_state{metric=\"network_ledger_gate\"}",
|
|
"sync_state{metric=\"server_stall_seconds\"}",
|
|
"sync_state{metric=\"ledgers_behind\"}",
|
|
"server_stall_events_total",
|
|
"state_changes_total{from!=\"\",to!=\"\"}",
|
|
"sync_acquire{metric=\"missing_state_nodes_max\"}",
|
|
"sync_acquire{metric=\"missing_tx_nodes_max\"}",
|
|
"sync_acquire{metric=\"received_data_depth\"}",
|
|
"sync_acquire{metric=\"in_flight\"}",
|
|
"shamap_cache_hit_rate{metric=\"treenode\"}",
|
|
"jobq_backlog{metric=\"waiting\",job_type=\"ledgerData\"}",
|
|
"jobq_backlog{metric=\"running\",job_type=\"ledgerData\"}",
|
|
"jobq_backlog{metric=\"deferred\",job_type=\"ledgerData\"}",
|
|
"jobq_backlog{metric=\"deferred\",job_type=\"ledgerRequest\"}",
|
|
"jobq_saturation{metric=\"running_tasks\"}",
|
|
"jobq_saturation{metric=\"worker_threads\"}",
|
|
"jobq_saturation{metric=\"total_waiting\"}"
|
|
],
|
|
"_acquire_note": "The four sync_acquire sub-series and shamap_cache_hit_rate are unconditional: both are observable gauges whose callbacks observe every series on each collection tick, so each is present even when the value is 0 (an idle node reports in_flight=0 and missing_state_nodes_max=0, and a cold cache reports a 0.0 hit rate). Absence, not a zero, is the regression. The three WP-A3 counters (sync_acquire_source_total, sync_addnode_total, sync_acquire_no_progress_total) are deliberately NOT asserted here: all three are emitted only from InboundLedger, which runs only when a node must fetch a ledger it lacks. expected_spans.json already marks the ledger.acquire span optional for exactly this reason (\"A healthy local cluster rarely back-fills history\"), and the metric validator has no per-metric optional flag, so listing them would fail the harness red on a healthy run. They are covered by exact-value unit tests in src/tests/libxrpl/telemetry/MetricMacros.cpp and by the ledger-sync-health panels; add them here only alongside a harness step that forces a real acquire (e.g. starting a node against an existing ledger history).",
|
|
"_jobq_note": "The jobq_backlog and jobq_saturation series are unconditional: both are observable gauges whose callbacks iterate EVERY registered JobType (jobData_ is populated from JobTypes at JobQueue construction) and observe all three fields on each collection tick, so a series exists even when the value is 0. That is why an idle-but-registered type like ledgerData is safe to assert by name here — a fresh harness node that never defers a single job still reports jobq_backlog{metric=\"deferred\",job_type=\"ledgerData\"} = 0, and absence, not the zero, is the regression. Two job_type values are asserted (ledgerData and ledgerRequest) because they are the sync-critical types capped at concurrency 3 in JobTypes.h, so they are the ones whose deferred series must never silently vanish. Only deferred is asserted for ledgerRequest to keep the list short: the three-field fan-out is already proven by ledgerData. worker_threads is asserted because it is the denominator of the dashboard saturation ratio, and it is always at least 1 (the JobQueue ctor gives standalone mode exactly one worker), so a zero or missing reading there means the accessor regressed rather than the node being idle.",
|
|
"_conditional_note": "handshake_negotiation_fail_total and unl_fetch_total are conditional under the local harness: the first only exists once a handshake is rejected, and the second needs a [validator_list_sites] entry (run-full-validation.sh generates a static [validators] file instead). The validator has no per-metric optional flag, so if either reports 0 series in a harness run, move it out of this group rather than weakening the check.",
|
|
"_sync_state_note": "The four sync_state sub-series are unconditional: the gauge observes all four on every collection tick, so each is present as a series even when its value is 0 (a node that never reached FULL reports initial_full_duration_us=0, and a healthy node reports server_stall_seconds=0). The check asserts series presence, not a non-zero value, which is exactly right here — a zero is a meaningful reading for these signals, and absence is the regression. server_stall_events_total is likewise always present because the observable counter reports the tally (0 or more) every tick. state_changes_total is asserted here with a from!=\"\",to!=\"\" selector rather than bare (parity_counters already asserts the bare name): the selector is what proves the WP-A2 {from,to} label dimension actually reached Prometheus, so a regression to the old unlabelled counter fails this check instead of silently passing on the bare name. It needs at least one real mode transition, which any node reaching connected/syncing produces during startup."
|
|
},
|
|
"grafana_dashboards": {
|
|
"description": "All Grafana dashboards that must render data (UIDs as provisioned on disk under docker/telemetry/grafana/dashboards/).",
|
|
"uids": [
|
|
"rpc-performance",
|
|
"transaction-overview",
|
|
"consensus-health",
|
|
"ledger-operations",
|
|
"peer-network",
|
|
"peer-quality",
|
|
"fee-market",
|
|
"job-queue",
|
|
"validator-health",
|
|
"node-health",
|
|
"network-traffic",
|
|
"rpc-pathfinding",
|
|
"overlay-traffic-detail",
|
|
"ledger-data-sync",
|
|
"ledger-sync-health"
|
|
]
|
|
}
|
|
}
|