{ "description": "Expected metric inventory for xrpld telemetry validation. Metric names have no prefix (the xrpld_ prefix was removed). beast::insight metrics are lowercased by formatName. Every name here was verified against its declaration in MetricsRegistry.cpp or include/xrpl/telemetry/GetObjectMetricNames.h and against a panel query under docker/telemetry/grafana/dashboards/. IMPORTANT: validate_telemetry.py has no notion of an optional metric — validate_metrics() iterates every group that has a \"metrics\" key and hard-fails any name with 0 Prometheus series after a 45 s poll. A metric is therefore listed only when the harness workload guarantees it will appear: observable gauges/counters whose callbacks Observe unconditionally (series exist at value 0), or push counters/histograms on a path every run exercises. Workload-gated and defect-gated names are recorded in the \"not_asserted\" group, which intentionally has no \"metrics\" key so the validator skips it. Only series existence is checked, never a value, except for the four bounds checks hardcoded in PARITY_VALUE_SANITY.", "spanmetrics": { "description": "SpanMetrics-derived RED metrics from the OTel Collector spanmetrics connector.", "metrics": [ "span_calls_total", "span_duration_milliseconds_bucket", "span_duration_milliseconds_count", "span_duration_milliseconds_sum" ], "required_labels": [ "span_name", "status_code", "service_name", "span_kind" ], "dimension_labels": [ "command", "rpc_status", "consensus_mode", "local", "proposal_trusted", "validation_trusted", "tx_type", "ter_result", "stage", "txq_status", "close_time_correct", "consensus_state", "suppressed" ], "_dimension_labels_note": "Bare label names as configured in otel-collector-config.yaml spanmetrics dimensions. Informational only (not asserted by the validator)." }, "statsd_gauges": { "description": "beast::insight gauges exported via OTLP/HTTP to the collector (server=otel).", "metrics": [ "ledgermaster_validated_ledger_age", "ledgermaster_published_ledger_age", "state_accounting_full_duration", "peer_finder_active_inbound_peers", "peer_finder_active_outbound_peers", "jobq_job_count" ] }, "statsd_counters": { "description": "beast::insight counters exported via OTLP/HTTP. The OTel Prometheus exporter appends _total to monotonic counters.", "metrics": ["rpc_requests_total", "ledger_fetches_total"] }, "overlay_traffic": { "description": "Overlay traffic metrics (subset \u2014 full list has 45+ categories).", "metrics": [ "total_bytes_in", "total_bytes_out", "total_messages_in", "total_messages_out" ] }, "nodestore_io": { "description": "NodeStore I/O observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label distinguishing sub-metrics.", "metrics": ["nodestore_state"] }, "cache_hit_rates": { "description": "Cache hit rate observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.", "metrics": ["cache_metrics"] }, "transaction_queue": { "description": "Transaction queue observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.", "metrics": ["txq_metrics"] }, "rpc_method_detail": { "description": "Per-RPC-method counters and duration histogram (MetricsRegistry.cpp:351-357). rpc_method_errored_total is deliberately absent — see not_asserted below. rpc_method_us is a Histogram, so the Prometheus exporter emits only the _bucket/_count/_sum triple and there is no bare rpc_method_us series to match — same convention as span_duration_milliseconds in the spanmetrics group above.", "metrics": [ "rpc_method_started_total", "rpc_method_finished_total", "rpc_method_us_bucket", "rpc_method_us_count", "rpc_method_us_sum" ] }, "job_queue": { "description": "Job-queue counters and latency histograms (MetricsRegistry.cpp:360-366). Every xrpld job passes through these, so they populate under any workload. Both histograms are recorded in the same function bodies as job_started_total / job_finished_total, under the same guard and with the same labels, so their presence is equally guaranteed. They are named with the _bucket/_count/_sum suffixes the Prometheus exporter emits: regression-metrics.json and the job-queue dashboard both query job_queued_us_bucket / job_running_us_bucket, and no bare series exists.", "metrics": [ "job_queued_total", "job_started_total", "job_finished_total", "job_queued_us_bucket", "job_queued_us_count", "job_queued_us_sum", "job_running_us_bucket", "job_running_us_count", "job_running_us_sum" ] }, "rpc_in_flight": { "description": "In-flight RPC gauge via the XRPL_METRIC_UPDOWN_ADD call-site macro (PerfLogImp.cpp, +1 rpcStart / -1 rpcEnd). UpDownCounter: no _total suffix.", "metrics": ["rpc_in_flight_requests"] }, "object_counts": { "description": "Counted object instances observable gauge (MetricsRegistry via OTLP).", "metrics": ["object_count"] }, "load_factors": { "description": "Fee escalation and load factor observable gauge (MetricsRegistry via OTLP).", "metrics": ["load_factor_metrics"] }, "parity_validation_agreement": { "description": "External dashboard parity: validation agreement percentages (MetricsRegistry).", "metrics": [ "validation_agreement{metric=\"agreement_pct_1h\"}", "validation_agreement{metric=\"agreement_pct_24h\"}" ] }, "parity_validator_health": { "description": "External dashboard parity: validator health indicators (MetricsRegistry).", "metrics": [ "validator_health{metric=\"amendment_blocked\"}", "validator_health{metric=\"unl_expiry_days\"}" ] }, "parity_peer_quality": { "description": "External dashboard parity: peer quality metrics (MetricsRegistry).", "metrics": [ "peer_quality{metric=\"peer_latency_p90_ms\"}", "peer_quality{metric=\"peers_insane_count\"}" ] }, "parity_ledger_economy": { "description": "External dashboard parity: ledger economy metrics (MetricsRegistry.cpp:1401). transaction_rate is observed on every export, in both branches of the ledger-age test (MetricsRegistry.cpp:1444-1451). base_fee_xrp is observed only inside the 'if (ledger)' guard on getValidatedLedger() (MetricsRegistry.cpp:1418-1423), and that returns validLedger_ (LedgerMaster.cpp:1569-1572), which stays null until a ledger validates — the same precondition complete_ledgers has. Both are asserted because run-full-validation.sh waits for a validated ledger before running the workload. base_fee_xrp absent while transaction_rate is present is the signature of a cluster that never validated, not of a missing metric.", "metrics": [ "ledger_economy{metric=\"base_fee_xrp\"}", "ledger_economy{metric=\"transaction_rate\"}" ] }, "parity_state_tracking": { "description": "External dashboard parity: server state tracking (MetricsRegistry).", "metrics": ["state_tracking{metric=\"state_value\"}"] }, "parity_counters": { "description": "External dashboard parity: monotonic counters (MetricsRegistry). validations_checked_total is incremented unconditionally at the top of NetworkOPsImp::recvValidation (NetworkOPs.cpp:2681), and run-full-validation.sh brings up a 5-node validator cluster, so inbound validations are guaranteed.", "metrics": [ "ledgers_closed_total", "validations_sent_total", "validations_checked_total", "state_changes_total" ] }, "parity_storage": { "description": "External dashboard parity: storage detail metrics (MetricsRegistry).", "metrics": ["storage_detail{metric=\"stored_object_bytes\"}"] }, "sync_diagnostics": { "description": "Fresh-node sync diagnostics (native metrics). Bootstrap (Domain 0) and acquire-pipeline signals rendered by the ledger-sync-health dashboard. Names are appended one per signal. Histograms are listed by their Prometheus _bucket series (the bare instrument name is not a series). The two observable gauges carry an inline metric= selector so the specific sub-series is asserted, matching the parity_* groups.", "metrics": [ "dns_resolve_total", "dns_resolve_latency_ms_bucket", "overlay_connect_total", "overlay_dial_latency_ms_bucket", "handshake_negotiation_fail_total", "unl_fetch_total", "unl_quorum{metric=\"trusted_keys\"}", "unl_quorum{metric=\"quorum\"}", "clock_close_offset_seconds{metric=\"offset\"}", "sync_state{metric=\"initial_full_duration_us\"}", "sync_state{metric=\"network_ledger_gate\"}", "sync_state{metric=\"server_stall_seconds\"}", "sync_state{metric=\"ledgers_behind\"}", "server_stall_events_total", "state_changes_total{from!=\"\",to!=\"\"}", "sync_acquire{metric=\"missing_state_nodes_max\"}", "sync_acquire{metric=\"missing_tx_nodes_max\"}", "sync_acquire{metric=\"received_data_depth\"}", "sync_acquire{metric=\"in_flight\"}", "shamap_cache_hit_rate{metric=\"treenode\"}", "jobq_saturation{metric=\"running_tasks\"}", "jobq_saturation{metric=\"worker_threads\"}", "jobq_saturation{metric=\"total_waiting\"}", "peer_ledger_supply{metric=\"peers_reporting\"}", "peer_ledger_supply{metric=\"peers_serving_validated\"}", "peer_ledger_supply{metric=\"peers_serving_next\"}", "peer_ledger_supply{metric=\"supply_min_seq\"}", "peer_ledger_supply{metric=\"supply_max_seq\"}", "peerfinder_slot_census{metric=\"out_active\"}", "peerfinder_slot_census{metric=\"out_max\"}", "peerfinder_slot_census{metric=\"in_active\"}", "peerfinder_slot_census{metric=\"in_max\"}", "peerfinder_slot_census{metric=\"connecting\"}", "peerfinder_slot_census{metric=\"fixed_configured\"}", "peerfinder_slot_census{metric=\"fixed_active\"}", "peerfinder_slot_census{metric=\"bootcache\"}", "peerfinder_slot_census{metric=\"livecache\"}", "amendment_block{metric=\"warned\"}", "amendment_block{metric=\"seconds_to_block\"}", "peer_accept_total", "nodestore_state{metric=\"node_writes\"}", "nodestore_state{metric=\"node_reads_total\"}", "nodestore_state{metric=\"read_mean_us\"}", "ledger_quorum_publish{metric=\"trusted_validation_tally\"}", "ledger_quorum_publish{metric=\"quorum_target\"}", "ledger_quorum_publish{metric=\"time_to_first_validated_us\"}", "ledger_quorum_publish{metric=\"publish_lag\"}", "ledger_quorum_shortfall_total{stage=\"pre_accept\"}", "consensus_round_duration_ms_bucket", "consensus_round_duration_ms_count", "nodestore_state{metric=\"node_writes_duration_us\"}", "nodestore_state{metric=\"node_reads_duration_us\"}", "unl_quorum{metric=\"quorum_disabled\"}", "sweep_malloc_trim_us_bucket", "sweep_malloc_trim_us_count" ], "_acquire_note": "The four sync_acquire sub-series and shamap_cache_hit_rate are unconditional: both are observable gauges whose callbacks observe every series on each collection tick, so each is present even when the value is 0 (an idle node reports in_flight=0 and missing_state_nodes_max=0, and a cold cache reports a 0.0 hit rate). Absence, not a zero, is the regression. The three WP-A3 counters (sync_acquire_source_total, sync_addnode_total, sync_acquire_no_progress_total) are deliberately NOT asserted here: all three are emitted only from InboundLedger, which runs only when a node must fetch a ledger it lacks. expected_spans.json already marks the ledger.acquire span optional for exactly this reason (\"A healthy local cluster rarely back-fills history\"), and the metric validator has no per-metric optional flag, so listing them would fail the harness red on a healthy run. They are covered by exact-value unit tests in src/tests/libxrpl/telemetry/MetricMacros.cpp and by the ledger-sync-health panels; add them here only alongside a harness step that forces a real acquire (e.g. starting a node against an existing ledger history).", "_jobq_note": "The three jobq_saturation series are unconditional: it is an observable gauge whose callback observes all three fields on every collection tick, so each series exists even when the value is 0, and absence rather than a zero is the regression. worker_threads is asserted because it is the denominator of the dashboard saturation ratio, and it is always at least 1 (the JobQueue ctor gives standalone mode exactly one worker), so a zero or missing reading there means the accessor regressed rather than the node being idle. The per-job-type waiting/running/deferred counts are published separately by JobQueue::collect() as the beast::insight gauges jobq__waiting / _running / _deferred, which the collector translates; they are covered by the StatsD-derived groups, not here.", "_conditional_note": "handshake_negotiation_fail_total and unl_fetch_total are conditional under the local harness: the first only exists once a handshake is rejected, and the second needs a [validator_list_sites] entry (run-full-validation.sh generates a static [validators] file instead). The validator has no per-metric optional flag, so if either reports 0 series in a harness run, move it out of this group rather than weakening the check.", "_sync_state_note": "The four sync_state sub-series are unconditional: the gauge observes all four on every collection tick, so each is present as a series even when its value is 0 (a node that never reached FULL reports initial_full_duration_us=0, and a healthy node reports server_stall_seconds=0). The check asserts series presence, not a non-zero value, which is exactly right here \u2014 a zero is a meaningful reading for these signals, and absence is the regression. server_stall_events_total is likewise always present because the observable counter reports the tally (0 or more) every tick. state_changes_total is asserted here with a from!=\"\",to!=\"\" selector rather than bare (parity_counters already asserts the bare name): the selector is what proves the WP-A2 {from,to} label dimension actually reached Prometheus, so a regression to the old unlabelled counter fails this check instead of silently passing on the bare name. It needs at least one real mode transition, which any node reaching connected/syncing produces during startup.", "_a7_note": "WP-A7 adds three observable gauges and four counters. The 16 gauge sub-series (peer_ledger_supply, peerfinder_slot_census, amendment_block) are unconditional and asserted individually: each callback in MetricsRegistry.cpp calls observe() for every field on every collection tick with no early return between them, so the series exists whatever the value. That includes the two sentinel readings \u2014 a node whose peers have advertised nothing reports peer_ledger_supply{metric=\"supply_min_seq\"} = 0 meaning unknown, and a node with no pending amendment reports amendment_block{metric=\"seconds_to_block\"} = -1 meaning healthy. Absence, not the sentinel, is the regression. Of the four counters only peer_accept_total is asserted: run-full-validation.sh gives every node a [port_peer] on 0.0.0.0 and lists the other four nodes in [ips], so all 5 nodes dial each other and each one is also dialled, which means OverlayImpl::onHandoff runs and reports outcome=accepted (or slot_refused/no_slot on the duplicate half of each mutual dial) on every node. It is asserted bare rather than with an outcome= selector because which outcome a given node records depends on dial ordering, which the harness does not control. The other three counters are deliberately NOT asserted. peer_disconnect_total is emitted only from PeerImp::close, and a healthy 5-node localhost cluster holds its 4 fixed peers for the whole run: the timer-driven reasons need maxUnknownTime (600 s) or maxDivergedTime (300 s) to elapse (Config.h) while the full-validation profile totals well under that, and the shutdown reasons only fire during teardown, which happens in run-full-validation.sh after Step 5 has already scraped. serve_refused_total needs a peer to ask this node for a ledger, tx set or object it cannot serve \u2014 on a cluster where every node has the same complete history from genesis, getLedger()/getTxSet() succeed and the send queues never approach Tuning::kDropSendQueue. ledger_jump_total needs NetworkOPsImp::switchLastClosedLedger, reached only when consensus reports an LCL this node did not build on; a healthy 5-node cluster agrees every round, so it never jumps. The metric validator has no per-metric optional flag, so listing any of the three would fail the harness red on a healthy run \u2014 the same reasoning _acquire_note applies to the WP-A3 InboundLedger counters. All four counters are covered by exact-value unit tests in src/tests/libxrpl/telemetry/MetricMacros.cpp and rendered by the ledger-sync-health panels Peer Disconnects (Count By Reason & Direction), Ledger/Object Serve Refusals and Byzantine Ledger Jumps, and by the peer-quality panels Peer Disconnect Rate and Peer Disconnects By Reason & Direction. To make them assertable the harness would need a fault-injection step: kill one node mid-run and re-scrape before teardown (peer_disconnect_total, reason=read_error/graceful), request a ledger sequence outside the cluster's history or drive a node past its send-queue limit (serve_refused_total), and start a node on a divergent chain tip or partition the cluster and heal it (ledger_jump_total).", "_a6_note": "WP-A6 originally added a separate nodestore_latency gauge; it was retired because every one of its sub-metrics had an exact counterpart on nodestore_state computed from the same Database accessor, so the five entries above are the surviving equivalents (write_count -> node_writes, read_count -> node_reads_total, write_duration_us -> node_writes_duration_us, read_duration_us -> node_reads_duration_us, read_mean_us unchanged). All four totals are unconditional: MetricsRegistry::observeNodeStoreTotals observes them on every collection tick with no early return before them, so a series exists whatever the value and a node that has written nothing reports node_writes=0 rather than dropping the series. read_mean_us is safe because any node that has opened a ledger has already fetched objects, so the fetch count is non-zero. write_mean_us is NOT asserted, but only because the two means are the sub-series that scaledMean() omits when their denominator is zero, and this validator has no per-metric optional flag -- not because the numerator is missing. That older caveat is gone: all three concrete store paths now time themselves through Database::recordStoreDuration() (DatabaseNodeImp::store, DatabaseRotatingImp::store and Database::importInternal), so write_mean_us is live on an ordinary node and only a node that has performed literally zero stores would lack it. It can be promoted to an assertion once a harness run confirms it present. WP-A6's two replay counters (ledger_replay_fallback_total, ledger_replay_outcome_total) are likewise NOT asserted, for the same reason _acquire_note gives for the WP-A3 InboundLedger counters: both are emitted only from the ledger-replay path, which requires the [ledger_replay] config stanza AND a real historical back-fill against peers that support the LedgerReplay protocol feature. run-full-validation.sh starts a fresh local cluster with no history to back-fill, so no replay task is ever created and neither counter can produce a series. Both are covered by exact-value unit tests in src/tests/libxrpl/telemetry/MetricMacros.cpp and rendered by the ledger-sync-health panels Replay Fallback to Full Acquire and Replay Outcomes. To make them assertable the harness would need to enable [ledger_replay] and start a node against an existing ledger history so it back-fills through the replay path.", "_a5_note": "WP-A5 adds one observable gauge (ledger_quorum_publish) and one counter (ledger_quorum_shortfall_total). All four gauge sub-series are asserted and are unconditional: registerLedgerQuorumPublishGauge's callback in MetricsRegistry.cpp calls observe() for every field on every collection tick with no early return between them, and each accessor is a plain relaxed atomic load that always returns a value, so the series exists whatever the reading. That deliberately includes the three diagnostic zeros: a node that has never had a gate evaluated reports trusted_validation_tally=0 and quorum_target=0, one that has never fully validated reports time_to_first_validated_us=0, and one that is caught up reports publish_lag=0. Absence, not the zero, is the regression -- the same reasoning _sync_state_note gives for initial_full_duration_us. Note the sentinel: when the trusted list disables quorum entirely, getNeededValidations() returns SIZE_MAX and LedgerMaster reports quorum_target as int64 max rather than letting the cast wrap to -1, so the target reads far above any tally instead of inverting the comparison (the same fix as the unl_quorum gauge). ledger_quorum_shortfall_total IS asserted, which differs from the WP-A3/A6/A7 counters, and the reason is that this counter does not need a fault to fire. RCLConsensus::Adaptor::doAccept issues this node's own validation and then calls ledgerMaster_.consensusBuilt immediately (RCLConsensus.cpp), which calls checkAccept on the freshly built ledger (LedgerMaster.cpp) BEFORE the peers' validations for that same ledger have arrived. With the harness's 5 validators the quorum is max(ceil(5*0.8), ceil(5*0.6)) = 4 (ValidatorList::calculateQuorum), so that first evaluation of each round tallies short of 4 and takes the shortfall early return; the gate is then re-entered from RCLValidations handleNewValidation as each trusted validation arrives and eventually passes. A HEALTHY 5-node cluster therefore emits this counter every round, which is why it is safe to assert on a clean run -- unlike peer_disconnect_total or ledger_replay_fallback_total, it needs no fault injection, no [ledger_replay] stanza and no historical back-fill. The stage=\"pre_accept\" selector is asserted rather than the bare name so that the label dimension is proven to have reached Prometheus, matching the state_changes_total{from,to} pattern. Consequence for readers of the panels: a non-zero rate on Pre-Accept Quorum Shortfall Rate is NOT by itself a fault, and the panel description says so; the fault signature is that rate climbing well above the ledger-close rate while the tally on Trusted Validations vs Quorum Target stays flat below its target. If a future harness change makes the cluster single-node or standalone this assertion must move to a note: standalone_ short-circuits consensusBuilt before checkAccept, and getNeededValidations() returns 0 in standalone mode, so the gate can never report a shortfall.", "_b5_sweep_note": "WP-B5 Suspect 3 (per-sweep malloc_trim) adds one histogram and two counters. Only the histogram is asserted, by its Prometheus _bucket and _count series because the bare instrument name is not a series. It is unconditional on any running node: ApplicationImp::start arms the sweep timer before the workload begins, the interval is SizedItem::SweepInterval (Config.cpp: 10 s at nodeSize 0 through 120 s at nodeSize 4), and the full-validation profile runs 270 s of workload before Step 5 scrapes -- so at least two sweeps complete even in the slowest case, and each one records exactly one sample. The measurement itself is now unconditional too: it used to sit inside `if (journal.debug())` in MallocTrim.cpp, so a node at ordinary log level measured nothing; that gate now covers only the JLOG. The two counters are deliberately NOT asserted. sweep_malloc_trim_minor_faults_total is emitted only when the trim's minor-fault delta is greater than zero, and a trim on the small heap of a fresh localhost node routinely faults zero times -- the emit site publishes nothing rather than a zero, because a zero-valued series would claim the trim was measured as free when the honest statement is that there was nothing to fault on. sweep_malloc_trim_reclaimed_kb_total is emitted only when resident memory actually FELL across the trim, which on a node whose caches are still filling frequently does not happen (glibc has nothing above the top of the heap to release, and mmap-backed allocations are returned on free regardless of trimming). The validator has no per-metric optional flag, so asserting either would ship a permanently red CI check for a healthy run. Both are covered by exact-value unit tests in src/tests/libxrpl/telemetry/MetricMacros.cpp -- including the skip paths -- and rendered by the ledger-sync-health panel Sweep Heap-Trim Faults & Reclaim Rate. To make them assertable the harness would need a node with a large enough resident heap for a trim to reclaim, e.g. starting against an existing populated database rather than from genesis.", "_b5_rotation_note": "WP-B5 Suspect 4 (online_delete rotation extra writes) adds one observable gauge (rotation_state, sub-series in_flight and copy_forward) and one counter (rotation_copy_node_restore_total). NONE is asserted, because the 5-node localhost harness structurally CANNOT produce any of them -- this is a documented note rather than a check that would fail CI red. Two independent reasons. First, no rotation ever runs: xrpld-validator.cfg.template sets online_delete=256 and does not set advisory_delete, so SHAMapStoreImp's gate is validatedSeq >= lastRotated + 256 (SHAMapStoreImp.cpp), which needs 256 validated ledgers; at the network's several-seconds-per-ledger close rate that is on the order of 15-20 minutes, while the full-validation profile totals 270 s of workload before Step 5 scrapes. Second, even the in_flight flag needs a rotation to have started, and copy_forward additionally needs an ARCHIVE holding data that a fetch actually reads during the rotation window -- which requires a populated, already-rotated database, exactly the condition the hypothesis says is why this slowdown never appears on a fresh node. rotation_copy_node_restore_total is narrower still: it fires only for a clean tree node reachable from the validated state map whose sole on-disk copy was removed by an EARLIER rotation, so it needs at least two rotations plus real prior data loss. Note that rotation_state publishes no series at all when online_delete is not configured, by design: MetricsRegistry::registerRotationStateGauge dynamic_casts the node store to DatabaseRotating and returns early on failure, so an absent series means 'rotation is not configured' rather than the false 'rotation is free' a zero would report. All four signals are covered by exact-value unit tests in src/tests/libxrpl/telemetry/MetricMacros.cpp (including the not-configured and between-rotations cases) and rendered by the ledger-sync-health panels Online-Delete Rotation Window & Copy-Forward Writes and Rotation Node Re-Store Rate. To make them assertable the harness would need a step that starts a node against a pre-populated database that has already rotated at least once, or that lowers online_delete and advisory_delete far enough to force a rotation inside the run window and then re-scrapes before teardown.", "_round_histogram_note": "consensus_round_duration_ms is a native OTel histogram recorded once per consensus round in RCLConsensus, so it needs no collector configuration -- it rides the existing OTLP -> Prometheus path. It is asserted by its Prometheus _bucket and _count series because the bare instrument name is not a series. Both are unconditional on any running cluster: every node closes ledgers continuously, so a round completes within the harness window and the histogram is populated. Absence means the record site or the explicit-bucket view regressed, not that the node was idle. The instrument carries NO labels, so exactly one series exists per node and per bucket boundary." }, "node_health_gauges": { "description": "Node-health observable gauges (MetricsRegistry.cpp:997, :1081, :1102, :1161). server_info, build_info and db_metrics Observe unconditionally on every periodic export (build_info observes a literal 1; server_info and db_metrics read live services), so their series exist regardless of workload shape. complete_ledgers is the exception and is asserted on a narrower guarantee: its callback returns without observing when the range is empty (MetricsRegistry.cpp:1113-1114) and skips any segment that carries no '-' (:1122-1127), and a one-sequence range renders with no '-' (RangeSet.h:70-71), so it needs a complete range spanning at least two sequences. completeLedgers_ is filled by setFullLedger (LedgerMaster.cpp:862-863), which on a peered node is reached only from the publish path in doAdvance (LedgerMaster.cpp:1972) — closing a ledger is not enough, it has to validate. run-full-validation.sh waits for that before the workload starts, so on a healthy cluster the series always exists — a 5-node run yields 10 series, one start and one end per node. If this check ever fails, read the Step 3 output first: a run that logged 'No validated ledger' cannot produce this series and the cluster, not the exporter, is what broke.", "metrics": ["server_info", "build_info", "complete_ledgers", "db_metrics"] }, "overlay_reduce_relay": { "description": "Transaction reduce-relay efficiency gauge (MetricsRegistry.cpp:1354, peer-network dashboard). Backed by Overlay::txMetrics(); TxMetrics::json() emits txr_selected_cnt / txr_suppressed_cnt / txr_not_enabled_cnt unconditionally (TxMetrics.cpp:121-127), so the gauge always reports at least the selected_peers series.", "metrics": ["reduce_relay_metrics"] }, "overlay_overflow": { "description": "Job-queue transaction overflow total (MetricsRegistry.cpp:609, job-queue dashboard). An ObservableCounter that reads Overlay::getJqTransOverflow() and Observes unconditionally, so the series exists at value 0 even when no overflow occurs.", "metrics": ["jq_trans_overflow_total"] }, "validation_lifetime_counters": { "description": "Lifetime validation agreement/miss ObservableCounters (MetricsRegistry.cpp:1636, :1658, validator-health dashboard). Both callbacks reconcile the tracker and Observe unconditionally, so the series exist even on a node that has not yet agreed or missed (value 0). Only existence is asserted, never the value — validation_missed_total legitimately dominates on a non-validating node.", "metrics": ["validation_agreements_total", "validation_missed_total"] }, "not_asserted": { "description": "Emitted-and-dashboarded metrics deliberately left unasserted because they are workload-gated or defect-gated: the harness workload cannot guarantee they appear, and a check that fails on a healthy run is worse than no check. This group has no \"metrics\" key, so validate_telemetry.py skips it (validate_metrics iterates category_data.get(\"metrics\", [])). Promote an entry into an asserted group only after the workload is changed to guarantee it.", "metrics_excluded": { "rpc_method_errored_total": "MetricsRegistry.cpp:354, push counter — needs an RPC that returns an error. rpc_load_generator.py issues only well-formed server_info / fee / ledger / ripple_path_find calls, so no series may ever be created.", "ledger_history_mismatch_total": "MetricsRegistry.cpp:377, incremented only from LedgerHistory.cpp:332 on a built-vs-validated ledger mismatch. On a healthy run it never fires — asserting it would mean asserting a defect.", "txq_expired_total": "MetricsRegistry.cpp:379, incremented only at TxQ.cpp:1428 when a queued tx expires past its LastLedgerSequence. CI does run a txq-burst phase (workload-profiles.json:41, 30 s of single-type Payment at 60 TPS), but that does not guarantee sustained fee escalation followed by expiry: a run in which every other check passed still exposed only txq_metrics and no txq_expired_total.", "txq_dropped_total": "MetricsRegistry.cpp:381, incremented only at TxQ.cpp:1302 / :1347 on queue-full admission refusal. Same reason as txq_expired_total.", "getobject_rejected_total": "GetObjectMetricNames.h:81, emitted from PeerImp.cpp:2725/:2743 only for a TMGetObjectByHash message refused as oversize or malformed_ledgerhash. A cooperating cluster never sends one.", "getobject_request_objects": "GetObjectMetricNames.h:86, emitted from PeerImp.cpp:2926 only while serving an inbound TMGetObjectByHash. The XRPL_METRIC_* macros create their instrument lazily on first use (MetricMacros.h:174-285), so no series exists until a peer actually requests objects by hash — which a 5-node cluster started at genesis and already in sync may never do.", "getobject_lookup_us": "GetObjectMetricNames.h:95, PeerImp.cpp:2929. Same lazy-creation and same inbound-request gate as getobject_request_objects.", "getobject_lookups_total": "GetObjectMetricNames.h:100, PeerImp.cpp:2949/:2956. Same gate.", "getobject_charge": "GetObjectMetricNames.h:105, PeerImp.cpp:2931. Same gate." } }, "grafana_dashboards": { "description": "All 15 Grafana dashboards provisioned on disk under docker/telemetry/grafana/dashboards/ (UID == file stem for every one). validate_dashboards() checks that each UID resolves via GET /api/dashboards/uid/ and reports its panel count — it verifies provisioning and loadability, not panel data. log-derived-insights is included on that basis even though its panels are Loki-backed and CI runs with --skip-loki: the dashboard itself must still provision cleanly. Its panel data is not asserted anywhere.", "uids": [ "rpc-performance", "transaction-overview", "consensus-health", "ledger-operations", "peer-network", "peer-quality", "fee-market", "job-queue", "validator-health", "node-health", "network-traffic", "rpc-pathfinding", "overlay-traffic-detail", "ledger-data-sync", "ledger-sync-health", "log-derived-insights" ] } }