rippled/docker/telemetry/workload/expected_metrics.json

{
  "description": "Expected metric inventory for xrpld telemetry validation. Metric names have no prefix (the xrpld_ prefix was removed). beast::insight metrics are lowercased by formatName. Sourced from the live Grafana dashboards and MetricsRegistry.cpp.",
  "spanmetrics": {
    "description": "SpanMetrics-derived RED metrics from the OTel Collector spanmetrics connector.",
    "metrics": [
      "span_calls_total",
      "span_duration_milliseconds_bucket",
      "span_duration_milliseconds_count",
      "span_duration_milliseconds_sum"
    ],
    "required_labels": [
      "span_name",
      "status_code",
      "service_name",
      "span_kind"
    ],
    "dimension_labels": [
      "command",
      "rpc_status",
      "consensus_mode",
      "local",
      "proposal_trusted",
      "validation_trusted",
      "tx_type",
      "ter_result",
      "stage",
      "txq_status",
      "close_time_correct",
      "consensus_state",
      "suppressed"
    ],
    "_dimension_labels_note": "Bare label names as configured in otel-collector-config.yaml spanmetrics dimensions. Informational only (not asserted by the validator)."
  },
  "statsd_gauges": {
    "description": "beast::insight gauges exported via OTLP/HTTP to the collector (server=otel).",
    "metrics": [
      "ledgermaster_validated_ledger_age",
      "ledgermaster_published_ledger_age",
      "state_accounting_full_duration",
      "peer_finder_active_inbound_peers",
      "peer_finder_active_outbound_peers",
      "jobq_job_count"
    ]
  },
  "statsd_counters": {
    "description": "beast::insight counters exported via OTLP/HTTP. The OTel Prometheus exporter appends _total to monotonic counters.",
    "metrics": ["rpc_requests_total", "ledger_fetches_total"]
  },
  "overlay_traffic": {
    "description": "Overlay traffic metrics (subset \u2014 full list has 45+ categories).",
    "metrics": [
      "total_bytes_in",
      "total_bytes_out",
      "total_messages_in",
      "total_messages_out"
    ]
  },
  "phase9_nodestore": {
    "description": "Phase 9 NodeStore I/O observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label distinguishing sub-metrics.",
    "metrics": ["nodestore_state"]
  },
  "phase9_cache": {
    "description": "Phase 9 cache hit rate observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.",
    "metrics": ["cache_metrics"]
  },
  "phase9_txq": {
    "description": "Phase 9 transaction queue observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.",
    "metrics": ["txq_metrics"]
  },
  "phase9_rpc_method": {
    "description": "Phase 9 per-RPC-method counters (MetricsRegistry via OTLP).",
    "metrics": ["rpc_method_started_total"]
  },
  "rpc_in_flight": {
    "description": "In-flight RPC gauge via the XRPL_METRIC_UPDOWN_ADD call-site macro (PerfLogImp.cpp, +1 rpcStart / -1 rpcEnd). UpDownCounter: no _total suffix.",
    "metrics": ["rpc_in_flight_requests"]
  },
  "phase9_objects": {
    "description": "Phase 9 counted object instances observable gauge (MetricsRegistry via OTLP).",
    "metrics": ["object_count"]
  },
  "phase9_load": {
    "description": "Phase 9 fee escalation and load factor observable gauge (MetricsRegistry via OTLP).",
    "metrics": ["load_factor_metrics"]
  },
  "parity_validation_agreement": {
    "description": "External dashboard parity: validation agreement percentages (MetricsRegistry).",
    "metrics": [
      "validation_agreement{metric=\"agreement_pct_1h\"}",
      "validation_agreement{metric=\"agreement_pct_24h\"}"
    ]
  },
  "parity_validator_health": {
    "description": "External dashboard parity: validator health indicators (MetricsRegistry).",
    "metrics": [
      "validator_health{metric=\"amendment_blocked\"}",
      "validator_health{metric=\"unl_expiry_days\"}"
    ]
  },
  "parity_peer_quality": {
    "description": "External dashboard parity: peer quality metrics (MetricsRegistry).",
    "metrics": [
      "peer_quality{metric=\"peer_latency_p90_ms\"}",
      "peer_quality{metric=\"peers_insane_count\"}"
    ]
  },
  "parity_ledger_economy": {
    "description": "External dashboard parity: ledger economy metrics (MetricsRegistry).",
    "metrics": [
      "ledger_economy{metric=\"base_fee_xrp\"}",
      "ledger_economy{metric=\"transaction_rate\"}"
    ]
  },
  "parity_state_tracking": {
    "description": "External dashboard parity: server state tracking (MetricsRegistry).",
    "metrics": ["state_tracking{metric=\"state_value\"}"]
  },
  "parity_counters": {
    "description": "External dashboard parity: monotonic counters (MetricsRegistry).",
    "metrics": [
      "ledgers_closed_total",
      "validations_sent_total",
      "state_changes_total"
    ]
  },
  "parity_storage": {
    "description": "External dashboard parity: storage detail metrics (MetricsRegistry).",
    "metrics": ["storage_detail{metric=\"stored_object_bytes\"}"]
  },
  "sync_diagnostics": {
    "description": "Fresh-node sync diagnostics (native metrics). Bootstrap (Domain 0) and acquire-pipeline signals rendered by the ledger-sync-health dashboard. Names are appended one per signal. Histograms are listed by their Prometheus _bucket series (the bare instrument name is not a series). The two observable gauges carry an inline metric= selector so the specific sub-series is asserted, matching the parity_* groups.",
    "metrics": [
      "dns_resolve_total",
      "dns_resolve_latency_ms_bucket",
      "overlay_connect_total",
      "overlay_dial_latency_ms_bucket",
      "handshake_negotiation_fail_total",
      "unl_fetch_total",
      "unl_quorum{metric=\"trusted_keys\"}",
      "unl_quorum{metric=\"quorum\"}",
      "clock_close_offset_seconds{metric=\"offset\"}",
      "sync_state{metric=\"initial_full_duration_us\"}",
      "sync_state{metric=\"network_ledger_gate\"}",
      "sync_state{metric=\"server_stall_seconds\"}",
      "sync_state{metric=\"ledgers_behind\"}",
      "server_stall_events_total",
      "state_changes_total{from!=\"\",to!=\"\"}",
      "sync_acquire{metric=\"missing_state_nodes_max\"}",
      "sync_acquire{metric=\"missing_tx_nodes_max\"}",
      "sync_acquire{metric=\"received_data_depth\"}",
      "sync_acquire{metric=\"in_flight\"}",
      "shamap_cache_hit_rate{metric=\"treenode\"}",
      "jobq_saturation{metric=\"running_tasks\"}",
      "jobq_saturation{metric=\"worker_threads\"}",
      "jobq_saturation{metric=\"total_waiting\"}",
      "peer_ledger_supply{metric=\"peers_reporting\"}",
      "peer_ledger_supply{metric=\"peers_serving_validated\"}",
      "peer_ledger_supply{metric=\"peers_serving_next\"}",
      "peer_ledger_supply{metric=\"supply_min_seq\"}",
      "peer_ledger_supply{metric=\"supply_max_seq\"}",
      "peerfinder_slot_census{metric=\"out_active\"}",
      "peerfinder_slot_census{metric=\"out_max\"}",
      "peerfinder_slot_census{metric=\"in_active\"}",
      "peerfinder_slot_census{metric=\"in_max\"}",
      "peerfinder_slot_census{metric=\"connecting\"}",
      "peerfinder_slot_census{metric=\"fixed_configured\"}",
      "peerfinder_slot_census{metric=\"fixed_active\"}",
      "peerfinder_slot_census{metric=\"bootcache\"}",
      "peerfinder_slot_census{metric=\"livecache\"}",
      "amendment_block{metric=\"warned\"}",
      "amendment_block{metric=\"seconds_to_block\"}",
      "peer_accept_total",
      "nodestore_state{metric=\"node_writes\"}",
      "nodestore_state{metric=\"node_reads_total\"}",
      "nodestore_state{metric=\"read_mean_us\"}",
      "ledger_quorum_publish{metric=\"trusted_validation_tally\"}",
      "ledger_quorum_publish{metric=\"quorum_target\"}",
      "ledger_quorum_publish{metric=\"time_to_first_validated_us\"}",
      "ledger_quorum_publish{metric=\"publish_lag\"}",
      "ledger_quorum_shortfall_total{stage=\"pre_accept\"}",
      "consensus_round_duration_ms_bucket",
      "consensus_round_duration_ms_count",
      "nodestore_state{metric=\"node_writes_duration_us\"}",
      "nodestore_state{metric=\"node_reads_duration_us\"}",
      "unl_quorum{metric=\"quorum_disabled\"}",
      "sweep_malloc_trim_us_bucket",
      "sweep_malloc_trim_us_count"
    ],
    "_acquire_note": "The four sync_acquire sub-series and shamap_cache_hit_rate are unconditional: both are observable gauges whose callbacks observe every series on each collection tick, so each is present even when the value is 0 (an idle node reports in_flight=0 and missing_state_nodes_max=0, and a cold cache reports a 0.0 hit rate). Absence, not a zero, is the regression. The three WP-A3 counters (sync_acquire_source_total, sync_addnode_total, sync_acquire_no_progress_total) are deliberately NOT asserted here: all three are emitted only from InboundLedger, which runs only when a node must fetch a ledger it lacks. expected_spans.json already marks the ledger.acquire span optional for exactly this reason (\"A healthy local cluster rarely back-fills history\"), and the metric validator has no per-metric optional flag, so listing them would fail the harness red on a healthy run. They are covered by exact-value unit tests in src/tests/libxrpl/telemetry/MetricMacros.cpp and by the ledger-sync-health panels; add them here only alongside a harness step that forces a real acquire (e.g. starting a node against an existing ledger history).",
    "_jobq_note": "The three jobq_saturation series are unconditional: it is an observable gauge whose callback observes all three fields on every collection tick, so each series exists even when the value is 0, and absence rather than a zero is the regression. worker_threads is asserted because it is the denominator of the dashboard saturation ratio, and it is always at least 1 (the JobQueue ctor gives standalone mode exactly one worker), so a zero or missing reading there means the accessor regressed rather than the node being idle. The per-job-type waiting/running/deferred counts are published separately by JobQueue::collect() as the beast::insight gauges jobq_<type>_waiting / _running / _deferred, which the collector translates; they are covered by the StatsD-derived groups, not here.",
    "_conditional_note": "handshake_negotiation_fail_total and unl_fetch_total are conditional under the local harness: the first only exists once a handshake is rejected, and the second needs a [validator_list_sites] entry (run-full-validation.sh generates a static [validators] file instead). The validator has no per-metric optional flag, so if either reports 0 series in a harness run, move it out of this group rather than weakening the check.",
    "_sync_state_note": "The four sync_state sub-series are unconditional: the gauge observes all four on every collection tick, so each is present as a series even when its value is 0 (a node that never reached FULL reports initial_full_duration_us=0, and a healthy node reports server_stall_seconds=0). The check asserts series presence, not a non-zero value, which is exactly right here \u2014 a zero is a meaningful reading for these signals, and absence is the regression. server_stall_events_total is likewise always present because the observable counter reports the tally (0 or more) every tick. state_changes_total is asserted here with a from!=\"\",to!=\"\" selector rather than bare (parity_counters already asserts the bare name): the selector is what proves the WP-A2 {from,to} label dimension actually reached Prometheus, so a regression to the old unlabelled counter fails this check instead of silently passing on the bare name. It needs at least one real mode transition, which any node reaching connected/syncing produces during startup.",
    "_a7_note": "WP-A7 adds three observable gauges and four counters. The 16 gauge sub-series (peer_ledger_supply, peerfinder_slot_census, amendment_block) are unconditional and asserted individually: each callback in MetricsRegistry.cpp calls observe() for every field on every collection tick with no early return between them, so the series exists whatever the value. That includes the two sentinel readings \u2014 a node whose peers have advertised nothing reports peer_ledger_supply{metric=\"supply_min_seq\"} = 0 meaning unknown, and a node with no pending amendment reports amendment_block{metric=\"seconds_to_block\"} = -1 meaning healthy. Absence, not the sentinel, is the regression. Of the four counters only peer_accept_total is asserted: run-full-validation.sh gives every node a [port_peer] on 0.0.0.0 and lists the other four nodes in [ips], so all 5 nodes dial each other and each one is also dialled, which means OverlayImpl::onHandoff runs and reports outcome=accepted (or slot_refused/no_slot on the duplicate half of each mutual dial) on every node. It is asserted bare rather than with an outcome= selector because which outcome a given node records depends on dial ordering, which the harness does not control. The other three counters are deliberately NOT asserted. peer_disconnect_total is emitted only from PeerImp::close, and a healthy 5-node localhost cluster holds its 4 fixed peers for the whole run: the timer-driven reasons need maxUnknownTime (600 s) or maxDivergedTime (300 s) to elapse (Config.h) while the full-validation profile totals well under that, and the shutdown reasons only fire during teardown, which happens in run-full-validation.sh after Step 5 has already scraped. serve_refused_total needs a peer to ask this node for a ledger, tx set or object it cannot serve \u2014 on a cluster where every node has the same complete history from genesis, getLedger()/getTxSet() succeed and the send queues never approach Tuning::kDropSendQueue. ledger_jump_total needs NetworkOPsImp::switchLastClosedLedger, reached only when consensus reports an LCL this node did not build on; a healthy 5-node cluster agrees every round, so it never jumps. The metric validator has no per-metric optional flag, so listing any of the three would fail the harness red on a healthy run \u2014 the same reasoning _acquire_note applies to the WP-A3 InboundLedger counters. All four counters are covered by exact-value unit tests in src/tests/libxrpl/telemetry/MetricMacros.cpp and rendered by the ledger-sync-health panels Peer Disconnects by Reason, Ledger/Object Serve Refusals and Byzantine Ledger Jumps. To make them assertable the harness would need a fault-injection step: kill one node mid-run and re-scrape before teardown (peer_disconnect_total, reason=read_error/graceful), request a ledger sequence outside the cluster's history or drive a node past its send-queue limit (serve_refused_total), and start a node on a divergent chain tip or partition the cluster and heal it (ledger_jump_total).",
    "_a6_note": "WP-A6 originally added a separate nodestore_latency gauge; it was retired because every one of its sub-metrics had an exact counterpart on nodestore_state computed from the same Database accessor, so the five entries above are the surviving equivalents (write_count -> node_writes, read_count -> node_reads_total, write_duration_us -> node_writes_duration_us, read_duration_us -> node_reads_duration_us, read_mean_us unchanged). All four totals are unconditional: MetricsRegistry::observeNodeStoreTotals observes them on every collection tick with no early return before them, so a series exists whatever the value and a node that has written nothing reports node_writes=0 rather than dropping the series. read_mean_us is safe because any node that has opened a ledger has already fetched objects, so the fetch count is non-zero. write_mean_us is NOT asserted, but only because the two means are the sub-series that scaledMean() omits when their denominator is zero, and this validator has no per-metric optional flag -- not because the numerator is missing. That older caveat is gone: all three concrete store paths now time themselves through Database::recordStoreDuration() (DatabaseNodeImp::store, DatabaseRotatingImp::store and Database::importInternal), so write_mean_us is live on an ordinary node and only a node that has performed literally zero stores would lack it. It can be promoted to an assertion once a harness run confirms it present. WP-A6's two replay counters (ledger_replay_fallback_total, ledger_replay_outcome_total) are likewise NOT asserted, for the same reason _acquire_note gives for the WP-A3 InboundLedger counters: both are emitted only from the ledger-replay path, which requires the [ledger_replay] config stanza AND a real historical back-fill against peers that support the LedgerReplay protocol feature. run-full-validation.sh starts a fresh local cluster with no history to back-fill, so no replay task is ever created and neither counter can produce a series. Both are covered by exact-value unit tests in src/tests/libxrpl/telemetry/MetricMacros.cpp and rendered by the ledger-sync-health panels Replay Fallback to Full Acquire and Replay Outcomes. To make them assertable the harness would need to enable [ledger_replay] and start a node against an existing ledger history so it back-fills through the replay path.",
    "_a5_note": "WP-A5 adds one observable gauge (ledger_quorum_publish) and one counter (ledger_quorum_shortfall_total). All four gauge sub-series are asserted and are unconditional: registerLedgerQuorumPublishGauge's callback in MetricsRegistry.cpp calls observe() for every field on every collection tick with no early return between them, and each accessor is a plain relaxed atomic load that always returns a value, so the series exists whatever the reading. That deliberately includes the three diagnostic zeros: a node that has never had a gate evaluated reports trusted_validation_tally=0 and quorum_target=0, one that has never fully validated reports time_to_first_validated_us=0, and one that is caught up reports publish_lag=0. Absence, not the zero, is the regression -- the same reasoning _sync_state_note gives for initial_full_duration_us. Note the sentinel: when the trusted list disables quorum entirely, getNeededValidations() returns SIZE_MAX and LedgerMaster reports quorum_target as int64 max rather than letting the cast wrap to -1, so the target reads far above any tally instead of inverting the comparison (the same fix as the unl_quorum gauge). ledger_quorum_shortfall_total IS asserted, which differs from the WP-A3/A6/A7 counters, and the reason is that this counter does not need a fault to fire. RCLConsensus::Adaptor::doAccept issues this node's own validation and then calls ledgerMaster_.consensusBuilt immediately (RCLConsensus.cpp), which calls checkAccept on the freshly built ledger (LedgerMaster.cpp) BEFORE the peers' validations for that same ledger have arrived. With the harness's 5 validators the quorum is max(ceil(5*0.8), ceil(5*0.6)) = 4 (ValidatorList::calculateQuorum), so that first evaluation of each round tallies short of 4 and takes the shortfall early return; the gate is then re-entered from RCLValidations handleNewValidation as each trusted validation arrives and eventually passes. A HEALTHY 5-node cluster therefore emits this counter every round, which is why it is safe to assert on a clean run -- unlike peer_disconnect_total or ledger_replay_fallback_total, it needs no fault injection, no [ledger_replay] stanza and no historical back-fill. The stage=\"pre_accept\" selector is asserted rather than the bare name so that the label dimension is proven to have reached Prometheus, matching the state_changes_total{from,to} pattern. Consequence for readers of the panels: a non-zero rate on Pre-Accept Quorum Shortfall Rate is NOT by itself a fault, and the panel description says so; the fault signature is that rate climbing well above the ledger-close rate while the tally on Trusted Validations vs Quorum Target stays flat below its target. If a future harness change makes the cluster single-node or standalone this assertion must move to a note: standalone_ short-circuits consensusBuilt before checkAccept, and getNeededValidations() returns 0 in standalone mode, so the gate can never report a shortfall.",
    "_b5_sweep_note": "WP-B5 Suspect 3 (per-sweep malloc_trim) adds one histogram and two counters. Only the histogram is asserted, by its Prometheus _bucket and _count series because the bare instrument name is not a series. It is unconditional on any running node: ApplicationImp::start arms the sweep timer before the workload begins, the interval is SizedItem::SweepInterval (Config.cpp: 10 s at nodeSize 0 through 120 s at nodeSize 4), and the full-validation profile runs 270 s of workload before Step 5 scrapes -- so at least two sweeps complete even in the slowest case, and each one records exactly one sample. The measurement itself is now unconditional too: it used to sit inside `if (journal.debug())` in MallocTrim.cpp, so a node at ordinary log level measured nothing; that gate now covers only the JLOG. The two counters are deliberately NOT asserted. sweep_malloc_trim_minor_faults_total is emitted only when the trim's minor-fault delta is greater than zero, and a trim on the small heap of a fresh localhost node routinely faults zero times -- the emit site publishes nothing rather than a zero, because a zero-valued series would claim the trim was measured as free when the honest statement is that there was nothing to fault on. sweep_malloc_trim_reclaimed_kb_total is emitted only when resident memory actually FELL across the trim, which on a node whose caches are still filling frequently does not happen (glibc has nothing above the top of the heap to release, and mmap-backed allocations are returned on free regardless of trimming). The validator has no per-metric optional flag, so asserting either would ship a permanently red CI check for a healthy run. Both are covered by exact-value unit tests in src/tests/libxrpl/telemetry/MetricMacros.cpp -- including the skip paths -- and rendered by the ledger-sync-health panel Sweep Heap-Trim Faults & Reclaim Rate. To make them assertable the harness would need a node with a large enough resident heap for a trim to reclaim, e.g. starting against an existing populated database rather than from genesis.",
    "_b5_rotation_note": "WP-B5 Suspect 4 (online_delete rotation extra writes) adds one observable gauge (rotation_state, sub-series in_flight and copy_forward) and one counter (rotation_copy_node_restore_total). NONE is asserted, because the 5-node localhost harness structurally CANNOT produce any of them -- this is a documented note rather than a check that would fail CI red. Two independent reasons. First, no rotation ever runs: xrpld-validator.cfg.template sets online_delete=256 and does not set advisory_delete, so SHAMapStoreImp's gate is validatedSeq >= lastRotated + 256 (SHAMapStoreImp.cpp), which needs 256 validated ledgers; at the network's several-seconds-per-ledger close rate that is on the order of 15-20 minutes, while the full-validation profile totals 270 s of workload before Step 5 scrapes. Second, even the in_flight flag needs a rotation to have started, and copy_forward additionally needs an ARCHIVE holding data that a fetch actually reads during the rotation window -- which requires a populated, already-rotated database, exactly the condition the hypothesis says is why this slowdown never appears on a fresh node. rotation_copy_node_restore_total is narrower still: it fires only for a clean tree node reachable from the validated state map whose sole on-disk copy was removed by an EARLIER rotation, so it needs at least two rotations plus real prior data loss. Note that rotation_state publishes no series at all when online_delete is not configured, by design: MetricsRegistry::registerRotationStateGauge dynamic_casts the node store to DatabaseRotating and returns early on failure, so an absent series means 'rotation is not configured' rather than the false 'rotation is free' a zero would report. All four signals are covered by exact-value unit tests in src/tests/libxrpl/telemetry/MetricMacros.cpp (including the not-configured and between-rotations cases) and rendered by the ledger-sync-health panels Online-Delete Rotation Window & Copy-Forward Writes and Rotation Node Re-Store Rate. To make them assertable the harness would need a step that starts a node against a pre-populated database that has already rotated at least once, or that lowers online_delete and advisory_delete far enough to force a rotation inside the run window and then re-scrapes before teardown.",
    "_round_histogram_note": "consensus_round_duration_ms is a native OTel histogram recorded once per consensus round in RCLConsensus, so it needs no collector configuration -- it rides the existing OTLP -> Prometheus path. It is asserted by its Prometheus _bucket and _count series because the bare instrument name is not a series. Both are unconditional on any running cluster: every node closes ledgers continuously, so a round completes within the harness window and the histogram is populated. Absence means the record site or the explicit-bucket view regressed, not that the node was idle. The instrument carries NO labels, so exactly one series exists per node and per bucket boundary."
  },
  "grafana_dashboards": {
    "description": "All Grafana dashboards that must render data (UIDs as provisioned on disk under docker/telemetry/grafana/dashboards/).",
    "uids": [
      "rpc-performance",
      "transaction-overview",
      "consensus-health",
      "ledger-operations",
      "peer-network",
      "peer-quality",
      "fee-market",
      "job-queue",
      "validator-health",
      "node-health",
      "network-traffic",
      "rpc-pathfinding",
      "overlay-traffic-detail",
      "ledger-data-sync",
      "ledger-sync-health"
    ]
  }
}