rippled/docker/telemetry/workload/expected_metrics.json

{
  "description": "Expected metric inventory for xrpld telemetry validation. Metric names have no prefix (the xrpld_ prefix was removed). beast::insight metrics are lowercased by formatName. Every name here was verified against its declaration in MetricsRegistry.cpp or include/xrpl/telemetry/GetObjectMetricNames.h and against a panel query under docker/telemetry/grafana/dashboards/. IMPORTANT: validate_telemetry.py has no notion of an optional metric — validate_metrics() iterates every group that has a \"metrics\" key and hard-fails any name with 0 Prometheus series after a 45 s poll. A metric is therefore listed only when the harness workload guarantees it will appear: observable gauges/counters whose callbacks Observe unconditionally (series exist at value 0), or push counters/histograms on a path every run exercises. Workload-gated and defect-gated names are recorded in the \"not_asserted\" group, which intentionally has no \"metrics\" key so the validator skips it. Only series existence is checked, never a value, except for the four bounds checks hardcoded in PARITY_VALUE_SANITY. A group may additionally declare \"required_labels\": each label there becomes one check that at least one of the group's series carries it with a non-empty value, matched as <label>!=\"\" because Prometheus cannot tell an absent label from an empty one. The same guarantee rule applies — list a label only where the workload guarantees it.",
  "spanmetrics": {
    "description": "SpanMetrics-derived RED metrics from the OTel Collector spanmetrics connector.",
    "metrics": [
      "span_calls_total",
      "span_duration_milliseconds_bucket",
      "span_duration_milliseconds_count",
      "span_duration_milliseconds_sum"
    ],
    "required_labels": [
      "span_name",
      "status_code",
      "service_name",
      "span_kind"
    ],
    "dimension_labels": [
      "command",
      "rpc_status",
      "consensus_mode",
      "local",
      "proposal_trusted",
      "validation_trusted",
      "tx_type",
      "ter_result",
      "stage",
      "txq_status",
      "close_time_correct",
      "consensus_state",
      "suppressed"
    ],
    "_dimension_labels_note": "Bare label names as configured in otel-collector-config.yaml spanmetrics dimensions. Informational only (not asserted by the validator)."
  },
  "statsd_gauges": {
    "description": "beast::insight gauges exported via OTLP/HTTP to the collector (server=otel).",
    "metrics": [
      "ledgermaster_validated_ledger_age",
      "ledgermaster_published_ledger_age",
      "state_accounting_full_duration",
      "peer_finder_active_inbound_peers",
      "peer_finder_active_outbound_peers",
      "jobq_job_count"
    ]
  },
  "statsd_counters": {
    "description": "beast::insight counters exported via OTLP/HTTP. The OTel Prometheus exporter appends _total to monotonic counters.",
    "metrics": ["rpc_requests_total", "ledger_fetches_total"]
  },
  "io_latency": {
    "description": "io_context scheduling-latency histogram — a beast::insight Event exported over OTLP (Application.cpp:515, makeEvent('ios_latency') with the default millisecond unit). It is the one beast Event the node itself guarantees, which is why it is asserted while rpc_time_milliseconds and the jobq_* pairs sit in not_asserted: the sampler starts on the unconditional startup path (Application.cpp:1697, outside the 'if (withTimers)' guard), and its handler always emits the first sample whatever its value (Application.cpp:185-189, 'firstSample_.exchange(false) || lastSample >= 10ms', with a comment stating the point is to register the metric downstream). The OTel histogram is cumulative, so that one notify creates series that persist for the rest of the run. Named with the _bucket/_count/_sum suffixes the Prometheus exporter emits for a histogram — no bare ios_latency_milliseconds series exists, the same convention rpc_method_us and span_duration_milliseconds follow, and all 6 panels that query it use ios_latency_milliseconds_bucket. Only existence is asserted: past the first sample the handler reports only latencies >= 10 ms, so neither the sample count nor the value is predictable.",
    "metrics": [
      "ios_latency_milliseconds_bucket",
      "ios_latency_milliseconds_count",
      "ios_latency_milliseconds_sum"
    ]
  },
  "overlay_traffic": {
    "description": "Overlay traffic metrics (subset — full list has 45+ categories).",
    "metrics": [
      "total_bytes_in",
      "total_bytes_out",
      "total_messages_in",
      "total_messages_out"
    ]
  },
  "nodestore_io": {
    "description": "NodeStore I/O observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label distinguishing sub-metrics.",
    "metrics": ["nodestore_state"]
  },
  "cache_hit_rates": {
    "description": "Cache hit rate observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.",
    "metrics": ["cache_metrics"]
  },
  "transaction_queue": {
    "description": "Transaction queue observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.",
    "metrics": ["txq_metrics"]
  },
  "rpc_method_detail": {
    "description": "Per-RPC-method counters and duration histogram (MetricsRegistry.cpp:351-357). rpc_method_errored_total is deliberately absent — see not_asserted below. rpc_method_us is a Histogram, so the Prometheus exporter emits only the _bucket/_count/_sum triple and there is no bare rpc_method_us series to match — same convention as span_duration_milliseconds in the spanmetrics group above.",
    "metrics": [
      "rpc_method_started_total",
      "rpc_method_finished_total",
      "rpc_method_us_bucket",
      "rpc_method_us_count",
      "rpc_method_us_sum"
    ]
  },
  "job_queue": {
    "description": "Job-queue counters and latency histograms (MetricsRegistry.cpp:360-366). Every xrpld job passes through these, so they populate under any workload. Both histograms are recorded in the same function bodies as job_started_total / job_finished_total, under the same guard and with the same labels, so their presence is equally guaranteed. They are named with the _bucket/_count/_sum suffixes the Prometheus exporter emits: regression-metrics.json and the job-queue dashboard both query job_queued_us_bucket / job_running_us_bucket, and no bare series exists. This group also carries the xrpl_node_id required_labels assertion. 070d29b465 added the xrpl.node.id resource attribute so Grafana Cloud trace ingest would stop folding distinct nodes into one — ingest groups ResourceSpans but ignores service.instance.id, so before the attribute existed the boards reported one node's ledger.build five times over and saw 2 of 9 nodes. Nothing asserted it, so a regression would silently restore the folding. It is asserted here rather than on a statsd_* group because these metrics are MetricsRegistry-backed and guaranteed by any workload, whereas the beast::insight meter provider is constructed before the wallet DB exists and so has no node public key to stamp: those metrics legitimately lack the label and 070d29b465 omits it rather than writing it blank.",
    "required_labels": ["xrpl_node_id"],
    "metrics": [
      "job_queued_total",
      "job_started_total",
      "job_finished_total",
      "job_queued_us_bucket",
      "job_queued_us_count",
      "job_queued_us_sum",
      "job_running_us_bucket",
      "job_running_us_count",
      "job_running_us_sum"
    ]
  },
  "rpc_in_flight": {
    "description": "In-flight RPC gauge via the XRPL_METRIC_UPDOWN_ADD call-site macro (PerfLogImp.cpp, +1 rpcStart / -1 rpcEnd). UpDownCounter: no _total suffix.",
    "metrics": ["rpc_in_flight_requests"]
  },
  "object_counts": {
    "description": "Counted object instances observable gauge (MetricsRegistry via OTLP).",
    "metrics": ["object_count"]
  },
  "load_factors": {
    "description": "Fee escalation and load factor observable gauge (MetricsRegistry via OTLP).",
    "metrics": ["load_factor_metrics"]
  },
  "parity_validation_agreement": {
    "description": "External dashboard parity: validation agreement percentages (MetricsRegistry).",
    "metrics": [
      "validation_agreement{metric=\"agreement_pct_1h\"}",
      "validation_agreement{metric=\"agreement_pct_24h\"}"
    ]
  },
  "parity_validator_health": {
    "description": "External dashboard parity: validator health indicators (MetricsRegistry).",
    "metrics": [
      "validator_health{metric=\"amendment_blocked\"}",
      "validator_health{metric=\"unl_expiry_days\"}"
    ]
  },
  "parity_peer_quality": {
    "description": "External dashboard parity: peer quality metrics (MetricsRegistry).",
    "metrics": [
      "peer_quality{metric=\"peer_latency_p90_ms\"}",
      "peer_quality{metric=\"peers_insane_count\"}"
    ]
  },
  "parity_ledger_economy": {
    "description": "External dashboard parity: ledger economy metrics (MetricsRegistry.cpp:1401). transaction_rate is observed on every export, in both branches of the ledger-age test (MetricsRegistry.cpp:1444-1451). base_fee_xrp is observed only inside the 'if (ledger)' guard on getValidatedLedger() (MetricsRegistry.cpp:1418-1423), and that returns validLedger_ (LedgerMaster.cpp:1569-1572), which stays null until a ledger validates — the same precondition complete_ledgers has. Both are asserted because run-full-validation.sh waits for a validated ledger before running the workload. base_fee_xrp absent while transaction_rate is present is the signature of a cluster that never validated, not of a missing metric.",
    "metrics": [
      "ledger_economy{metric=\"base_fee_xrp\"}",
      "ledger_economy{metric=\"transaction_rate\"}"
    ]
  },
  "parity_state_tracking": {
    "description": "External dashboard parity: server state tracking (MetricsRegistry).",
    "metrics": ["state_tracking{metric=\"state_value\"}"]
  },
  "parity_counters": {
    "description": "External dashboard parity: monotonic counters (MetricsRegistry). validations_checked_total is incremented unconditionally at the top of NetworkOPsImp::recvValidation (NetworkOPs.cpp:2681), and run-full-validation.sh brings up a 5-node validator cluster, so inbound validations are guaranteed.",
    "metrics": [
      "ledgers_closed_total",
      "validations_sent_total",
      "validations_checked_total",
      "state_changes_total"
    ]
  },
  "parity_storage": {
    "description": "External dashboard parity: storage detail metrics (MetricsRegistry).",
    "metrics": ["storage_detail{metric=\"stored_object_bytes\"}"]
  },
  "node_health_gauges": {
    "description": "Node-health observable gauges (MetricsRegistry.cpp:997, :1081, :1102, :1161). server_info, build_info and db_metrics Observe unconditionally on every periodic export (build_info observes a literal 1; server_info and db_metrics read live services), so their series exist regardless of workload shape. complete_ledgers is the exception and is asserted on a narrower guarantee: its callback returns without observing when the range is empty (MetricsRegistry.cpp:1113-1114) and skips any segment that carries no '-' (:1122-1127), and a one-sequence range renders with no '-' (RangeSet.h:70-71), so it needs a complete range spanning at least two sequences. completeLedgers_ is filled by setFullLedger (LedgerMaster.cpp:862-863), which on a peered node is reached only from the publish path in doAdvance (LedgerMaster.cpp:1972) — closing a ledger is not enough, it has to validate. run-full-validation.sh waits for that before the workload starts, so on a healthy cluster the series always exists — a 5-node run yields 10 series, one start and one end per node. If this check ever fails, read the Step 3 output first: a run that logged 'No validated ledger' cannot produce this series and the cluster, not the exporter, is what broke.",
    "metrics": ["server_info", "build_info", "complete_ledgers", "db_metrics"]
  },
  "overlay_reduce_relay": {
    "description": "Transaction reduce-relay efficiency gauge (MetricsRegistry.cpp:1354, peer-network dashboard). Backed by Overlay::txMetrics(); TxMetrics::json() emits txr_selected_cnt / txr_suppressed_cnt / txr_not_enabled_cnt unconditionally (TxMetrics.cpp:121-127), so the gauge always reports at least the selected_peers series.",
    "metrics": ["reduce_relay_metrics"]
  },
  "overlay_overflow": {
    "description": "Job-queue transaction overflow total (MetricsRegistry.cpp:609, job-queue dashboard). An ObservableCounter that reads Overlay::getJqTransOverflow() and Observes unconditionally, so the series exists at value 0 even when no overflow occurs.",
    "metrics": ["jq_trans_overflow_total"]
  },
  "validation_lifetime_counters": {
    "description": "Lifetime validation agreement/miss ObservableCounters (MetricsRegistry.cpp:1636, :1658, validator-health dashboard). Both callbacks reconcile the tracker and Observe unconditionally, so the series exist even on a node that has not yet agreed or missed (value 0). Only existence is asserted, never the value — validation_missed_total legitimately dominates on a non-validating node.",
    "metrics": ["validation_agreements_total", "validation_missed_total"]
  },
  "not_asserted": {
    "description": "Emitted-and-dashboarded metrics deliberately left unasserted because they are workload-gated or defect-gated: the harness workload cannot guarantee they appear, and a check that fails on a healthy run is worse than no check. This group has no \"metrics\" key, so validate_telemetry.py skips it (validate_metrics iterates category_data.get(\"metrics\", [])). Promote an entry into an asserted group only after the workload is changed to guarantee it.",
    "metrics_excluded": {
      "rpc_method_errored_total": "MetricsRegistry.cpp:354, push counter — needs an RPC that returns an error. rpc_load_generator.py issues only well-formed server_info / fee / ledger / ripple_path_find calls, so no series may ever be created.",
      "ledger_history_mismatch_total": "MetricsRegistry.cpp:377, incremented only from LedgerHistory.cpp:332 on a built-vs-validated ledger mismatch. On a healthy run it never fires — asserting it would mean asserting a defect.",
      "txq_expired_total": "MetricsRegistry.cpp:379, incremented only at TxQ.cpp:1428 when a queued tx expires past its LastLedgerSequence. CI does run a txq-burst phase (workload-profiles.json:41, 30 s of single-type Payment at 60 TPS), but that does not guarantee sustained fee escalation followed by expiry: a run in which every other check passed still exposed only txq_metrics and no txq_expired_total.",
      "txq_dropped_total": "MetricsRegistry.cpp:381, incremented only at TxQ.cpp:1302 / :1347 on queue-full admission refusal. Same reason as txq_expired_total.",
      "getobject_rejected_total": "GetObjectMetricNames.h:81, emitted from PeerImp.cpp:2725/:2743 only for a TMGetObjectByHash message refused as oversize or malformed_ledgerhash. A cooperating cluster never sends one.",
      "getobject_request_objects": "GetObjectMetricNames.h:86, emitted from PeerImp.cpp:2926 only while serving an inbound TMGetObjectByHash. The XRPL_METRIC_* macros create their instrument lazily on first use (MetricMacros.h:174-285), so no series exists until a peer actually requests objects by hash — which a 5-node cluster started at genesis and already in sync may never do.",
      "getobject_lookup_us": "GetObjectMetricNames.h:95, PeerImp.cpp:2929. Same lazy-creation and same inbound-request gate as getobject_request_objects.",
      "getobject_lookups_total": "GetObjectMetricNames.h:100, PeerImp.cpp:2949/:2956. Same gate.",
      "getobject_charge": "GetObjectMetricNames.h:105, PeerImp.cpp:2931. Same gate.",
      "rpc_size_bytes": "ServerHandler.cpp:191, group('rpc')->makeEvent('size', Unit::Bytes). The OTLP Prometheus exporter derives the metric-name suffix from the declared unit, so a byte unit yields rpc_size_bytes. The Unit::Bytes declaration itself landed earlier, in 76c9051203; what 24094e427b changed was the exporter finally consuming it, replacing a hardcoded CreateDoubleHistogram(name, 'Duration in ms', 'ms') with otelUnitDescription(unit)/otelUnitCode(unit), and that is what renamed the series off rpc_size_milliseconds and the millisecond bucket ladder. Neither name was ever recorded here, so the harness could confirm neither the rename nor a regression back onto that ladder. Notified from ServerHandler::processRequest:1133, the HTTP JSON-RPC path — it computes an HTTP status and appends a trailing newline — and the load generators are WebSocket-only, the same gate regression-metrics.json:4 records for rpc.process, so only the harness's handful of HTTP health polls reach it. Real coverage needs an HTTP JSON-RPC phase in rpc_load_generator.py; that is a workload change rather than a harness correction, and is deliberately out of scope here.",
      "rpc_time_milliseconds": "ServerHandler.cpp:192, group('rpc')->makeEvent('time') with the default millisecond unit. Notified from ServerHandler::processRequest:1129, the same HTTP JSON-RPC call site as rpc_size_bytes and behind the same WebSocket-only gate.",
      "jobq_*_milliseconds, jobq_*_q_milliseconds": "This key is a pattern rather than a literal metric name — unlike every other entry in this map it stands for a whole family, one pair per job type. Created per job type in JobTypeData.h:97-98 from info.name() and info.name() + kSuffixQueued ('_q'), so the exported names are jobq_<jobtype>_milliseconds and jobq_<jobtype>_q_milliseconds with the job type lowercased by formatName. Which job types appear depends on which jobs a run happens to schedule, so no individual name is guaranteed. They are also rounded up to a whole millisecond at source (Event.h:47-51 applies ceil to a millisecond value type), which is why 6e2b2da772 moved the ledger-data-sync q-wait panels off jobq_<jobtype>_q_milliseconds_bucket onto job_queued_us_bucket — they are poor assertion targets regardless."
    }
  },
  "grafana_dashboards": {
    "description": "All 15 Grafana dashboards provisioned on disk under docker/telemetry/grafana/dashboards/ (UID == file stem for every one). validate_dashboards() checks that each UID resolves via GET /api/dashboards/uid/<uid> and reports its panel count — it verifies provisioning and loadability, not panel data. log-derived-insights is included on that basis even though its panels are Loki-backed and CI runs with --skip-loki: the dashboard itself must still provision cleanly. Its panel data is not asserted anywhere.",
    "uids": [
      "rpc-performance",
      "transaction-overview",
      "consensus-health",
      "ledger-operations",
      "peer-network",
      "peer-quality",
      "fee-market",
      "job-queue",
      "validator-health",
      "node-health",
      "network-traffic",
      "rpc-pathfinding",
      "overlay-traffic-detail",
      "ledger-data-sync",
      "log-derived-insights"
    ]
  }
}