mirror of
https://github.com/XRPLF/rippled.git
synced 2026-10-10 05:38:10 +00:00
207 lines
17 KiB
JSON
207 lines
17 KiB
JSON
{
|
|
"description": "Expected metric inventory for xrpld telemetry validation. Metric names have no prefix (the xrpld_ prefix was removed). beast::insight metrics are lowercased by formatName. Every name here was verified against its declaration in MetricsRegistry.cpp or include/xrpl/telemetry/GetObjectMetricNames.h and against a panel query under docker/telemetry/grafana/dashboards/. IMPORTANT: validate_telemetry.py has no notion of an optional metric — validate_metrics() iterates every group that has a \"metrics\" key and hard-fails any name with 0 Prometheus series after a 45 s poll. A metric is therefore listed only when the harness workload guarantees it will appear: observable gauges/counters whose callbacks Observe unconditionally (series exist at value 0), or push counters/histograms on a path every run exercises. Workload-gated and defect-gated names are recorded in the \"not_asserted\" group, which intentionally has no \"metrics\" key so the validator skips it. Only series existence is checked, never a value, except for the four bounds checks hardcoded in PARITY_VALUE_SANITY. A group may additionally declare \"required_labels\": each label there becomes one check that at least one of the group's series carries it with a non-empty value, matched as <label>!=\"\" because Prometheus cannot tell an absent label from an empty one. The same guarantee rule applies — list a label only where the workload guarantees it.",
|
|
"spanmetrics": {
|
|
"description": "SpanMetrics-derived RED metrics from the OTel Collector spanmetrics connector.",
|
|
"metrics": [
|
|
"span_calls_total",
|
|
"span_duration_milliseconds_bucket",
|
|
"span_duration_milliseconds_count",
|
|
"span_duration_milliseconds_sum"
|
|
],
|
|
"required_labels": [
|
|
"span_name",
|
|
"status_code",
|
|
"service_name",
|
|
"span_kind"
|
|
],
|
|
"dimension_labels": [
|
|
"command",
|
|
"rpc_status",
|
|
"consensus_mode",
|
|
"local",
|
|
"proposal_trusted",
|
|
"validation_trusted",
|
|
"tx_type",
|
|
"ter_result",
|
|
"stage",
|
|
"txq_status",
|
|
"close_time_correct",
|
|
"consensus_state",
|
|
"suppressed"
|
|
],
|
|
"_dimension_labels_note": "Bare label names as configured in otel-collector-config.yaml spanmetrics dimensions. Informational only (not asserted by the validator)."
|
|
},
|
|
"statsd_gauges": {
|
|
"description": "beast::insight gauges exported via OTLP/HTTP to the collector (server=otel).",
|
|
"metrics": [
|
|
"ledgermaster_validated_ledger_age",
|
|
"ledgermaster_published_ledger_age",
|
|
"state_accounting_full_duration",
|
|
"peer_finder_active_inbound_peers",
|
|
"peer_finder_active_outbound_peers",
|
|
"jobq_job_count"
|
|
]
|
|
},
|
|
"statsd_counters": {
|
|
"description": "beast::insight counters exported via OTLP/HTTP. The OTel Prometheus exporter appends _total to monotonic counters.",
|
|
"metrics": ["rpc_requests_total", "ledger_fetches_total"]
|
|
},
|
|
"overlay_traffic": {
|
|
"description": "Overlay traffic metrics (subset — full list has 45+ categories).",
|
|
"metrics": [
|
|
"total_bytes_in",
|
|
"total_bytes_out",
|
|
"total_messages_in",
|
|
"total_messages_out"
|
|
]
|
|
},
|
|
"nodestore_io": {
|
|
"description": "NodeStore I/O observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label distinguishing sub-metrics.",
|
|
"metrics": ["nodestore_state"]
|
|
},
|
|
"cache_hit_rates": {
|
|
"description": "Cache hit rate observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.",
|
|
"metrics": ["cache_metrics"]
|
|
},
|
|
"transaction_queue": {
|
|
"description": "Transaction queue observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.",
|
|
"metrics": ["txq_metrics"]
|
|
},
|
|
"rpc_method_detail": {
|
|
"description": "Per-RPC-method counters and duration histogram (MetricsRegistry.cpp:351-357). rpc_method_errored_total is deliberately absent — see not_asserted below. rpc_method_us is a Histogram, so the Prometheus exporter emits only the _bucket/_count/_sum triple and there is no bare rpc_method_us series to match — same convention as span_duration_milliseconds in the spanmetrics group above.",
|
|
"metrics": [
|
|
"rpc_method_started_total",
|
|
"rpc_method_finished_total",
|
|
"rpc_method_us_bucket",
|
|
"rpc_method_us_count",
|
|
"rpc_method_us_sum"
|
|
]
|
|
},
|
|
"job_queue": {
|
|
"description": "Job-queue counters and latency histograms (MetricsRegistry.cpp:360-366). Every xrpld job passes through these, so they populate under any workload. Both histograms are recorded in the same function bodies as job_started_total / job_finished_total, under the same guard and with the same labels, so their presence is equally guaranteed. They are named with the _bucket/_count/_sum suffixes the Prometheus exporter emits: regression-metrics.json and the job-queue dashboard both query job_queued_us_bucket / job_running_us_bucket, and no bare series exists. This group also carries the xrpl_node_id required_labels assertion. 070d29b465 added the xrpl.node.id resource attribute so Grafana Cloud trace ingest would stop folding distinct nodes into one — ingest groups ResourceSpans but ignores service.instance.id, so before the attribute existed the boards reported one node's ledger.build five times over and saw 2 of 9 nodes. Nothing asserted it, so a regression would silently restore the folding. It is asserted here rather than on a statsd_* group because these metrics are MetricsRegistry-backed and guaranteed by any workload, whereas the beast::insight meter provider is constructed before the wallet DB exists and so has no node public key to stamp: those metrics legitimately lack the label and 070d29b465 omits it rather than writing it blank.",
|
|
"required_labels": ["xrpl_node_id"],
|
|
"metrics": [
|
|
"job_queued_total",
|
|
"job_started_total",
|
|
"job_finished_total",
|
|
"job_queued_us_bucket",
|
|
"job_queued_us_count",
|
|
"job_queued_us_sum",
|
|
"job_running_us_bucket",
|
|
"job_running_us_count",
|
|
"job_running_us_sum"
|
|
]
|
|
},
|
|
"rpc_in_flight": {
|
|
"description": "In-flight RPC gauge via the XRPL_METRIC_UPDOWN_ADD call-site macro (PerfLogImp.cpp, +1 rpcStart / -1 rpcEnd). UpDownCounter: no _total suffix.",
|
|
"metrics": ["rpc_in_flight_requests"]
|
|
},
|
|
"object_counts": {
|
|
"description": "Counted object instances observable gauge (MetricsRegistry via OTLP).",
|
|
"metrics": ["object_count"]
|
|
},
|
|
"load_factors": {
|
|
"description": "Fee escalation and load factor observable gauge (MetricsRegistry via OTLP).",
|
|
"metrics": ["load_factor_metrics"]
|
|
},
|
|
"parity_validation_agreement": {
|
|
"description": "External dashboard parity: validation agreement percentages (MetricsRegistry).",
|
|
"metrics": [
|
|
"validation_agreement{metric=\"agreement_pct_1h\"}",
|
|
"validation_agreement{metric=\"agreement_pct_24h\"}"
|
|
]
|
|
},
|
|
"parity_validator_health": {
|
|
"description": "External dashboard parity: validator health indicators (MetricsRegistry).",
|
|
"metrics": [
|
|
"validator_health{metric=\"amendment_blocked\"}",
|
|
"validator_health{metric=\"unl_expiry_days\"}"
|
|
]
|
|
},
|
|
"parity_peer_quality": {
|
|
"description": "External dashboard parity: peer quality metrics (MetricsRegistry).",
|
|
"metrics": [
|
|
"peer_quality{metric=\"peer_latency_p90_ms\"}",
|
|
"peer_quality{metric=\"peers_insane_count\"}"
|
|
]
|
|
},
|
|
"parity_ledger_economy": {
|
|
"description": "External dashboard parity: ledger economy metrics (MetricsRegistry.cpp:1401). transaction_rate is observed on every export, in both branches of the ledger-age test (MetricsRegistry.cpp:1444-1451). base_fee_xrp is observed only inside the 'if (ledger)' guard on getValidatedLedger() (MetricsRegistry.cpp:1418-1423), and that returns validLedger_ (LedgerMaster.cpp:1569-1572), which stays null until a ledger validates — the same precondition complete_ledgers has. Both are asserted because run-full-validation.sh waits for a validated ledger before running the workload. base_fee_xrp absent while transaction_rate is present is the signature of a cluster that never validated, not of a missing metric.",
|
|
"metrics": [
|
|
"ledger_economy{metric=\"base_fee_xrp\"}",
|
|
"ledger_economy{metric=\"transaction_rate\"}"
|
|
]
|
|
},
|
|
"parity_state_tracking": {
|
|
"description": "External dashboard parity: server state tracking (MetricsRegistry).",
|
|
"metrics": ["state_tracking{metric=\"state_value\"}"]
|
|
},
|
|
"parity_counters": {
|
|
"description": "External dashboard parity: monotonic counters (MetricsRegistry). validations_checked_total is incremented unconditionally at the top of NetworkOPsImp::recvValidation (NetworkOPs.cpp:2681), and run-full-validation.sh brings up a 5-node validator cluster, so inbound validations are guaranteed.",
|
|
"metrics": [
|
|
"ledgers_closed_total",
|
|
"validations_sent_total",
|
|
"validations_checked_total",
|
|
"state_changes_total"
|
|
]
|
|
},
|
|
"parity_storage": {
|
|
"description": "External dashboard parity: storage detail metrics (MetricsRegistry).",
|
|
"metrics": ["storage_detail{metric=\"stored_object_bytes\"}"]
|
|
},
|
|
"node_health_gauges": {
|
|
"description": "Node-health observable gauges (MetricsRegistry.cpp:997, :1081, :1102, :1161). server_info, build_info and db_metrics Observe unconditionally on every periodic export (build_info observes a literal 1; server_info and db_metrics read live services), so their series exist regardless of workload shape. complete_ledgers is the exception and is asserted on a narrower guarantee: its callback returns without observing when the range is empty (MetricsRegistry.cpp:1113-1114) and skips any segment that carries no '-' (:1122-1127), and a one-sequence range renders with no '-' (RangeSet.h:70-71), so it needs a complete range spanning at least two sequences. completeLedgers_ is filled by setFullLedger (LedgerMaster.cpp:862-863), which on a peered node is reached only from the publish path in doAdvance (LedgerMaster.cpp:1972) — closing a ledger is not enough, it has to validate. run-full-validation.sh waits for that before the workload starts, so on a healthy cluster the series always exists — a 5-node run yields 10 series, one start and one end per node. If this check ever fails, read the Step 3 output first: a run that logged 'No validated ledger' cannot produce this series and the cluster, not the exporter, is what broke.",
|
|
"metrics": ["server_info", "build_info", "complete_ledgers", "db_metrics"]
|
|
},
|
|
"overlay_reduce_relay": {
|
|
"description": "Transaction reduce-relay efficiency gauge (MetricsRegistry.cpp:1354, peer-network dashboard). Backed by Overlay::txMetrics(); TxMetrics::json() emits txr_selected_cnt / txr_suppressed_cnt / txr_not_enabled_cnt unconditionally (TxMetrics.cpp:121-127), so the gauge always reports at least the selected_peers series.",
|
|
"metrics": ["reduce_relay_metrics"]
|
|
},
|
|
"overlay_overflow": {
|
|
"description": "Job-queue transaction overflow total (MetricsRegistry.cpp:609, job-queue dashboard). An ObservableCounter that reads Overlay::getJqTransOverflow() and Observes unconditionally, so the series exists at value 0 even when no overflow occurs.",
|
|
"metrics": ["jq_trans_overflow_total"]
|
|
},
|
|
"validation_lifetime_counters": {
|
|
"description": "Lifetime validation agreement/miss ObservableCounters (MetricsRegistry.cpp:1636, :1658, validator-health dashboard). Both callbacks reconcile the tracker and Observe unconditionally, so the series exist even on a node that has not yet agreed or missed (value 0). Only existence is asserted, never the value — validation_missed_total legitimately dominates on a non-validating node.",
|
|
"metrics": ["validation_agreements_total", "validation_missed_total"]
|
|
},
|
|
"not_asserted": {
|
|
"description": "Emitted-and-dashboarded metrics deliberately left unasserted because they are workload-gated or defect-gated: the harness workload cannot guarantee they appear, and a check that fails on a healthy run is worse than no check. This group has no \"metrics\" key, so validate_telemetry.py skips it (validate_metrics iterates category_data.get(\"metrics\", [])). Promote an entry into an asserted group only after the workload is changed to guarantee it.",
|
|
"metrics_excluded": {
|
|
"rpc_method_errored_total": "MetricsRegistry.cpp:354, push counter — needs an RPC that returns an error. rpc_load_generator.py issues only well-formed server_info / fee / ledger / ripple_path_find calls, so no series may ever be created.",
|
|
"ledger_history_mismatch_total": "MetricsRegistry.cpp:377, incremented only from LedgerHistory.cpp:332 on a built-vs-validated ledger mismatch. On a healthy run it never fires — asserting it would mean asserting a defect.",
|
|
"txq_expired_total": "MetricsRegistry.cpp:379, incremented only at TxQ.cpp:1428 when a queued tx expires past its LastLedgerSequence. CI does run a txq-burst phase (workload-profiles.json:41, 30 s of single-type Payment at 60 TPS), but that does not guarantee sustained fee escalation followed by expiry: a run in which every other check passed still exposed only txq_metrics and no txq_expired_total.",
|
|
"txq_dropped_total": "MetricsRegistry.cpp:381, incremented only at TxQ.cpp:1302 / :1347 on queue-full admission refusal. Same reason as txq_expired_total.",
|
|
"getobject_rejected_total": "GetObjectMetricNames.h:81, emitted from PeerImp.cpp:2725/:2743 only for a TMGetObjectByHash message refused as oversize or malformed_ledgerhash. A cooperating cluster never sends one.",
|
|
"getobject_request_objects": "GetObjectMetricNames.h:86, emitted from PeerImp.cpp:2926 only while serving an inbound TMGetObjectByHash. The XRPL_METRIC_* macros create their instrument lazily on first use (MetricMacros.h:174-285), so no series exists until a peer actually requests objects by hash — which a 5-node cluster started at genesis and already in sync may never do.",
|
|
"getobject_lookup_us": "GetObjectMetricNames.h:95, PeerImp.cpp:2929. Same lazy-creation and same inbound-request gate as getobject_request_objects.",
|
|
"getobject_lookups_total": "GetObjectMetricNames.h:100, PeerImp.cpp:2949/:2956. Same gate.",
|
|
"getobject_charge": "GetObjectMetricNames.h:105, PeerImp.cpp:2931. Same gate.",
|
|
"rpc_size_bytes": "ServerHandler.cpp:191, group('rpc')->makeEvent('size', Unit::Bytes). The OTLP Prometheus exporter derives the metric-name suffix from the declared unit, so a byte unit yields rpc_size_bytes; before 24094e427b the event declared no unit and exported as rpc_size_milliseconds on the millisecond bucket ladder. Neither name was ever recorded here, so the harness could confirm neither the rename nor a regression back onto that ladder. Notified from ServerHandler::processRequest:1133, the HTTP JSON-RPC path — it computes an HTTP status and appends a trailing newline — and the load generators are WebSocket-only, the same gate regression-metrics.json:4 records for rpc.process, so only the harness's handful of HTTP health polls reach it. Real coverage needs an HTTP JSON-RPC phase in rpc_load_generator.py; that is a workload change rather than a harness correction, and is deliberately out of scope here.",
|
|
"rpc_time_milliseconds": "ServerHandler.cpp:192, group('rpc')->makeEvent('time') with the default millisecond unit. Notified from ServerHandler::processRequest:1129, the same HTTP JSON-RPC call site as rpc_size_bytes and behind the same WebSocket-only gate.",
|
|
"ios_latency_milliseconds": "Application.cpp:515, makeEvent('ios_latency') with the default millisecond unit. Emitted and dashboarded, but it measures io-service scheduling delay rather than anything the workload drives, so no workload phase guarantees it.",
|
|
"jobq_*_milliseconds, jobq_*_q_milliseconds": "Created per job type in JobTypeData.h:97-98 from info.name() and info.name() + kSuffixQueued ('_q'), so the exported names are jobq_<jobtype>_milliseconds and jobq_<jobtype>_q_milliseconds with the job type lowercased by formatName. Which job types appear depends on which jobs a run happens to schedule, so no individual name is guaranteed. They are also rounded up to a whole millisecond at source (Event.h:47-51 applies ceil to a millisecond value type), which is why 6e2b2da772 moved the ledger-data-sync q-wait panels off jobq_<jobtype>_q_milliseconds_bucket onto job_queued_us_bucket — they are poor assertion targets regardless."
|
|
}
|
|
},
|
|
"grafana_dashboards": {
|
|
"description": "All 15 Grafana dashboards provisioned on disk under docker/telemetry/grafana/dashboards/ (UID == file stem for every one). validate_dashboards() checks that each UID resolves via GET /api/dashboards/uid/<uid> and reports its panel count — it verifies provisioning and loadability, not panel data. log-derived-insights is included on that basis even though its panels are Loki-backed and CI runs with --skip-loki: the dashboard itself must still provision cleanly. Its panel data is not asserted anywhere.",
|
|
"uids": [
|
|
"rpc-performance",
|
|
"transaction-overview",
|
|
"consensus-health",
|
|
"ledger-operations",
|
|
"peer-network",
|
|
"peer-quality",
|
|
"fee-market",
|
|
"job-queue",
|
|
"validator-health",
|
|
"node-health",
|
|
"network-traffic",
|
|
"rpc-pathfinding",
|
|
"overlay-traffic-detail",
|
|
"ledger-data-sync",
|
|
"log-derived-insights"
|
|
]
|
|
}
|
|
}
|