#!/usr/bin/env python3 """Telemetry Validation Suite for xrpld. Validates that the full telemetry stack is emitting expected data after a workload run. Queries Tempo (spans), Prometheus (metrics), Loki (logs), and Grafana (dashboards) APIs to produce a pass/fail report. Validation categories: 1. Span validation — Every required span type in expected_spans.json, each carrying its required attributes 2. Metric validation — SpanMetrics, StatsD, and MetricsRegistry OTLP metrics are non-zero, and each group's required_labels reach Prometheus with non-empty values 3. Sync diagnostics — Fresh-node sync signals (bootstrap + acquire pipeline) declared in the "sync_diagnostics" group 4. Log-trace correlation — Loki logs contain trace_id/span_id fields 5. Dashboard validation — Every dashboard uid in expected_metrics.json provisions and loads (panel count only, not panel data) 6. External parity — Span attrs, metric existence, and value sanity for external dashboard parity (validator-health, peer-quality, node-health) Usage: python3 validate_telemetry.py --report /tmp/validation-report.json # Custom API endpoints: python3 validate_telemetry.py \\ --tempo http://localhost:3200 \\ --prometheus http://localhost:9090 \\ --loki http://localhost:3100 \\ --grafana http://localhost:3000 """ import argparse import asyncio import collections import fnmatch import json import logging import re import sys import time from dataclasses import dataclass, field from pathlib import Path from typing import Any import aiohttp # Loki's default query window is the last hour. A validation run finishes in # minutes, but bounding the range explicitly keeps the query reproducible when # someone re-runs it later to investigate a result. LOG_QUERY_WINDOW_SECONDS = 4 * 60 * 60 # Loki's OTLP ingestion promotes service.name to the stream label # `service_name`; a `job` attribute arrives as structured metadata, which a # stream selector cannot match (see otel-collector-config.yaml). Declared once # so both log checks and the diagnostic below report on the same query -- a # diagnostic that queried something else would describe a different failure # than the one being investigated. LOG_STREAM_SELECTOR = '{service_name="xrpld"}' LOG_TRACE_LINE_FILTER = '|= "trace_id="' LOG_CORRELATION_QUERY = f"{LOG_STREAM_SELECTOR} {LOG_TRACE_LINE_FILTER}" logger = logging.getLogger("validate_telemetry") # --------------------------------------------------------------------------- # Configuration defaults # --------------------------------------------------------------------------- DEFAULT_TEMPO = "http://localhost:3200" DEFAULT_PROMETHEUS = "http://localhost:9090" DEFAULT_LOKI = "http://localhost:3100" DEFAULT_GRAFANA = "http://localhost:3000" SCRIPT_DIR = Path(__file__).parent EXPECTED_SPANS_FILE = SCRIPT_DIR / "expected_spans.json" EXPECTED_METRICS_FILE = SCRIPT_DIR / "expected_metrics.json" # Some beast::insight gauges/counters (ledger-age, peer-finder, overlay # traffic) only populate after the node validates ledgers and sustains peer # traffic, then travel a 1s periodic OTLP export + a 15s Prometheus scrape # before they are queryable. On a slow CI runner the fixed post-workload wait # can end before that pipeline settles, so a single query races and reports 0 # series. Poll each missing metric for up to this long (covering two scrape # cycles) before failing, so the check is robust to runner speed. METRIC_POLL_TIMEOUT_SEC = 45.0 METRIC_POLL_INTERVAL_SEC = 5.0 # Group key in expected_metrics.json holding the fresh-node sync-diagnostics # metrics (bootstrap + acquire pipeline). Owned by # assert_sync_diagnostics_metrics() so those signals get their own report # category and a single, explicit failure per missing metric. SYNC_DIAGNOSTICS_GROUP = "sync_diagnostics" # All metrics are polled concurrently against ONE shared deadline, so the # metric phase costs a single poll window instead of one per metric. This caps # how many /api/v1/series requests are in flight at a time, so the fan-out does # not hammer the single-container Prometheus the harness runs. METRIC_POLL_CONCURRENCY = 8 # Bound on ONE HTTP request to Tempo, Prometheus, Loki or Grafana. aiohttp's # own default is total=300s, which is longer than any poll window here: a # single wedged endpoint would blow the shared deadline above and then keep the # run alive until the CI job's own budget killed it, losing the report and the # artifacts with it. Failing one request fast and reporting it beats being # killed with nothing. # # Derived from the poll window rather than picked: no single request may outlast # the phase budget it sits inside, since a request that does can only ever blow # that deadline. Connecting is held to one poll interval, because an endpoint # that is absent or wedged should be named immediately, not waited on. REQUEST_TIMEOUT = aiohttp.ClientTimeout( total=METRIC_POLL_TIMEOUT_SEC, sock_connect=METRIC_POLL_INTERVAL_SEC ) # The Prometheus exporter splits one histogram instrument into three series # names. Reverse coverage folds them back onto the base family so a contract # entry (or an accounted_patterns regex) written for the family accounts for # all three, and so a triple is never reported as three separate gaps. HISTOGRAM_SUFFIXES = ("_bucket", "_count", "_sum") # --------------------------------------------------------------------------- # Data classes # --------------------------------------------------------------------------- @dataclass class CheckResult: """Result of a single validation check. Attributes: name: Check identifier (e.g., "span.rpc.ws_message"). category: Validation category (span, metric, log, dashboard). passed: Whether the check passed. message: Human-readable description of the result. details: Optional additional data (counts, values, etc.). """ name: str category: str passed: bool message: str details: dict[str, Any] = field(default_factory=dict) def to_dict(self) -> dict[str, Any]: """Serialize to a JSON-compatible dict.""" return { "name": self.name, "category": self.category, "passed": self.passed, "message": self.message, "details": self.details, } @dataclass class ValidationReport: """Aggregated validation report. Attributes: checks: List of all individual check results. start_time: ISO timestamp when validation started. end_time: ISO timestamp when validation completed. """ checks: list[CheckResult] = field(default_factory=list) start_time: str = "" end_time: str = "" @property def total_checks(self) -> int: """Total number of checks executed.""" return len(self.checks) @property def passed(self) -> int: """Number of checks that passed.""" return sum(1 for c in self.checks if c.passed) @property def failed(self) -> int: """Number of checks that failed.""" return sum(1 for c in self.checks if not c.passed) @property def all_passed(self) -> bool: """Whether all checks passed.""" return self.failed == 0 def add(self, check: CheckResult) -> None: """Add a check result to the report.""" self.checks.append(check) status = "PASS" if check.passed else "FAIL" logger.info("[%s] %s: %s", status, check.name, check.message) def to_dict(self) -> dict[str, Any]: """Serialize to a JSON-compatible dict.""" return { "summary": { "total": self.total_checks, "passed": self.passed, "failed": self.failed, "all_passed": self.all_passed, }, "start_time": self.start_time, "end_time": self.end_time, "checks": [c.to_dict() for c in self.checks], } # --------------------------------------------------------------------------- # Reverse coverage: what is emitted but not accounted for # --------------------------------------------------------------------------- # # The forward checks read the contract and ask Prometheus/Tempo whether each # listed name exists. That direction cannot see a name the contract omits, and # omissions are how a 345-family metric gap and 7 unknown spans went unnoticed: # both emitted inventories were already being fetched, and neither was compared # back against the contract. # # These helpers close the loop. They are deliberately WARN-ONLY. Downstream # branches legitimately add telemetry that an upstream contract has not seen # (the sync-diagnostics branch emits 7 spans this contract does not list), so a # hard failure here would redden every one of them for doing the right thing. # The value is visibility: name the gaps, in a form a human can read in a CI # log and diff between runs, and let a person decide. def _log_name_list(header: str, names: list[str]) -> None: """Log a name list one entry per line, sorted. A single-line Python list repr of 422 metric families is ~15 kB on one CI log line: unreadable, and impossible to diff between runs. One name per line makes two runs' logs comparable. Emitted as a single log record so the lines cannot be interleaved by another task's output. Args: header: Line printed above the list; the count is appended to it. names: Names to print. Sorted here, so callers need not be. """ body = "\n".join(f" {name}" for name in sorted(names)) or " (none)" logger.info("%s (%d total):\n%s", header, len(names), body) def _reverse_coverage_result( check_name: str, category: str, noun: str, emitted_count: int, unaccounted: list[str], ) -> CheckResult: """Build the always-passing CheckResult for one reverse coverage check. ``passed`` is hardcoded True. This is the single place that decides the check cannot fail CI, so the guarantee is auditable in one line rather than spread over two call sites. The finding travels in ``message`` and in ``details["unaccounted"]``, and the names are also logged one per line by the caller. Args: check_name: Report check name (e.g. "metric.reverse_coverage"). category: Report category ("metric" or "span"). noun: Plural noun for the message ("metric families", "span names"). emitted_count: How many names the backend reported. unaccounted: Names the contract does not account for. Returns: A CheckResult that always passes. """ if emitted_count == 0: message = ( f"reverse coverage not evaluated: no {noun} were reported " f"(backend unreachable or empty) — warning only, never fails" ) elif unaccounted: message = ( f"WARNING: {len(unaccounted)} of {emitted_count} emitted {noun} are " f"not accounted for by the contract: {', '.join(sorted(unaccounted)[:5])}" f"{' …' if len(unaccounted) > 5 else ''} " f"(full list logged above; warning only, never fails)" ) else: message = f"all {emitted_count} emitted {noun} are accounted for" return CheckResult( name=check_name, category=category, passed=True, message=message, details={ "emitted": emitted_count, "accounted": emitted_count - len(unaccounted), "unaccounted_count": len(unaccounted), "unaccounted": sorted(unaccounted), "enforced": False, }, ) # --------------------------------------------------------------------------- # Tempo API helpers # --------------------------------------------------------------------------- def _log_query_window() -> dict[str, str]: """Loki query_range bounds covering a validation run. Returns: start/end parameters in nanoseconds since the epoch. """ now = time.time() return { "start": str(int((now - LOG_QUERY_WINDOW_SECONDS) * 1_000_000_000)), "end": str(int(now * 1_000_000_000)), } class TempoQueryError(RuntimeError): """Tempo answered a query with a failure, as distinct from "nothing found". The two must not be conflated. An empty search result and a 404 on a trace id are legitimate answers meaning the data is not there; an HTTP 5xx, a 401 or a 400 mean the question was never answered, and reporting that as "0 traces" or "0 spans" turns a broken backend into a green-looking negative result. Callers that loop over candidates catch absence and move on; this exception is what they must NOT swallow. """ def _short_body(text: str, limit: int = 300) -> str: """Collapse a response body to one bounded line, for a log or a message.""" return " ".join(text.split())[:limit] or "(empty body)" async def _tempo_search( session: aiohttp.ClientSession, tempo_url: str, query: str, limit: int = 20, ) -> list[dict[str, Any]]: """Search traces in Tempo using TraceQL. Args: session: aiohttp client session. tempo_url: Base URL for Tempo API (e.g., http://localhost:3200). query: TraceQL query string. limit: Maximum number of traces to return. Returns: List of trace summary dicts from Tempo search results. """ params = {"q": query, "limit": str(limit)} async with session.get(f"{tempo_url}/api/search", params=params) as resp: # /api/search answers 200 with an empty (or absent) "traces" list when # nothing matches, so there is no "legitimately absent" status to # tolerate here: every non-200 is a real failure. Left unchecked, # resp.json() on an error body yields a dict with no "traces" key and # this returns [], which every caller reports as "no traces found" -- # indistinguishable from a healthy Tempo holding nothing. That is the # same silent-failure shape as a span query that returned 200 with an # empty list, and it must not be reproduced here. if resp.status != 200: raise TempoQueryError( f"GET /api/search returned HTTP {resp.status}: " f"{_short_body(await resp.text())}" ) data = await resp.json() return data.get("traces", []) async def _tempo_get_trace( session: aiohttp.ClientSession, tempo_url: str, trace_id: str, ) -> list[dict[str, Any]]: """Fetch a full trace from Tempo by trace ID. Returns the list of spans extracted from the OTLP-format response. Args: session: aiohttp client session. tempo_url: Tempo API base URL. trace_id: Hex trace ID string. A 404 means Tempo holds no such trace, which is returned as an empty list rather than raised: see the note in the body. Any other non-200 raises TempoQueryError. Returns: Flat list of span dicts as Tempo returned them, carrying at least 'name', 'attributes', 'spanId' and, for non-root spans, 'parentSpanId'. Each span also gains '_instance', the service.instance.id of the batch resource it arrived under, or '' when that resource carries none. The leading underscore marks it as added here rather than sent by Tempo. Empty when Tempo has no trace with this id. Raises: TempoQueryError: Tempo answered with a status other than 200 or 404. """ async with session.get(f"{tempo_url}/api/traces/{trace_id}") as resp: # 404 is Tempo's answer for "no trace with that id", and for an id read # out of a log line that is an ordinary outcome, not an error: the span # may not have been exported yet, or its block may not be searchable. # Every caller either loops over candidate ids or over several traces # and relies on an empty list meaning "not this one", so a 404 must stay # absent -- raising here would turn a normal miss into a check failure # and abort the loop over the remaining candidates. if resp.status == 404: return [] # Anything else is a real failure. Unchecked, resp.json() on an error # body yields a dict with no "batches" key, so this returned [] and the # caller reported "0 spans" -- a backend fault laundered into a # negative result. if resp.status != 200: raise TempoQueryError( f"GET /api/traces/{trace_id} returned HTTP {resp.status}: " f"{_short_body(await resp.text())}" ) data = await resp.json() spans: list[dict[str, Any]] = [] for batch in data.get("batches", []): # The resource is per batch and flattening drops it, but which node # a span came from is the difference between a mis-parented span and # an ordinary cross-node parent. Stamp it onto each span. instance = "" for attr in batch.get("resource", {}).get("attributes", []): if attr.get("key") == "service.instance.id": instance = str(attr.get("value", {}).get("stringValue", "")) break for scope_spans in batch.get("scopeSpans", []): for span in scope_spans.get("spans", []): span["_instance"] = instance spans.append(span) return spans def _otlp_span_attr_keys(span: dict[str, Any]) -> set[str]: """Extract all attribute key names from an OTLP span. Args: span: OTLP span dict with an 'attributes' list. Returns: Set of attribute key strings. """ return {a["key"] for a in span.get("attributes", []) if "key" in a} def _traceql_name_predicate(expected_name: str) -> str: """Build the TraceQL `name` predicate that selects a contract span name. A literal name becomes an equality test. A glob becomes a regex, since TraceQL has no glob operator: `rpc.command.*` is sent as `name=~"rpc[.]command[.].*"`. Dots use the `[.]` character class, not `\\.`: TraceQL's string lexer rejects an escape it does not recognise, so `\\.` returns HTTP 400. Bare dots would parse but match any character there. Args: expected_name: Span name or glob from expected_spans.json. Returns: A TraceQL predicate on the bare `name` intrinsic, without braces. """ if "*" not in expected_name: return f'name="{expected_name}"' # Span names are lower_snake_case segments joined by dots, so `.` and `*` are # the only characters here that mean anything to a regex engine. Anything # else appearing would need its own handling rather than silent passthrough. unexpected = set(expected_name) - set("abcdefghijklmnopqrstuvwxyz0123456789_.*") if unexpected: raise ValueError( f"span name {expected_name!r} contains {sorted(unexpected)}, which " "this predicate builder does not know how to escape for TraceQL" ) pattern = "".join( ".*" if ch == "*" else "[.]" if ch == "." else ch for ch in expected_name ) return f'name=~"{pattern}"' def _span_name_matches(emitted_name: str, expected_name: str) -> bool: """Test an emitted span name against a name from expected_spans.json. Contract names are either literals or globs containing "*" (for example "rpc.command.*"). Literals are compared for exact equality so a longer emitted name cannot satisfy a shorter contract: "consensus.accept.apply" must not stand in for "consensus.accept". Args: emitted_name: Span name as reported by Tempo. expected_name: Span name or glob pattern from expected_spans.json. Returns: True when the emitted name satisfies the expected name. """ if "*" in expected_name: return fnmatch.fnmatchcase(emitted_name, expected_name) return emitted_name == expected_name def _unaccounted_span_names(emitted: list[str], expected: dict[str, Any]) -> list[str]: """Names Tempo reports that no expected_spans.json entry accounts for. "Accounted for" is the same relation the forward check uses, so a contract glob such as "rpc.command.*" covers every command it expands to, and an entry marked ``optional`` still accounts for its name — being unasserted is not the same as being unknown. No separate pattern list is needed for spans because the contract already carries globs. Args: emitted: Span names as reported by Tempo's span.name tag values. expected: The parsed expected_spans.json contract. Returns: Sorted list of emitted names with no matching contract entry. """ contract = [span_def["name"] for span_def in expected.get("spans", [])] return sorted( name for name in emitted if name and not any( _span_name_matches(name, expected_name) for expected_name in contract ) ) # --------------------------------------------------------------------------- # Span Validation (Tempo API) # --------------------------------------------------------------------------- def _load_expected_spans() -> dict[str, Any]: """Parse expected_spans.json. Every span check reads the contract through this one function so that two checks cannot end up disagreeing about it -- an inline open() in each would let one of them be pointed at a different file or key during a refactor while the other kept passing. Returns: The parsed contract: a dict with 'spans' and 'parent_child_relationships' keys. """ with open(EXPECTED_SPANS_FILE) as f: contract: dict[str, Any] = json.load(f) return contract async def validate_spans( session: aiohttp.ClientSession, tempo_url: str, report: ValidationReport, ) -> None: """Validate that all expected spans appear in Tempo. Queries the Tempo TraceQL API for each expected span name and checks that traces exist. Also validates required attributes on spans and parent-child relationships. Args: session: aiohttp client session. tempo_url: Base URL for Tempo API (e.g., http://localhost:3200). report: ValidationReport to accumulate results. """ logger.info("--- Span Validation (Tempo) ---") # Load expected spans. expected = _load_expected_spans() # Check service registration. try: async with session.get( f"{tempo_url}/api/v2/search/tag/resource.service.name/values" ) as resp: data = await resp.json() tag_values = data.get("tagValues", []) services = [tv.get("value", "") for tv in tag_values] has_xrpld = "xrpld" in services report.add( CheckResult( name="span.service_registration", category="span", passed=has_xrpld, message=( f"Service 'xrpld' registered (found: {services})" if has_xrpld else f"Service 'xrpld' NOT found (found: {services})" ), ) ) except Exception as exc: report.add( CheckResult( name="span.service_registration", category="span", passed=False, message=f"Tempo API unreachable: {exc}", ) ) return # List every span name Tempo holds. This is both a diagnostic (it makes a # missing-span failure debuggable without reproducing the stack locally) # and the input to the reverse coverage check added at the end of this # function. Note the tag-values API is not service-scoped, so in a stack # where something other than xrpld also sent traces this list would be a # superset; on the harness only xrpld exports spans. # # The tag is the bare intrinsic `name`, NOT `span.name`. A span's name is a # TraceQL intrinsic, not a span-scoped attribute, so `span.name` resolves to # an attribute nothing sets and Tempo answers 200 with an empty tagValues # list -- indistinguishable from an empty backend, which is how the wrong # tag went unnoticed. Verified against tempo 2.9.4 holding one span: # `span.name` -> {"tagValues":[]}, `name` -> that span's name. emitted_span_names: list[str] = [] try: async with session.get(f"{tempo_url}/api/v2/search/tag/name/values") as resp: ops_data = await resp.json() tag_values = ops_data.get("tagValues", []) emitted_span_names = [tv.get("value", "") for tv in tag_values] _log_name_list("Tempo span names", emitted_span_names) except Exception as exc: logger.warning("Failed to fetch Tempo operations: %s", exc) # Concrete probe names for wildcard span entries. Exact-match TraceQL can't # match a literal "*", so a representative operation name is substituted. # Wildcards without a known concrete example (e.g. grpc. when no # gRPC client runs) are skipped when marked optional. wildcard_probes = {"rpc.command.*": "rpc.command.server_info"} # Check each expected span. for span_def in expected["spans"]: span_name = span_def["name"] is_optional = span_def.get("optional", False) check_name = f"span.{span_name}" if "*" in span_name: operation = wildcard_probes.get(span_name) if operation is None: # No concrete probe. Optional wildcards (e.g. grpc.*) are skipped; # a required one would be a config error worth surfacing. if is_optional: logger.info( "[SKIP] %s: optional wildcard span with no concrete " "probe (not exercised by the workload)", check_name, ) continue report.add( CheckResult( name=check_name, category="span", passed=False, message=f"{span_name}: required wildcard has no probe name", ) ) continue else: operation = span_name try: query = '{resource.service.name="xrpld" && name="' + operation + '"}' traces = await _tempo_search(session, tempo_url, query, limit=5) count = len(traces) # Optional spans only fire under specific traffic (mode changes, # missing-ledger fetch, fee escalation). Absence is not a failure — # mirror the parent-child "skip" handling so CI stays green. if count == 0 and is_optional: logger.info( "[SKIP] %s: optional span not emitted under this workload", check_name, ) report.add( CheckResult( name=check_name, category="span", passed=True, message=f"{span_name}: optional, not emitted (skipped)", details={"trace_count": 0, "optional": True}, ) ) continue report.add( CheckResult( name=check_name, category="span", passed=count > 0, message=( f"{span_name}: {count} traces found" if count > 0 else f"{span_name}: 0 traces (expected > 0)" ), details={"trace_count": count}, ) ) # Validate required attributes on first trace. if count > 0 and span_def.get("required_attributes"): await _check_attributes_on_first_trace( session, tempo_url, traces, span_def, report ) except Exception as exc: report.add( CheckResult( name=check_name, category="span", passed=False, message=f"{span_name}: query failed ({exc})", ) ) # Validate parent-child relationships. for rel in expected.get("parent_child_relationships", []): # Skip relationships marked with "skip: true" (e.g., cross-thread # parent-child that requires a C++ fix to propagate span context). if rel.get("skip", False): reason = rel.get("skip_reason", "marked skip in expected_spans.json") logger.info( "[SKIP] span.hierarchy.%s->%s: %s", rel["parent"], rel["child"], reason, ) continue await _validate_parent_child(session, tempo_url, rel, report) # Reverse direction: span names Tempo holds that the contract never mentions. # Added last so no existing check's position in the report moves. unaccounted_spans = _unaccounted_span_names(emitted_span_names, expected) if unaccounted_spans: _log_name_list( "Span names emitted but NOT accounted for by expected_spans.json " "(warning only, does not fail CI)", unaccounted_spans, ) report.add( _reverse_coverage_result( check_name="span.reverse_coverage", category="span", noun="span names", emitted_count=len(emitted_span_names), unaccounted=unaccounted_spans, ) ) async def _check_attributes_on_first_trace( session: aiohttp.ClientSession, tempo_url: str, traces: list[dict[str, Any]], span_def: dict[str, Any], report: ValidationReport, ) -> None: """Fetch the first trace and check the span's required attributes. Fetching the trace is a second network call, so it carries its own error handling. Letting it fall through to the caller's handler would add a second result under the span's own check name, which has already recorded the trace as found -- one entry passing and one failing for the same name, inflating the check total and blaming the trace-existence check for an attribute-fetch failure. Args: session: aiohttp client session. tempo_url: Base URL for the Tempo API. traces: Traces returned for this span, most recent first. span_def: The span's entry from expected_spans.json. report: ValidationReport to accumulate results. """ span_name = span_def["name"] try: trace_id = traces[0].get("traceID", "") if not trace_id: # Recorded rather than returned on silently. Returning with no # result would drop this span's attribute contract out of the # report and shrink the check total, so the surface would look # smaller with nothing saying why. Reported the same way as a # fetched trace holding no matching span, below: being unable to # verify is itself the finding. The caller only reaches here for a # span that declares required_attributes, so this adds no check # where none was expected. report.add( CheckResult( name=f"span.attrs.{span_name}", category="span", passed=False, message=( f"{span_name}: newest trace carried no traceID, cannot " "verify its attributes" ), ) ) return spans = await _tempo_get_trace(session, tempo_url, trace_id) await _validate_span_attributes_otlp(spans, span_def, report) except Exception as exc: report.add( CheckResult( name=f"span.attrs.{span_name}", category="span", passed=False, message=f"{span_name}: attribute check failed ({exc})", ) ) async def _validate_span_attributes_otlp( spans: list[dict[str, Any]], span_def: dict[str, Any], report: ValidationReport, ) -> None: """Check that the contract's own span carries its required attributes. Only spans whose name matches ``span_def["name"]`` are inspected. Attributes are never borrowed from siblings: many span types share keys such as ledger_seq or tx_hash, so a trace-wide scan would satisfy every one of those contracts from a single carrier span and make the per-span contract unenforceable. A span type passes when at least one instance of it carries every required attribute. When none does, the closest instance's missing keys are reported. Args: spans: Every OTLP span dict in the fetched trace. span_def: Span definition from expected_spans.json. report: ValidationReport to accumulate results. """ required_attrs = span_def.get("required_attributes", []) if not required_attrs: return span_name = span_def["name"] check_name = f"span.attrs.{span_name}" matching = [s for s in spans if _span_name_matches(s.get("name", ""), span_name)] if not matching: report.add( CheckResult( name=check_name, category="span", passed=False, message=( f"{span_name}: no span named '{span_name}' in the fetched " "trace, cannot verify its attributes" ), details={"required": required_attrs, "instances": 0}, ) ) return # Keep the instance that is missing the fewest required attributes, so the # failure message names the closest witness rather than an arbitrary one. best_found: set[str] = set() best_missing: list[str] = list(required_attrs) for span in matching: found = _otlp_span_attr_keys(span) missing = [a for a in required_attrs if a not in found] if len(missing) < len(best_missing): best_found, best_missing = found, missing if not best_missing: break report.add( CheckResult( name=check_name, category="span", passed=not best_missing, message=( f"{span_name}: all {len(required_attrs)} attributes present" if not best_missing else f"{span_name}: no '{span_name}' span carried all " f"{len(required_attrs)} required attributes; closest of " f"{len(matching)} instance(s) missing {best_missing}" ), details={ "required": required_attrs, "found": sorted(best_found), "missing": best_missing, "instances": len(matching), }, ) ) # Verdicts _span_ancestry can return, most conclusive first. A single trace # proving "under" settles the relationship, so it wins outright. A definite # negative outranks an indefinite one: "not_under" means a chain was walked to a # root and the parent was not on it, while "broken_chain" only means ancestry # could not be established, which is a different thing to go and look at. # "no_child" ranks last because it is the starting value: a relationship with no # candidate trace at all reports it, and every other verdict must be able to # replace it. _ANCESTRY_PRIORITY = ("under", "not_under", "broken_chain", "no_parent", "no_child") def _span_ancestry( spans: list[dict[str, Any]], parent_name: str, child_name: str, ) -> tuple[str, str]: """Decide whether a matching child hangs under a matching parent in one trace. Walks each candidate child's parentSpanId chain upwards. Ancestry rather than a direct edge, because every relationship in expected_spans.json is worded as the parent "containing" the child: a scope appearing in between is a refactor, not a broken relationship. Span ids are compared as opaque strings and never decoded. Both fields come from the same Tempo response and so share whatever encoding it uses, which keeps this independent of whether that is hex or base64. Args: spans: Every OTLP span dict in one fetched trace. parent_name: Parent name from the contract; may be a glob. child_name: Child name from the contract; may be a glob. Returns: ``(verdict, detail)``. Verdict is one of ``_ANCESTRY_PRIORITY``. For ``broken_chain`` the detail is the span id the chain ran into. """ by_id = {s["spanId"]: s for s in spans if s.get("spanId")} parent_ids = { s["spanId"] for s in spans if s.get("spanId") and _span_name_matches(s.get("name", ""), parent_name) } children = [s for s in spans if _span_name_matches(s.get("name", ""), child_name)] if not children: return "no_child", "" if not parent_ids: return "no_parent", "" broken = "" walked_to_a_root = False for child in children: current = child.get("parentSpanId", "") # Guards against a cycle in malformed data, which would otherwise spin # here forever rather than failing the check. seen: set[str] = set() while current and current not in seen: if current in parent_ids: return "under", "" seen.add(current) if (next_span := by_id.get(current)) is None: # This child proves nothing either way. Keep the first gap seen, # in case no other child yields a definite answer. broken = broken or current break current = next_span.get("parentSpanId", "") else: # Ran out of chain rather than hitting a gap, so this child really is # not under the parent. One such child is a definite negative and # outranks any other child's gap. walked_to_a_root = True if walked_to_a_root or not broken: return "not_under", "" return "broken_chain", broken def _hierarchy_message( parent_name: str, child_name: str, verdict: str, detail: str ) -> str: """Phrase one ancestry verdict for the report. Each verdict gets its own wording because they send the reader somewhere different: a missing child is a workload or instrumentation gap, a child that is present but not under the parent is a hierarchy bug, and a broken chain is a span that never reached Tempo. """ if verdict == "under": return f"Found {child_name} under {parent_name}" if verdict == "not_under": return ( f"{child_name} shares a trace with {parent_name} but is not under " f"it -- co-occurrence is not a hierarchy" ) if verdict == "broken_chain": return ( f"{child_name}'s parent chain runs into span {detail}, which the " f"trace does not contain, so ancestry under {parent_name} cannot " f"be verified" ) if verdict == "no_parent": return f"{parent_name} absent from traces selected for containing it" return f"{child_name} not found in {parent_name} traces" async def _best_ancestry_verdict( session: aiohttp.ClientSession, tempo_url: str, traces: list[dict[str, Any]], parent_name: str, child_name: str, ) -> tuple[str, str]: """Fetch each candidate trace and keep the most conclusive verdict. Every candidate is examined rather than stopping at the first negative, because a conditional child may hang correctly in one trace while another carries the parent alone. Stops early once one trace proves "under", which nothing later can improve on. Names are re-checked per trace rather than trusted from the query that selected it: it keeps the glob handling in one place, so a wrongly built query cannot pass silently. Args: session: aiohttp client session. tempo_url: Base URL for the Tempo API. traces: Trace summaries that matched parent and child. parent_name: Parent name from the contract; may be a glob. child_name: Child name from the contract; may be a glob. Returns: The winning ``(verdict, detail)`` from _span_ancestry. """ verdict, detail = "no_child", "" for trace_summary in traces: trace_id = trace_summary.get("traceID", "") if not trace_id: continue spans = await _tempo_get_trace(session, tempo_url, trace_id) candidate, candidate_detail = _span_ancestry(spans, parent_name, child_name) if _ANCESTRY_PRIORITY.index(candidate) < _ANCESTRY_PRIORITY.index(verdict): verdict, detail = candidate, candidate_detail if verdict == "under": break return verdict, detail async def _validate_parent_child( session: aiohttp.ClientSession, tempo_url: str, relationship: dict[str, Any], report: ValidationReport, ) -> None: """Validate a parent-child span relationship in Tempo traces. Co-occurrence in a trace is the search filter, not the assertion: the check then walks the child's parentSpanId chain to confirm it really hangs under the parent. See _span_ancestry. Args: session: aiohttp client session. tempo_url: Base URL for Tempo API. relationship: Dict with 'parent' and 'child' span names. report: ValidationReport to accumulate results. """ parent_name = relationship["parent"] child_name = relationship["child"] try: # Query traces for the parent span. Kept as its own query so that "the # parent stopped being emitted" stays distinguishable from "the parent is # there but the child never co-occurs" — they mean different things to # whoever reads the report. query = '{resource.service.name="xrpld" && name="' + parent_name + '"}' traces = await _tempo_search(session, tempo_url, query, limit=3) if not traces: report.add( CheckResult( name=f"span.hierarchy.{parent_name}->{child_name}", category="span", passed=False, message=f"No {parent_name} traces to check hierarchy", ) ) return # Ask for traces holding BOTH rather than the newest parent traces. A # parent that fires on every close has newest traces least likely to # carry a conditional child. `{A} && {B}` matches at trace level, so # co-occurrence is found wherever it happened. both_query = query + " && {" + _traceql_name_predicate(child_name) + "}" traces = await _tempo_search(session, tempo_url, both_query, limit=3) verdict, detail = await _best_ancestry_verdict( session, tempo_url, traces, parent_name, child_name ) report.add( CheckResult( name=f"span.hierarchy.{parent_name}->{child_name}", category="span", passed=verdict == "under", message=_hierarchy_message(parent_name, child_name, verdict, detail), ) ) except Exception as exc: report.add( CheckResult( name=f"span.hierarchy.{parent_name}->{child_name}", category="span", passed=False, message=f"Hierarchy check failed: {exc}", ) ) _ALLOWED_PARENT_ROOT = "ROOT" # A parent span id of eight zero bytes is OTLP's "no parent". Tempo 2.9.4 omits # the field entirely for a root instead -- measured on a 114-span trace, 80 spans # with no parentSpanId, 34 with one, none empty and none all-zero -- but OTLP # permits the all-zero id, so an exporter or backend that writes it must not turn # every root into an unprovable parent. That would make this gate fail open on # exactly the spans it exists to judge, and silently: "nothing was provable" # passes. Both encodings a JSON OTLP payload can carry are listed, base64 (what # Tempo emits for a real id) and lowercase hex. _ROOT_PARENT_SPAN_IDS = frozenset({"AAAAAAAAAAA=", "0000000000000000"}) def _observed_parent_label( span: dict[str, Any], by_id: dict[str, dict[str, Any]], ) -> str | None: """Classify one span's parent as a label, or None when nothing is provable. Returns the parent span's name when the parent is in the trace and came from the same node, _ALLOWED_PARENT_ROOT when the span has no parent at all (either because the field is absent or because it holds the all-zero id), and None when the parent is on another node or is not in the trace. None is deliberately not a verdict. A cross-node parent is the design for the receive spans, and a parent id the trace does not hold means the parent has not been exported yet, which a rotation in flight produces routinely. Args: span: One OTLP span dict as _tempo_get_trace returned it, so carrying the '_instance' key that function stamps on. by_id: Every span in the same trace, keyed by its spanId. Returns: The parent's name, _ALLOWED_PARENT_ROOT, or None. """ parent_id = span.get("parentSpanId", "") if not parent_id or parent_id in _ROOT_PARENT_SPAN_IDS: return _ALLOWED_PARENT_ROOT parent = by_id.get(parent_id) if parent is None: return None if parent.get("_instance", "") != span.get("_instance", ""): return None return str(parent.get("name", "")) async def _validate_span_parents_for( session: aiohttp.ClientSession, tempo_url: str, span_def: dict[str, Any], report: ValidationReport, ) -> None: """Check that every emitted instance of one span has an allowed parent. Driven by the span's own allowed_parents list rather than by the declared relationship rows, so it covers every span in the inventory instead of the pairs somebody remembered to declare -- and it is the only check that can fail a span for being parented when it should be a root. Args: session: aiohttp client session. tempo_url: Base URL for Tempo API. span_def: One entry from expected_spans.json's 'spans' array. report: ValidationReport to accumulate results. """ # Read inside the try, so a contract entry missing its name is reported as # one failed check rather than raised out of validate_span_parents' loop and # taking every remaining span's check with it. name = "" check = "span.parent." try: name = str(span_def["name"]) check = f"span.parent.{name}" allowed = set(span_def.get("allowed_parents", [])) if not allowed: return query = ( '{resource.service.name="xrpld" && ' + _traceql_name_predicate(name) + "}" ) traces = await _tempo_search(session, tempo_url, query, limit=5) if not traces: optional = bool(span_def.get("optional", False)) report.add( CheckResult( name=check, category="span", passed=optional, message=f"{name}: not emitted under this workload, parent not checked", details={"optional": optional}, ) ) return observed: collections.Counter[str] = collections.Counter() unprovable = 0 for summary in traces: trace_id = summary.get("traceID", "") if not trace_id: continue spans = await _tempo_get_trace(session, tempo_url, trace_id) by_id = {s["spanId"]: s for s in spans if s.get("spanId")} for span in spans: if not _span_name_matches(span.get("name", ""), name): continue label = _observed_parent_label(span, by_id) if label is None: unprovable += 1 else: observed[label] += 1 # An allowed_parents entry may itself be a glob -- pathfind.request is # declared under rpc.command.* -- so membership goes through the same # matcher the contract's span names use rather than a set lookup, which # would read every concrete rpc.command. as a violation. violations = { lbl: n for lbl, n in observed.items() if not any(_span_name_matches(lbl, pattern) for pattern in allowed) } total = sum(observed.values()) + unprovable if violations: worst = max(violations.items(), key=lambda kv: kv[1]) message = ( f"{name}: parented to {worst[0]} on {worst[1]} of {total} " f"instance(s); allowed: {sorted(allowed)}" ) elif observed: message = f"{name}: every provable parent is one of {sorted(allowed)}" else: message = ( f"{name}: {unprovable} instance(s), every parent on another node or " "absent from the trace, so nothing was provable" ) report.add( CheckResult( name=check, category="span", passed=not violations, message=message, details={ "observed": dict(observed), "unprovable": unprovable, "allowed": sorted(allowed), }, ) ) except Exception as exc: # noqa: BLE001 - a backend fault is a check failure report.add( CheckResult( name=check, category="span", passed=False, message=f"{name}: parent check failed ({exc})", ) ) async def validate_span_parents( session: aiohttp.ClientSession, tempo_url: str, report: ValidationReport, ) -> None: """Run the parent gate over every span in the inventory. Args: session: aiohttp client session. tempo_url: Base URL for Tempo API. report: ValidationReport to accumulate results. """ logger.info("--- Span Parent Validation (Tempo) ---") for span_def in _load_expected_spans().get("spans", []): await _validate_span_parents_for(session, tempo_url, span_def, report) _ROUND_REQUIRED_CHILDREN = ( "consensus.phase.open", "consensus.ledger_close", "consensus.establish", "consensus.accept", ) async def validate_consensus_round_shape( session: aiohttp.ClientSession, tempo_url: str, report: ValidationReport, ) -> None: """Check a consensus round's child set, their order, and mode_change. The presence and hierarchy checks judge one span at a time, so a round missing a phase, or running its phases out of order, passes them both. The shape of a round is what an operator reads a trace for, so it is asserted directly: every required child under the same round span, on the same node, and their start times in protocol order. Candidate traces are selected by co-occurrence rather than recency -- see the comment on the query. mode_change rides along because it is a child of the same span. Its whole purpose is to record a transition, so mode_old == mode_new is a defect rather than a data point. Args: session: aiohttp client session. tempo_url: Base URL for Tempo API. report: ValidationReport to accumulate results. """ logger.info("--- Consensus Round Shape (Tempo) ---") try: # Select traces holding the round AND its last phase, the way # _validate_parent_child does, rather than the newest rounds. The accept # span always outlives the round span -- the round guard is reset inside # doAccept while the accept span's shared_ptr dies with the JtAccept # lambda -- so with batch_delay_ms=2000 the two can leave in different # export batches and the newest round becomes searchable before its # consensus.accept child arrives. Sampling the newest rounds therefore # reports a missing child on a perfectly shaped round, periodically. query = '{resource.service.name="xrpld" && name="consensus.round"}' traces = await _tempo_search( session, tempo_url, query + ' && {name="consensus.accept"}', limit=5, ) if not traces: report.add( CheckResult( name="span.round.children", category="span", passed=False, message=( "No trace holds both consensus.round and consensus.accept, " "so round shape could not be checked" ), ) ) return missing: collections.Counter[str] = collections.Counter() # Node ids are collected per failing check, not per round: a red here is # only actionable if it says which node produced the bad shape, and on a # five-node cluster "1 of 5 rounds" does not. missing_nodes: set[str] = set() disordered_nodes: set[str] = set() equal_mode_nodes: set[str] = set() rounds = out_of_order = mode_changes = equal_modes = 0 for summary in traces: trace_id = summary.get("traceID", "") if not trace_id: continue spans = await _tempo_get_trace(session, tempo_url, trace_id) for parent in [s for s in spans if s.get("name") == "consensus.round"]: rounds += 1 node = str(parent.get("_instance", "")) or "(unknown node)" # Same node as well as same parent id: one trace carries every # validator's view of the round, so a round span from node A and # a phase span from node B must not be read as one round. kids = [ s for s in spans if s.get("parentSpanId") == parent.get("spanId") and s.get("_instance", "") == parent.get("_instance", "") ] by_name = {s.get("name", ""): s for s in kids} for required in _ROUND_REQUIRED_CHILDREN: if required not in by_name: missing[required] += 1 missing_nodes.add(node) starts = [ int(by_name[n].get("startTimeUnixNano", "0")) for n in _ROUND_REQUIRED_CHILDREN if n in by_name ] if starts != sorted(starts): out_of_order += 1 disordered_nodes.add(node) for mc in [s for s in kids if s.get("name") == "consensus.mode_change"]: mode_changes += 1 attrs = { a["key"]: a.get("value", {}) for a in mc.get("attributes", []) } old = attrs.get("mode_old", {}).get("stringValue") new = attrs.get("mode_new", {}).get("stringValue") if old is not None and old == new: equal_modes += 1 equal_mode_nodes.add(node) report.add( CheckResult( name="span.round.children", category="span", passed=not missing, message=( f"{rounds} round(s): every required child present" if not missing else f"{rounds} round(s) missing children: {dict(missing)} " f"on {sorted(missing_nodes)}" ), details={ "rounds": rounds, "missing": dict(missing), "nodes": sorted(missing_nodes), }, ) ) report.add( CheckResult( name="span.round.phase_order", category="span", passed=out_of_order == 0, message=( f"{rounds} round(s): phases start in protocol order" if out_of_order == 0 else f"{out_of_order} of {rounds} round(s) started their phases " f"out of order on {sorted(disordered_nodes)}" ), details={ "rounds": rounds, "out_of_order": out_of_order, "nodes": sorted(disordered_nodes), }, ) ) report.add( CheckResult( name="span.mode_change.records_a_real_change", category="span", passed=equal_modes == 0, message=( f"{mode_changes} mode_change span(s), none with mode_old == mode_new" if equal_modes == 0 else f"{equal_modes} of {mode_changes} mode_change span(s) recorded " f"no change (mode_old == mode_new) on {sorted(equal_mode_nodes)}" ), details={ "mode_changes": mode_changes, "equal": equal_modes, "nodes": sorted(equal_mode_nodes), }, ) ) except Exception as exc: # noqa: BLE001 - a backend fault is a check failure report.add( CheckResult( name="span.round.children", category="span", passed=False, message=f"Round shape check failed ({exc})", ) ) # --------------------------------------------------------------------------- # Trace-join Validation (Tempo API) # --------------------------------------------------------------------------- async def assert_trace_join_groups( session: aiohttp.ClientSession, tempo_url: str, report: ValidationReport, ) -> None: """Assert each declared trace-join group really lands in ONE trace. A trace-join group is a set of spans that share one trace id with NO parent/child link between them: each derives its trace id deterministically from the same hash (SpanGuard::hashSpan over a ledger hash), which is how spans produced on unrelated threads are joined without propagating any context. The parent/child check above cannot express that -- there is no parent to look for -- so the assertion here is co-occurrence: search for the group's anchor span, then require at least one of its traces to also contain every span in required_members. A regression this catches: if the join key or the deterministic-root mechanism breaks, each span reverts to its own single-span trace and no trace contains the members together, so a slow ledger is no longer readable as one unit. That is invisible to every other check in this harness -- the spans are all still emitted with all their attributes. An absent "trace_join_groups" key is a genuine no-op (nothing declared yet), not a failure: unlike the sync_diagnostics metric group, this key is optional and older expected_spans.json files predate it. Args: session: aiohttp client session. tempo_url: Base URL for the Tempo API. report: ValidationReport to accumulate results. """ logger.info("--- Trace-Join Validation (Tempo) ---") with open(EXPECTED_SPANS_FILE) as f: expected = json.load(f) groups = expected.get("trace_join_groups", {}).get("groups", []) if not groups: logger.info("[SKIP] span.trace_join: no join groups declared") return for group in groups: await _validate_trace_join_group(session, tempo_url, group, report) async def _validate_trace_join_group( session: aiohttp.ClientSession, tempo_url: str, group: dict[str, Any], report: ValidationReport, ) -> None: """Validate one trace-join group: anchor and members share a trace. Args: session: aiohttp client session. tempo_url: Tempo API base URL. group: One entry from expected_spans.json trace_join_groups.groups. report: ValidationReport to accumulate results. """ name = group.get("name", "") anchor = group["anchor"] required = list(group.get("required_members", [])) check_name = f"span.trace_join.{name}" try: query = '{resource.service.name="xrpld" && name="' + anchor + '"}' traces = await _tempo_search(session, tempo_url, query, limit=10) if not traces: report.add( CheckResult( name=check_name, category="span", passed=False, message=f"{name}: no {anchor} traces to check the join against", details={"anchor": anchor, "required_members": required}, ) ) return # Walk the anchor's traces and keep the best result: the join holds as # soon as ONE trace carries every required member. Several traces are # examined because a given ledger may legitimately be missing an # optional member (e.g. a self-built ledger has no arriving validation). best_missing = required for summary in traces: trace_id = summary.get("traceID", "") if not trace_id: continue spans = await _tempo_get_trace(session, tempo_url, trace_id) present = {s.get("name", "") for s in spans} missing = [m for m in required if m not in present] if len(missing) < len(best_missing): best_missing = missing if not missing: break report.add( CheckResult( name=check_name, category="span", passed=not best_missing, message=( f"{name}: {anchor} shares a trace with {required}" if not best_missing else ( f"{name}: no {anchor} trace contained {best_missing} " f"-- the per-{group.get('join_key', 'hash')} trace join " f"is broken (spans are landing in separate traces)" ) ), details={ "anchor": anchor, "join_key": group.get("join_key"), "required_members": required, "missing": best_missing, "traces_examined": len(traces), }, ) ) except Exception as exc: report.add( CheckResult( name=check_name, category="span", passed=False, message=f"{name}: trace-join check failed ({exc})", ) ) # --------------------------------------------------------------------------- # Metric Validation (Prometheus API) # --------------------------------------------------------------------------- # Top-level keys of expected_metrics.json that validate_metrics() must not walk # with its generic group loop: "description" is prose, "grafana_dashboards" # holds dashboard UIDs (checked by validate_dashboards), and # "sync_diagnostics" has its own validator, assert_sync_diagnostics_metrics(). # # MERGE HAZARD, for whoever brings phase-10 forward into this branch. phase-10 # replaces the flatten below with _metric_check_targets(), which selects every # group satisfying isinstance(category_data, dict) and has no equivalent of this # tuple. Resolving that merge in phase-10's favour therefore lets # validate_metrics walk "sync_diagnostics" as well, while # assert_sync_diagnostics_metrics still walks it -- every metric in the group # polled twice and reported twice, inflating the check total. The resolution must # reinstate this exclusion inside _metric_check_targets, or fold # assert_sync_diagnostics_metrics into it and drop the separate pass. Either is # fine; keeping both walkers without an exclusion is not. SKIPPED_METRIC_GROUPS = ("description", "grafana_dashboards", SYNC_DIAGNOSTICS_GROUP) async def _log_prometheus_metric_names( session: aiohttp.ClientSession, prometheus_url: str ) -> list[str]: """Log every metric family name Prometheus knows, and return the list. Two jobs. It is the diagnostic that makes a name mismatch between expected_metrics.json and actual emissions debuggable from a CI log, and it is the emitted inventory the reverse coverage check compares the contract against. Failures are warnings, never check failures; an empty return means "could not determine", which the reverse check reports as not evaluated rather than as a clean run. Deliberately unfiltered. This used to keep only names matching 19 hard-coded prefixes, which made it useless for the job it exists to do: on the last CI run it printed 147 of 422 families, and none of the prefixes covered state_accounting_*, node_family_*, overlay_peer_disconnects or the pathfind_* histograms, so a coverage gap in exactly those families could not be seen here. An allow-list can only ever show names someone already thought to look for, which is the opposite of what a discovery aid needs to do. The whole list is a few kilobytes of CI log, so there is nothing to save by truncating it. Sorted and printed one name per line so two runs' output can be diffed directly; the Prometheus API does not promise an order, and the whole list on one log line was neither readable nor diffable. Args: session: aiohttp client session. prometheus_url: Prometheus base URL. Returns: Sorted metric family names, or an empty list if the fetch failed. """ try: async with session.get( f"{prometheus_url}/api/v1/label/__name__/values" ) as resp: label_data = await resp.json() all_metrics = sorted(label_data.get("data", [])) _log_name_list("Prometheus metric families", all_metrics) return all_metrics except Exception as exc: logger.warning("Failed to fetch Prometheus metric names: %s", exc) return [] def _selector_metric_name(selector: str) -> str: """Strip any label matcher from a contract selector, leaving the name. Contract entries are usually bare names but some carry a matcher, e.g. ``ledger_economy{metric="base_fee_xrp"}``. Reverse coverage compares family names, and Prometheus reports one ``__name__`` per family regardless of how many label combinations it has, so the matcher must come off first. Args: selector: A metric name, optionally followed by a brace matcher group. Returns: The bare metric family name. """ return selector.split("{", 1)[0].strip() def _metric_family_candidates(name: str) -> list[str]: """The names a contract entry could use to account for an emitted series. One histogram instrument reaches Prometheus as three names. Folding the ``_bucket``/``_count``/``_sum`` suffix back off gives the base family, so a contract entry or a regex written for the family covers all three. The unfolded name is always kept as a candidate too, and a name is accounted for if *any* candidate matches. That is what makes folding ``_count`` safe even though a plain gauge can legitimately end in it (``jobq_job_count`` does): the gauge matches on its own full name, and the extra ``jobq_job`` candidate can only ever add coverage, never remove it. Args: name: Emitted Prometheus metric family name. Returns: ``[name]``, plus the base family if ``name`` carries a histogram suffix. """ candidates = [name] for suffix in HISTOGRAM_SUFFIXES: if name.endswith(suffix) and len(name) > len(suffix): candidates.append(name[: -len(suffix)]) break return candidates def _accounted_metric_names( expected: dict[str, Any], ) -> tuple[set[str], list[re.Pattern[str]]]: """Everything expected_metrics.json accounts for, in family-name form. Three sources, all of which count as "the contract has considered this name" — which is what reverse coverage asks, not "is this name asserted": * every selector under a group's ``metrics`` (asserted), * every key of ``not_asserted.metrics_excluded`` (deliberately unasserted with a written reason), * every regex under the top-level ``accounted_patterns`` (bulk families). Literal histogram entries are expanded to the whole triple plus the base family, so listing only ``foo_bucket`` still accounts for ``foo_count`` and ``foo_sum``. The expansion is keyed off ``_bucket``/``_sum`` only, never off ``_count`` alone, so a gauge named ``..._count`` does not silently claim a shorter base family that nothing emits. Args: expected: The parsed expected_metrics.json contract. Returns: A (literal family names, compiled patterns) pair. """ literals: set[str] = set() for group in expected.values(): if not isinstance(group, dict): continue for selector in group.get("metrics", []): literals.add(_selector_metric_name(selector)) # not_asserted records its entries under metrics_excluded, keyed by name. literals.update(group.get("metrics_excluded", {})) for name in list(literals): for suffix in ("_bucket", "_sum"): if name.endswith(suffix) and len(name) > len(suffix): base = name[: -len(suffix)] literals.add(base) literals.update(base + s for s in HISTOGRAM_SUFFIXES) break patterns = [ re.compile(entry["pattern"]) for entry in expected.get("accounted_patterns", []) if entry.get("pattern") ] return literals, patterns def _unaccounted_metric_names( emitted: list[str], expected: dict[str, Any] ) -> list[str]: """Metric families Prometheus holds that the contract never mentions. Args: emitted: Metric family names from the Prometheus __name__ label. expected: The parsed expected_metrics.json contract. Returns: Sorted list of emitted family names nothing in the contract accounts for. """ literals, patterns = _accounted_metric_names(expected) unaccounted = [] for name in emitted: candidates = _metric_family_candidates(name) accounted = any( candidate in literals or any(pattern.fullmatch(candidate) for pattern in patterns) for candidate in candidates ) if not accounted: unaccounted.append(name) return sorted(unaccounted) def _metric_check_targets( expected: dict[str, Any], ) -> tuple[list[tuple[str, str]], list[tuple[str, str, list[str]]]]: """Flatten expected_metrics.json into the two lists of check targets. Args: expected: The parsed expected_metrics.json contract. Returns: A (metric targets, label targets) pair. Metric targets are (group, metric selector) for every name under a group's "metrics". Label targets are (group, label, that group's metric selectors) for every name under a group's "required_labels" — read for every group that declares it, not just spanmetrics. Nothing read the key at all until this was added, so the four labels the spanmetrics group documented as required had never actually been checked. Note: A group is any top-level object EXCEPT those in SKIPPED_METRIC_GROUPS. The contract also carries a string ("description"), a list ("accounted_patterns") and an object that declares no metrics ("grafana_dashboards"); the isinstance test skips the first two structurally rather than by name, so adding another non-group key cannot break this function, and the third contributes nothing because it has no "metrics" key. The name-based exclusion is still required for one group, so the isinstance test alone is NOT equivalent to it. "sync_diagnostics" is an object and does declare "metrics", but it has its own validator, assert_sync_diagnostics_metrics, which polls and reports the same names. Selecting it here as well would poll every metric in the group twice and report each check twice, inflating the totals. It is excluded by name because the reason is ownership, which no structural test can express. """ groups = [ (category_key, category_data) for category_key, category_data in expected.items() if isinstance(category_data, dict) and category_key not in SKIPPED_METRIC_GROUPS ] targets = [ (category_key, metric_name) for category_key, category_data in groups for metric_name in category_data.get("metrics", []) ] label_targets = [ (category_key, label, category_data.get("metrics", [])) for category_key, category_data in groups for label in category_data.get("required_labels", []) ] return targets, label_targets async def validate_metrics( session: aiohttp.ClientSession, prometheus_url: str, report: ValidationReport, ) -> None: """Validate that expected metrics appear in Prometheus with non-zero values. Two kinds of check come out of expected_metrics.json: every name under a group's "metrics" must have at least one series, and every label under a group's "required_labels" must reach at least one of that group's series with a non-empty value. Args: session: aiohttp client session. prometheus_url: Base URL for Prometheus API (e.g., http://localhost:9090). report: ValidationReport to accumulate results. """ logger.info("--- Metric Validation (Prometheus) ---") emitted_metric_names = await _log_prometheus_metric_names(session, prometheus_url) with open(EXPECTED_METRICS_FILE) as f: expected = json.load(f) # Flatten the contract, then poll every target concurrently against ONE # shared deadline. Polling them serially made each metric own its own # timeout, so the waits were additive: 58 metrics x 45 s = 43.5 min, which # overran the CI job budget and lost the artifact-upload and summary # diagnostics. Sharing the deadline bounds the whole phase to a single # poll window. # # _metric_check_targets applies SKIPPED_METRIC_GROUPS, which is what keeps # "sync_diagnostics" out of this walk: it has its own validator, # assert_sync_diagnostics_metrics, and letting both walk it would poll and # report every metric in the group twice. targets, label_targets = _metric_check_targets(expected) deadline = time.monotonic() + METRIC_POLL_TIMEOUT_SEC sem = asyncio.Semaphore(METRIC_POLL_CONCURRENCY) # Both kinds of check share the one deadline and the one concurrency bound, # so the label checks cost no extra poll window and add no extra load. metric_checks, label_checks = await asyncio.gather( asyncio.gather( *( _check_prometheus_metric( session, prometheus_url, metric_name, category, deadline, sem ) for category, metric_name in targets ) ), asyncio.gather( *( _check_metric_label( session, prometheus_url, category, label, metric_selectors, deadline, sem, ) for category, label, metric_selectors in label_targets ) ), ) # Add in contract order, not completion order, so the report and its log # lines stay deterministic across runs. Existence checks keep their place # ahead of the label checks, so no existing check's position moves. for check in [*metric_checks, *label_checks]: report.add(check) # Reverse direction: metric families Prometheus holds that the contract # never mentions. Added last so no existing check's position moves. unaccounted_metrics = _unaccounted_metric_names(emitted_metric_names, expected) if unaccounted_metrics: _log_name_list( "Metric families emitted but NOT accounted for by expected_metrics.json " "(warning only, does not fail CI)", unaccounted_metrics, ) report.add( _reverse_coverage_result( check_name="metric.reverse_coverage", category="metric", noun="metric families", emitted_count=len(emitted_metric_names), unaccounted=unaccounted_metrics, ) ) def _selector_with_label(metric_selector: str, label: str) -> str: """Add a "label is present and non-empty" matcher to a metric selector. ``