#!/usr/bin/env python3 """Tests for validate_telemetry.py's hierarchy check. Run with plain python3 -- there is no pytest in the harness requirements, and this file is deliberately runnable with nothing but the standard library plus the aiohttp that validate_telemetry.py already imports: python3 docker/telemetry/workload/test_validate_telemetry.py Why a stub Tempo rather than the real one: the behaviour under test is which traces the check ASKS FOR, which a live backend cannot demonstrate -- a passing query against real data proves the data happened to co-operate, not that the query was right. The stub records every request, so a test can assert on the query itself and on the answer the check derives from a known corpus. """ import asyncio import json import sys from pathlib import Path from typing import Any sys.path.insert(0, str(Path(__file__).parent)) import validate_telemetry as vt # noqa: E402 class FakeResponse: """Minimal stand-in for an aiohttp response used as an async context manager.""" def __init__(self, payload: dict[str, Any], status: int = 200) -> None: self._payload = payload self.status = status async def __aenter__(self) -> "FakeResponse": return self async def __aexit__(self, *exc: object) -> bool: return False async def json(self) -> dict[str, Any]: return self._payload async def text(self) -> str: return json.dumps(self._payload) class FakeTempo: """A Tempo whose corpus is fixed and whose queries are recorded. Args: traces: Maps a trace id to the list of span names that trace contains, ordered newest first, which is the order /api/search returns. """ def __init__(self, traces: dict[str, list[str]]) -> None: self.traces = traces self.queries: list[tuple[str, int]] = [] def get(self, url: str, params: dict[str, str] | None = None) -> FakeResponse: params = params or {} if "/api/search" in url: query, limit = params["q"], int(params.get("limit", 20)) self.queries.append((query, limit)) matched = [ tid for tid, names in self.traces.items() if _query_matches_trace(query, names) ] return FakeResponse({"traces": [{"traceID": t} for t in matched[:limit]]}) if "/api/traces/" in url: tid = url.rsplit("/", 1)[-1] spans = [{"name": n, "attributes": []} for n in self.traces.get(tid, [])] return FakeResponse({"batches": [{"scopeSpans": [{"spans": spans}]}]}) raise AssertionError(f"unexpected request: {url}") def _query_matches_trace(query: str, names: list[str]) -> bool: """Evaluate the subset of TraceQL this suite uses against one trace. Supports the trace-level conjunction of name predicates the hierarchy check builds: every `name="X"` (or `name=~"X"`) term must be satisfied by some span in the trace. That is the whole semantic the check relies on, so the stub models exactly it and nothing more. """ import re # `name` only as a bare intrinsic. The lookbehind is load-bearing: without it # this also matches the resource.service.name="xrpld" term every query # carries, and then demands a span literally named "xrpld" -- which made the # first run of these tests fail with "No traces" instead of the # sampling failure they exist to demonstrate. terms = re.findall(r'(? None: self.results: list[Any] = [] def add(self, result: Any) -> None: self.results.append(result) def run(coro: Any) -> Any: return asyncio.run(coro) def test_child_found_in_a_trace_outside_the_newest_three() -> None: """The check must find a child that co-occurs only in an older trace. This is the txq.accept_tx / ledger.acquire.txtree shape: the parent fires on every ledger close, the child only when a rarely-met condition holds, so the newest traces carry the parent alone. Sampling the newest N parent traces reports "not found" on a corpus that plainly contains the relationship. The production change that makes this fail: reverting the hierarchy check to search the parent alone and inspect only the first N results. """ tempo = FakeTempo( { # Newest first, as /api/search returns. The child is only in the oldest. "t5": ["txq.accept"], "t4": ["txq.accept"], "t3": ["txq.accept"], "t2": ["txq.accept"], "t1": ["txq.accept", "txq.accept_tx"], } ) report = Report() run( vt._validate_parent_child( tempo, "http://tempo", {"parent": "txq.accept", "child": "txq.accept_tx"}, report, ) ) assert len(report.results) == 1, report.results result = report.results[0] assert result.passed, f"expected PASS, got: {result.message}" assert result.name == "span.hierarchy.txq.accept->txq.accept_tx" def test_absent_child_still_fails() -> None: """A child that co-occurs in no trace must still fail. Guards the obvious way to "fix" the test above -- making the check pass whenever the parent exists. Without this, a harness that silently stopped emitting a child would go green. """ tempo = FakeTempo({"t2": ["txq.accept"], "t1": ["txq.accept"]}) report = Report() run( vt._validate_parent_child( tempo, "http://tempo", {"parent": "txq.accept", "child": "txq.accept_tx"}, report, ) ) assert len(report.results) == 1 assert not report.results[0].passed assert "txq.accept_tx" in report.results[0].message def test_missing_parent_reports_the_parent_not_the_child() -> None: """No parent traces at all is a distinct failure from a missing child. The two mean different things to whoever reads the report -- a missing parent says the span stopped being emitted, a missing child says the relationship broke -- so the messages must not collapse into one. """ tempo = FakeTempo({"t1": ["ledger.build"]}) report = Report() run( vt._validate_parent_child( tempo, "http://tempo", {"parent": "txq.accept", "child": "txq.accept_tx"}, report, ) ) assert len(report.results) == 1 assert not report.results[0].passed assert "No txq.accept traces" in report.results[0].message def test_wildcard_child_matches_any_family_member() -> None: """A wildcard child must be satisfied by any concrete member. rpc.command.* names vary per request, so pinning one literal would make the check depend on which command the sampled traces happened to carry. """ tempo = FakeTempo({"t1": ["rpc.ws_message", "rpc.command.fee"]}) report = Report() run( vt._validate_parent_child( tempo, "http://tempo", {"parent": "rpc.ws_message", "child": "rpc.command.*"}, report, ) ) assert report.results[0].passed, report.results[0].message def main() -> int: tests = [v for k, v in sorted(globals().items()) if k.startswith("test_")] failed = 0 for test in tests: try: test() except AssertionError as exc: failed += 1 print(f"FAIL {test.__name__}: {exc}") except Exception as exc: # noqa: BLE001 - report any error as a failure failed += 1 print(f"ERROR {test.__name__}: {type(exc).__name__}: {exc}") else: print(f"PASS {test.__name__}") print(f"\n{len(tests) - failed}/{len(tests)} passed") return 1 if failed else 0 if __name__ == "__main__": sys.exit(main())