diff --git a/docker/telemetry/workload/validate_telemetry.py b/docker/telemetry/workload/validate_telemetry.py index d9386641d0..e4e70c0acc 100644 --- a/docker/telemetry/workload/validate_telemetry.py +++ b/docker/telemetry/workload/validate_telemetry.py @@ -52,6 +52,16 @@ SCRIPT_DIR = Path(__file__).parent EXPECTED_SPANS_FILE = SCRIPT_DIR / "expected_spans.json" EXPECTED_METRICS_FILE = SCRIPT_DIR / "expected_metrics.json" +# Some beast::insight gauges/counters (ledger-age, peer-finder, overlay +# traffic) only populate after the node validates ledgers and sustains peer +# traffic, then travel a 1s periodic OTLP export + a 15s Prometheus scrape +# before they are queryable. On a slow CI runner the fixed post-workload wait +# can end before that pipeline settles, so a single query races and reports 0 +# series. Poll each missing metric for up to this long (covering two scrape +# cycles) before failing, so the check is robust to runner speed. +METRIC_POLL_TIMEOUT_SEC = 45.0 +METRIC_POLL_INTERVAL_SEC = 5.0 + # --------------------------------------------------------------------------- # Data classes @@ -595,26 +605,37 @@ async def _check_prometheus_metric( # stale in Prometheus and disappear from instant queries. The # series endpoint returns any metric that existed in the window, # regardless of staleness. + # + # Poll rather than query once: late-populating gauges/counters may + # not have completed the export+scrape pipeline when this runs, so a + # single query races. Re-query until the metric appears or the poll + # window elapses; a metric that never appears still fails after the + # timeout. params: dict[str, str] = {"match[]": metric_name} - async with session.get( - f"{prometheus_url}/api/v1/series", params=params - ) as resp: - data = await resp.json() - results = data.get("data", []) - series_count = len(results) - report.add( - CheckResult( - name=f"metric.{category}.{metric_name}", - category="metric", - passed=series_count > 0, - message=( - f"{metric_name}: {series_count} series" - if series_count > 0 - else f"{metric_name}: 0 series (expected > 0)" - ), - details={"series_count": series_count}, - ) + series_count = 0 + deadline = time.monotonic() + METRIC_POLL_TIMEOUT_SEC + while True: + async with session.get( + f"{prometheus_url}/api/v1/series", params=params + ) as resp: + data = await resp.json() + series_count = len(data.get("data", [])) + if series_count > 0 or time.monotonic() >= deadline: + break + await asyncio.sleep(METRIC_POLL_INTERVAL_SEC) + report.add( + CheckResult( + name=f"metric.{category}.{metric_name}", + category="metric", + passed=series_count > 0, + message=( + f"{metric_name}: {series_count} series" + if series_count > 0 + else f"{metric_name}: 0 series (expected > 0)" + ), + details={"series_count": series_count}, ) + ) except Exception as exc: report.add( CheckResult(