mirror of
https://github.com/XRPLF/rippled.git
synced 2026-09-27 15:28:03 +00:00
7f829a5929
compare_to_baseline took the unit from the baseline entry and dropped the current run's, and nothing compared the two, so a us -> ms change was scored as a numeric delta: four keys rewritten to the same physical durations reported 99.9% improvements and the gate exited 0. prom_queries.py says the baseline preserves the unit "so the comparator can sanity-check unit drift"; it never did. A unit mismatch now fails and names both units. The workflow's step summary printed total, regressions and improvements. total is every key in the report -- the union of baseline and current -- so it was neither the baseline count nor what was gated, and missing_in_current was computed and never printed. A run that gated 16 of 20 keys read as a full comparison. The comparator now reports a real "compared" count and the summary prints it beside the not-captured count, with a warning when any key was missed. The table also refused nothing on a truncated report; existence is not readability. check_regression_bounds told the operator to add max_abs_increase while reading max_abs_increase_ms / _us, so following the message added a key nothing reads and the gate kept failing with no explanation. The committed thresholds use only the suffixed spelling, so the message was the defect. Its three JSON inputs were also unchecked: a top-level null, list or number parsed and then died on the first .get, and a string "metrics" survived the placeholder test and reported its own characters as gated keys -- wrong advice rather than a crash. Four tests cover these; all four fail against the previous checker.
530 lines
18 KiB
Python
530 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Compare captured OTel timings against a committed baseline.
|
|
|
|
Operating modes (chosen automatically based on the baseline file contents):
|
|
|
|
1. **No baseline** — if ``baseline-timings.json`` has an empty
|
|
``metrics`` object (or is marked with ``"placeholder": true``), this
|
|
script is in "populate" mode. It prints the captured timings JSON in
|
|
the exact format expected for pasting into
|
|
``baselines/baseline-timings.json``, then exits 0. No regression check.
|
|
An INCOMPLETE capture is refused here instead (exit 2): the timings file
|
|
states its own completeness in its ``capture`` block, and a capture that
|
|
fell short of ``--min-capture-ratio`` describes metrics that were never
|
|
measured. Pasted in, it would narrow the gate to whichever keys came back
|
|
with nothing reporting that it had narrowed. Refusing prints nothing on
|
|
stdout, so a caller redirecting stdout to the baseline file cannot end up
|
|
with a truncated one blessed by exit 0.
|
|
|
|
2. **Populated baseline** — per-metric percentage AND absolute deltas are
|
|
computed against thresholds from ``regression-thresholds.json``. A
|
|
regression occurs when BOTH bounds are breached for the same quantile.
|
|
The one exception is a non-positive baseline, where the percentage is
|
|
undefined: there the absolute bound decides alone.
|
|
Prints a human-readable table and writes a full JSON report.
|
|
Exits 1 if any regression was detected, else 0.
|
|
|
|
Inputs:
|
|
--timings Captured timings JSON (from capture_timings.py)
|
|
--baseline Committed baseline JSON
|
|
--thresholds Threshold policy JSON
|
|
--report Where to write regression-report.json (optional)
|
|
|
|
Exit codes:
|
|
0 — No baseline (paste-me emitted), OR baseline populated and no regression
|
|
1 — Regression detected (at least one metric breached both bounds)
|
|
2 — Internal error (e.g. bad JSON, baseline/current key mismatch), OR the
|
|
baseline is a placeholder and the capture is too incomplete to seed one
|
|
|
|
Note that an incomplete capture is refused only on the paste-me path. Against a
|
|
POPULATED baseline the comparison still runs and still reports uncaptured keys
|
|
as ``not captured in current run``, exactly as before: there the thin capture
|
|
cannot corrupt anything, and reporting what is missing is more useful than
|
|
refusing to look.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import logging
|
|
import sys
|
|
from dataclasses import dataclass, asdict
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
logger = logging.getLogger("compare_to_baseline")
|
|
|
|
|
|
@dataclass
|
|
class MetricDelta:
|
|
"""Single metric's baseline-vs-current comparison outcome.
|
|
|
|
Attributes:
|
|
key: Flat metric key (e.g. span.tx.process.p99).
|
|
baseline: Baseline value (may be None if unpopulated).
|
|
current: Current run value (may be None if not captured).
|
|
delta: current - baseline (None if either side None).
|
|
pct_change: 100 * delta / baseline (None if baseline ≤ 0).
|
|
unit: Unit from baseline (preserved as-is).
|
|
threshold_pct: Resolved per-metric pct threshold.
|
|
threshold_abs: Resolved per-metric absolute threshold.
|
|
regressed: True iff both bounds breached, or — when the
|
|
baseline is not positive and pct_change is
|
|
therefore None — iff the absolute bound breached.
|
|
note: Human-readable classification of the outcome.
|
|
"""
|
|
|
|
key: str
|
|
baseline: float | None
|
|
current: float | None
|
|
delta: float | None
|
|
pct_change: float | None
|
|
unit: str
|
|
threshold_pct: float | None
|
|
threshold_abs: float | None
|
|
regressed: bool
|
|
note: str
|
|
|
|
|
|
def load_json(path: Path) -> dict:
|
|
with open(path) as f:
|
|
return json.load(f)
|
|
|
|
|
|
def is_placeholder(baseline: dict) -> bool:
|
|
"""A baseline is a placeholder if explicitly marked OR metrics are empty."""
|
|
if baseline.get("placeholder") is True:
|
|
return True
|
|
return not baseline.get("metrics")
|
|
|
|
|
|
def capture_is_complete(timings: dict) -> bool:
|
|
"""True only if the timings file states that its capture was complete.
|
|
|
|
The flag is written by ``capture_timings.py``, which computes it against
|
|
``--min-capture-ratio``; this reads it rather than re-deriving the rule, so
|
|
the two cannot disagree. Anything other than boolean ``true`` — the block
|
|
absent because the artifact predates it, a truncated file, a string
|
|
``"true"`` from a hand edit — is treated as not complete. Completeness has
|
|
to be proven, not assumed, because assuming it is how a thin capture
|
|
reaches a committed baseline.
|
|
"""
|
|
capture = timings.get("capture")
|
|
if not isinstance(capture, dict):
|
|
return False
|
|
return capture.get("complete") is True
|
|
|
|
|
|
def print_incomplete_capture(timings: dict) -> None:
|
|
"""Explain why no paste-me block was printed, naming the shortfall.
|
|
|
|
Deliberately writes to stderr only. The paste-me path's stdout is the
|
|
baseline file's contents, so leaving stdout empty is what stops a
|
|
``> baseline-timings.json`` redirect from producing a file that looks
|
|
captured.
|
|
"""
|
|
capture = timings.get("capture")
|
|
if isinstance(capture, dict):
|
|
detail = (
|
|
f"only {capture.get('captured')} of {capture.get('declared')} declared "
|
|
f"metrics came back, below the minimum ratio "
|
|
f"{capture.get('min_ratio')}"
|
|
)
|
|
else:
|
|
detail = (
|
|
"the file carries no 'capture' block, so its completeness cannot be "
|
|
"established — recapture with the current capture_timings.py"
|
|
)
|
|
|
|
banner = "=" * 72
|
|
print(banner, file=sys.stderr)
|
|
print(" CAPTURE INCOMPLETE — refusing to print a baseline block", file=sys.stderr)
|
|
print(f" {detail}.", file=sys.stderr)
|
|
print(
|
|
" These timings may describe metrics that were never measured. Pasting\n"
|
|
" them into baselines/baseline-timings.json would narrow the regression\n"
|
|
" gate to whichever keys were captured, with nothing reporting that it\n"
|
|
" had narrowed. Fix the capture and re-run.",
|
|
file=sys.stderr,
|
|
)
|
|
print(banner, file=sys.stderr)
|
|
|
|
|
|
def print_paste_me(timings: dict) -> None:
|
|
"""Print captured timings in the exact baseline-timings.json format.
|
|
|
|
The output between the two banner lines is the file contents to paste,
|
|
byte-for-byte — sorted keys, 2-space indent, trailing newline.
|
|
"""
|
|
banner = "=" * 72
|
|
print(banner, file=sys.stderr)
|
|
print(
|
|
" NO BASELINE FOUND — paste the JSON below into",
|
|
file=sys.stderr,
|
|
)
|
|
print(
|
|
" docker/telemetry/workload/baselines/baseline-timings.json",
|
|
file=sys.stderr,
|
|
)
|
|
print(banner, file=sys.stderr)
|
|
|
|
print(json.dumps(timings, indent=2, sort_keys=True))
|
|
|
|
print(banner, file=sys.stderr)
|
|
print(
|
|
" (End of paste-me JSON. Gate did NOT run — baseline is empty.)",
|
|
file=sys.stderr,
|
|
)
|
|
print(banner, file=sys.stderr)
|
|
|
|
|
|
def resolve_thresholds(
|
|
key: str,
|
|
thresholds: dict,
|
|
) -> tuple[float | None, float | None]:
|
|
"""Return ``(pct_threshold, abs_threshold)`` for a metric key.
|
|
|
|
Per-metric overrides win over defaults. Returns ``(None, None)`` if no
|
|
threshold is defined for this category/quantile — such metrics are
|
|
captured but never gate the build.
|
|
"""
|
|
parts = key.split(".")
|
|
if len(parts) < 3:
|
|
return (None, None)
|
|
category_key = parts[0]
|
|
quantile_key = parts[-1]
|
|
|
|
category_map = {
|
|
"span": "span",
|
|
"rpc": "rpc_method",
|
|
"job": "job_queue",
|
|
}
|
|
cat = category_map.get(category_key)
|
|
if cat is None:
|
|
return (None, None)
|
|
|
|
override_key = f"{category_key}.{'.'.join(parts[1:-1])}"
|
|
overrides = thresholds.get("overrides", {})
|
|
defaults = thresholds.get("defaults", {}).get(cat, {})
|
|
|
|
rule = overrides.get(override_key, {}).get(quantile_key)
|
|
if rule is None:
|
|
rule = defaults.get(quantile_key)
|
|
if rule is None:
|
|
return (None, None)
|
|
|
|
pct = rule.get("max_pct_increase")
|
|
abs_bound = rule.get("max_abs_increase_ms")
|
|
if abs_bound is None:
|
|
abs_bound = rule.get("max_abs_increase_us")
|
|
return (pct, abs_bound)
|
|
|
|
|
|
def _skip_delta(
|
|
key: str,
|
|
baseline: float | None,
|
|
current: float | None,
|
|
unit: str,
|
|
thresholds: dict,
|
|
note: str,
|
|
) -> MetricDelta:
|
|
"""Build a MetricDelta for cases where comparison is not possible."""
|
|
pct_threshold, abs_threshold = resolve_thresholds(key, thresholds)
|
|
return MetricDelta(
|
|
key=key,
|
|
baseline=baseline,
|
|
current=current,
|
|
delta=None,
|
|
pct_change=None,
|
|
unit=unit,
|
|
threshold_pct=pct_threshold,
|
|
threshold_abs=abs_threshold,
|
|
regressed=False,
|
|
note=note,
|
|
)
|
|
|
|
|
|
def _delta_note(regressed: bool, delta: float, pct_change: float | None) -> str:
|
|
"""Classify one comparison outcome for the report and the table."""
|
|
if regressed:
|
|
note = "REGRESSION"
|
|
elif delta < 0:
|
|
note = "improved"
|
|
else:
|
|
note = "within bounds"
|
|
if pct_change is None:
|
|
note += " (absolute bound only; baseline not positive)"
|
|
return note
|
|
|
|
|
|
# The regression rule, applied by compute_delta below.
|
|
#
|
|
# A regression normally requires BOTH bounds to be breached simultaneously.
|
|
# That tolerates small-value noise: a 100% increase on a 0.5 ms metric (to
|
|
# 1.0 ms) is not a regression under a 5 ms absolute bound.
|
|
#
|
|
# A non-positive baseline has no defined percentage change, so there the
|
|
# absolute bound decides alone. Requiring both bounds in that case would make
|
|
# the gate unreachable and let a 0 -> 500 ms jump pass as "within bounds".
|
|
def compute_delta(
|
|
key: str,
|
|
baseline_entry: dict | None,
|
|
current_entry: dict | None,
|
|
thresholds: dict,
|
|
) -> MetricDelta:
|
|
"""Compute a MetricDelta for one metric key.
|
|
|
|
Follows the regression rule set out in the comment above, including the
|
|
non-positive-baseline exception.
|
|
"""
|
|
baseline = baseline_entry.get("value") if baseline_entry else None
|
|
current = current_entry.get("value") if current_entry else None
|
|
unit = (baseline_entry or current_entry or {}).get("unit", "")
|
|
|
|
# A unit change makes the two numbers incomparable, so subtracting them is
|
|
# meaningless: us -> ms reads as a 99.9% improvement and the gate passes.
|
|
# Fail instead, and name both units so the baseline can be refreshed.
|
|
baseline_unit = (baseline_entry or {}).get("unit", "")
|
|
current_unit = (current_entry or {}).get("unit", "")
|
|
if baseline_unit and current_unit and baseline_unit != current_unit:
|
|
pct_threshold, abs_threshold = resolve_thresholds(key, thresholds)
|
|
return MetricDelta(
|
|
key=key,
|
|
baseline=baseline,
|
|
current=current,
|
|
delta=None,
|
|
pct_change=None,
|
|
unit=f"{baseline_unit}->{current_unit}",
|
|
threshold_pct=pct_threshold,
|
|
threshold_abs=abs_threshold,
|
|
regressed=True,
|
|
note=(
|
|
f"unit changed: baseline is {baseline_unit}, current run is "
|
|
f"{current_unit} -- refresh the baseline instead of comparing"
|
|
),
|
|
)
|
|
|
|
if baseline is None and current is None:
|
|
return _skip_delta(
|
|
key, None, None, unit, thresholds, "no data (neither baseline nor current)"
|
|
)
|
|
|
|
if baseline is None:
|
|
return _skip_delta(
|
|
key, None, current, unit, thresholds, "new metric (not in baseline)"
|
|
)
|
|
|
|
if current is None:
|
|
return _skip_delta(
|
|
key, baseline, None, unit, thresholds, "not captured in current run"
|
|
)
|
|
|
|
pct_threshold, abs_threshold = resolve_thresholds(key, thresholds)
|
|
delta = current - baseline
|
|
pct_change = (delta / baseline * 100.0) if baseline > 0 else None
|
|
|
|
if pct_threshold is None or abs_threshold is None:
|
|
return MetricDelta(
|
|
key=key,
|
|
baseline=baseline,
|
|
current=current,
|
|
delta=delta,
|
|
pct_change=pct_change,
|
|
unit=unit,
|
|
threshold_pct=pct_threshold,
|
|
threshold_abs=abs_threshold,
|
|
regressed=False,
|
|
note="no threshold configured",
|
|
)
|
|
|
|
abs_breach = delta > abs_threshold
|
|
if pct_change is None:
|
|
# Baseline is not positive, so there is no percentage to compare.
|
|
# The absolute bound is the only usable signal here.
|
|
regressed = abs_breach
|
|
else:
|
|
regressed = pct_change > pct_threshold and abs_breach
|
|
|
|
return MetricDelta(
|
|
key=key,
|
|
baseline=baseline,
|
|
current=current,
|
|
delta=delta,
|
|
pct_change=pct_change,
|
|
unit=unit,
|
|
threshold_pct=pct_threshold,
|
|
threshold_abs=abs_threshold,
|
|
regressed=regressed,
|
|
note=_delta_note(regressed, delta, pct_change),
|
|
)
|
|
|
|
|
|
def print_summary(deltas: list[MetricDelta]) -> None:
|
|
"""Print a sorted, human-readable table of per-metric results."""
|
|
regressions = [d for d in deltas if d.regressed]
|
|
improvements = [
|
|
d
|
|
for d in deltas
|
|
if d.delta is not None and d.delta < 0 and d.baseline not in (None, 0)
|
|
]
|
|
improvements.sort(key=lambda d: d.pct_change or 0)
|
|
regressions.sort(key=lambda d: -(d.pct_change or 0))
|
|
|
|
print("=" * 72)
|
|
print(f" Regression check: {len(regressions)} regression(s) detected")
|
|
print("=" * 72)
|
|
|
|
if regressions:
|
|
print(
|
|
"\nRegressions (breached BOTH pct AND absolute bounds, or the "
|
|
"absolute bound alone where the baseline is not positive):"
|
|
)
|
|
_print_table(regressions)
|
|
# A regression can also be recorded with no delta at all -- a unit
|
|
# change makes the two numbers incomparable. That row prints as dashes,
|
|
# so name the reason here or the table looks like a bug.
|
|
for d in regressions:
|
|
if d.delta is None:
|
|
print(f" {d.key}: {d.note}")
|
|
|
|
if improvements:
|
|
top = improvements[:5]
|
|
print("\nTop improvements:")
|
|
_print_table(top)
|
|
|
|
missing = [d for d in deltas if d.note == "not captured in current run"]
|
|
if missing:
|
|
print(f"\n{len(missing)} baseline metric(s) not captured in current run:")
|
|
for d in missing:
|
|
print(f" {d.key}")
|
|
|
|
|
|
def _print_table(rows: list[MetricDelta]) -> None:
|
|
"""Print a fixed-width table for a list of deltas."""
|
|
header = f" {'METRIC':<45} {'BASE':>10} {'CUR':>10} {'Δ':>10} {'%':>8} UNIT"
|
|
print(header)
|
|
print(" " + "-" * (len(header) - 2))
|
|
for d in rows:
|
|
base = f"{d.baseline:.2f}" if d.baseline is not None else "-"
|
|
cur = f"{d.current:.2f}" if d.current is not None else "-"
|
|
delta = f"{d.delta:+.2f}" if d.delta is not None else "-"
|
|
pct = f"{d.pct_change:+.1f}%" if d.pct_change is not None else "-"
|
|
print(f" {d.key:<45} {base:>10} {cur:>10} {delta:>10} {pct:>8} {d.unit}")
|
|
|
|
|
|
def write_report(
|
|
deltas: list[MetricDelta],
|
|
report_path: Path,
|
|
baseline: dict,
|
|
timings: dict,
|
|
) -> None:
|
|
"""Write regression-report.json — machine-readable artifact for CI."""
|
|
regressions = [d for d in deltas if d.regressed]
|
|
payload = {
|
|
"schema_version": 1,
|
|
"baseline_captured_at": baseline.get("captured_at"),
|
|
"baseline_git_sha": baseline.get("git_sha"),
|
|
"current_captured_at": timings.get("captured_at"),
|
|
"current_git_sha": timings.get("git_sha"),
|
|
"window": timings.get("window"),
|
|
"profile": timings.get("profile"),
|
|
"summary": {
|
|
# total is every key in the report, which is the UNION of the
|
|
# baseline and the current run -- not the baseline count. "compared"
|
|
# is the only number that says how much was actually gated: a delta
|
|
# exists only when both sides had a value.
|
|
"total": len(deltas),
|
|
"compared": sum(1 for d in deltas if d.delta is not None),
|
|
"regressions": len(regressions),
|
|
"improvements": sum(
|
|
1
|
|
for d in deltas
|
|
if d.delta is not None and d.delta < 0 and d.baseline not in (None, 0)
|
|
),
|
|
"missing_in_current": sum(
|
|
1 for d in deltas if d.note == "not captured in current run"
|
|
),
|
|
},
|
|
"metrics": [asdict(d) for d in deltas],
|
|
}
|
|
report_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(report_path, "w") as f:
|
|
json.dump(payload, f, indent=2, sort_keys=True)
|
|
f.write("\n")
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"--timings",
|
|
type=Path,
|
|
required=True,
|
|
help="Captured timings JSON (from capture_timings.py)",
|
|
)
|
|
parser.add_argument(
|
|
"--baseline",
|
|
type=Path,
|
|
required=True,
|
|
help="Committed baseline-timings.json",
|
|
)
|
|
parser.add_argument(
|
|
"--thresholds",
|
|
type=Path,
|
|
default=Path(__file__).parent / "regression-thresholds.json",
|
|
help="Threshold policy JSON",
|
|
)
|
|
parser.add_argument(
|
|
"--report",
|
|
type=Path,
|
|
default=None,
|
|
help="Where to write regression-report.json (optional)",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(levelname)s %(name)s: %(message)s",
|
|
)
|
|
|
|
try:
|
|
timings = load_json(args.timings)
|
|
baseline = load_json(args.baseline)
|
|
thresholds = load_json(args.thresholds)
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
logger.error("failed to load inputs: %s", exc)
|
|
return 2
|
|
|
|
if is_placeholder(baseline):
|
|
if not capture_is_complete(timings):
|
|
print_incomplete_capture(timings)
|
|
return 2
|
|
print_paste_me(timings)
|
|
return 0
|
|
|
|
baseline_metrics = baseline.get("metrics", {})
|
|
current_metrics = timings.get("metrics", {})
|
|
|
|
all_keys = sorted(set(baseline_metrics) | set(current_metrics))
|
|
deltas = [
|
|
compute_delta(
|
|
key,
|
|
baseline_metrics.get(key),
|
|
current_metrics.get(key),
|
|
thresholds,
|
|
)
|
|
for key in all_keys
|
|
]
|
|
|
|
print_summary(deltas)
|
|
|
|
if args.report:
|
|
write_report(deltas, args.report, baseline, timings)
|
|
logger.info("wrote %s", args.report)
|
|
|
|
return 1 if any(d.regressed for d in deltas) else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|