fix(telemetry): fail the regression gate on a unit change, and report what it gated

compare_to_baseline took the unit from the baseline entry and dropped the current
run's, and nothing compared the two, so a us -> ms change was scored as a numeric
delta: four keys rewritten to the same physical durations reported 99.9%
improvements and the gate exited 0. prom_queries.py says the baseline preserves
the unit "so the comparator can sanity-check unit drift"; it never did. A unit
mismatch now fails and names both units.

The workflow's step summary printed total, regressions and improvements. total is
every key in the report -- the union of baseline and current -- so it was neither
the baseline count nor what was gated, and missing_in_current was computed and
never printed. A run that gated 16 of 20 keys read as a full comparison. The
comparator now reports a real "compared" count and the summary prints it beside
the not-captured count, with a warning when any key was missed. The table also
refused nothing on a truncated report; existence is not readability.

check_regression_bounds told the operator to add max_abs_increase while reading
max_abs_increase_ms / _us, so following the message added a key nothing reads and
the gate kept failing with no explanation. The committed thresholds use only the
suffixed spelling, so the message was the defect. Its three JSON inputs were also
unchecked: a top-level null, list or number parsed and then died on the first
.get, and a string "metrics" survived the placeholder test and reported its own
characters as gated keys -- wrong advice rather than a crash.

Four tests cover these; all four fail against the previous checker.

.github/scripts/telemetry/check_regression_bounds.py

@@ -87,6 +87,11 @@ UNIT_TO_MS = {"ms": 1.0, "s": 1000.0}
 REL_TOLERANCE = 1e-12
 
 
+def is_number(value):
+    """True for a real JSON number. bool is an int subclass, so exclude it."""
+    return not isinstance(value, bool) and isinstance(value, (int, float))
+
+
 def read_text_or_exit(path):
     """Read a required text input, or exit 1 naming the input that failed."""
     try:
@@ -96,11 +101,22 @@ def read_text_or_exit(path):
 
 
 def read_json_or_exit(path):
-    """Read and parse a required JSON input, or exit 1 naming what failed."""
+    """Read and parse a required JSON object, or exit 1 naming what failed.
+
+    All three JSON inputs are objects. A top-level null, list or number parses
+    fine and then dies on the first .get, so check the shape here rather than
+    report it as a traceback pointing into this script.
+    """
     try:
-        return json.loads(read_text_or_exit(path))
+        parsed = json.loads(read_text_or_exit(path))
     except json.JSONDecodeError as exc:
         sys.exit(f"{path}: required input is not valid JSON -- {exc}")
+    if not isinstance(parsed, dict):
+        sys.exit(
+            f"{path}: required input is valid JSON but its top level is "
+            f"{type(parsed).__name__}, not an object -- nothing can be read from it"
+        )
+    return parsed
 
 
 def span_edges_ms():
@@ -264,7 +280,7 @@ def _unusable_baseline(key, value, unit):
         A failure string, or None when the value is usable.
     """
     # bool is a subclass of int; True would otherwise pass as the number 1.
-    if isinstance(value, bool) or not isinstance(value, (int, float)):
+    if not is_number(value):
         return (
             f"{key}: baseline value {value!r} is not a number, so no bound can be "
             f"derived from it. Recapture the baseline from a CI run rather than "
@@ -338,12 +354,24 @@ def check_key(key, entry, thresholds, ladders):
     if rule is None:
         failures.append(
             f"{key}: no per-metric override, so it falls back to the defaults and "
-            f"gates on the percentage bound alone. Add an override with "
-            f"max_abs_increase = {hi_next - value!r} (rule B)"
+            f"gates on the percentage bound alone. Add an override in "
+            f"{THRESHOLDS} with "
+            f"max_abs_increase_{unit} = {hi_next - value!r} (rule B)"
         )
         return failures
 
+    if not isinstance(rule, dict):
+        return [
+            f"{key}: threshold override {rule!r} is not an object carrying "
+            f"max_abs_increase_{unit} and max_pct_increase -- fix it in {THRESHOLDS}"
+        ]
+
     bound = rule.get("max_abs_increase_ms", rule.get("max_abs_increase_us"))
+    if bound is not None and not is_number(bound):
+        return [
+            f"{key}: max_abs_increase_{unit} is {bound!r}, not a number, so it "
+            f"cannot be compared with the derived bound -- fix it in {THRESHOLDS}"
+        ]
     expected = hi_next - value
     if bound is None or abs(bound - expected) > REL_TOLERANCE * expected:
         failures.append(
@@ -354,6 +382,11 @@ def check_key(key, entry, thresholds, ladders):
     pct = rule.get("max_pct_increase")
     if pct is None:
         failures.append(f"{key}: no max_pct_increase, so the metric never gates")
+    elif not is_number(pct):
+        failures.append(
+            f"{key}: max_pct_increase is {pct!r}, not a number -- fix it in "
+            f"{THRESHOLDS}"
+        )
     elif bound is not None and pct >= 100.0 * bound / value:
         failures.append(
             f"{key}: max_pct_increase {pct:g}% is at or above the absolute bound's "
@@ -391,6 +424,13 @@ def main():
 
     ladders = {"ms": span_edges_ms(), "us": microsecond_edges()}
     gated = baseline["metrics"]
+    # A list or a string is truthy, so it survives the placeholder test above
+    # and then either crashes or reports its characters as gated keys.
+    if not isinstance(gated, dict):
+        sys.exit(
+            f"{BASELINE}: 'metrics' is {type(gated).__name__}, not an object of "
+            f"key -> {{value, unit}} -- recapture the baseline from a CI run"
+        )
     failures = []
 
     declared = declared_keys(metrics_cfg)

.github/scripts/telemetry/test_check_regression_bounds.py

@@ -156,6 +156,45 @@ class TestInputHandling(CheckerCase):
         self.assertEqual(code, 1, out)
         self.assertIn("valid JSON", out)
 
+    def test_non_object_top_level_fails_naming_the_input(self):
+        """Valid JSON of the wrong shape must be named, not raise a traceback.
+
+        A top-level null, list or number parses, so it reaches the first .get
+        and dies pointing at a line in the checker rather than at the file the
+        operator has to fix.
+        """
+        for rel, text in (
+            (BASELINE, "null"),
+            (THRESHOLDS, "[]"),
+            (METRICS, "5"),
+        ):
+            with self.subTest(input=rel):
+                self.setUp()
+                (self.tree / rel).write_text(text)
+                code, out = self.run_checker()
+                self.assertEqual(code, 1, out)
+                self.assertIn(rel, out)
+                self.assertIn("not an object", out)
+                self.assertNotIn("Traceback", out)
+
+    def test_non_object_metrics_map_fails_naming_the_input(self):
+        """A string 'metrics' is truthy, so it slips past the placeholder test.
+
+        Left unchecked it reports the string's own characters as gated keys,
+        which is worse than a crash: the advice is wrong rather than absent.
+        An empty map still has to pass, because that is the bootstrap state.
+        """
+        self.edit_json(BASELINE, lambda d: d.update(metrics="span.tx.process.p99"))
+        code, out = self.run_checker()
+        self.assertEqual(code, 1, out)
+        self.assertIn("'metrics' is str", out)
+        self.assertNotIn("Traceback", out)
+
+        self.setUp()
+        self.edit_json(BASELINE, lambda d: d.update(metrics={}))
+        code, out = self.run_checker()
+        self.assertEqual(code, 0, out)
+
 
 class TestRules(CheckerCase):
     """One case per rule, so a rule that stops flagging is caught."""
@@ -183,6 +222,40 @@ class TestRules(CheckerCase):
         self.assertEqual(code, 1, out)
         self.assertIn("(rule B)", out)
 
+    def test_rule_b_names_the_unit_suffixed_key(self):
+        """The key it tells the operator to add must be the key the code reads.
+
+        The bound is stored as max_abs_increase_ms or _us. A message naming a
+        bare max_abs_increase sends the operator to add a key nothing reads, so
+        the gate keeps failing with no explanation. Both suffixes are covered,
+        because a test on the ms side alone passes on a hard-coded "_ms".
+        """
+        for group, suffix in (
+            ("span.ledger.build", "max_abs_increase_ms"),
+            ("job.transaction.queued", "max_abs_increase_us"),
+        ):
+            with self.subTest(group=group):
+                self.setUp()
+                self.edit_json(THRESHOLDS, lambda d: d["overrides"].pop(group))
+                code, out = self.run_checker()
+                self.assertEqual(code, 1, out)
+                self.assertIn("(rule B)", out)
+                self.assertIn(suffix, out)
+                self.assertNotIn("max_abs_increase =", out)
+
+    def test_non_numeric_threshold_is_reported_not_crashed(self):
+        """A hand-edited bound that is a string must be named, not raise."""
+        self.edit_json(
+            THRESHOLDS,
+            lambda d: d["overrides"]["span.ledger.build"]["p99"].update(
+                max_abs_increase_ms="5.5"
+            ),
+        )
+        code, out = self.run_checker()
+        self.assertEqual(code, 1, out)
+        self.assertIn("not a number", out)
+        self.assertNotIn("Traceback", out)
+
     def test_rule_c_flags_rounded_bound(self):
         """A bound rounded for readability is still not the derived bound."""
         _, exact = self.gated("span.tx.process.p99")