merge: bring the lock-free ValidationTracker forward from phase10-workload-validation

Only the workload README conflicted; the tracker, config and test files merged
clean, which closes the chain from phase-7.

.github/scripts/telemetry/check_regression_bounds.py

@@ -87,6 +87,11 @@ UNIT_TO_MS = {"ms": 1.0, "s": 1000.0}
 REL_TOLERANCE = 1e-12
 
 
+def is_number(value):
+    """True for a real JSON number. bool is an int subclass, so exclude it."""
+    return not isinstance(value, bool) and isinstance(value, (int, float))
+
+
 def read_text_or_exit(path):
     """Read a required text input, or exit 1 naming the input that failed."""
     try:
@@ -96,11 +101,22 @@ def read_text_or_exit(path):
 
 
 def read_json_or_exit(path):
-    """Read and parse a required JSON input, or exit 1 naming what failed."""
+    """Read and parse a required JSON object, or exit 1 naming what failed.
+
+    All three JSON inputs are objects. A top-level null, list or number parses
+    fine and then dies on the first .get, so check the shape here rather than
+    report it as a traceback pointing into this script.
+    """
     try:
-        return json.loads(read_text_or_exit(path))
+        parsed = json.loads(read_text_or_exit(path))
     except json.JSONDecodeError as exc:
         sys.exit(f"{path}: required input is not valid JSON -- {exc}")
+    if not isinstance(parsed, dict):
+        sys.exit(
+            f"{path}: required input is valid JSON but its top level is "
+            f"{type(parsed).__name__}, not an object -- nothing can be read from it"
+        )
+    return parsed
 
 
 def span_edges_ms():
@@ -264,7 +280,7 @@ def _unusable_baseline(key, value, unit):
         A failure string, or None when the value is usable.
     """
     # bool is a subclass of int; True would otherwise pass as the number 1.
-    if isinstance(value, bool) or not isinstance(value, (int, float)):
+    if not is_number(value):
         return (
             f"{key}: baseline value {value!r} is not a number, so no bound can be "
             f"derived from it. Recapture the baseline from a CI run rather than "
@@ -338,12 +354,24 @@ def check_key(key, entry, thresholds, ladders):
     if rule is None:
         failures.append(
             f"{key}: no per-metric override, so it falls back to the defaults and "
-            f"gates on the percentage bound alone. Add an override with "
-            f"max_abs_increase = {hi_next - value!r} (rule B)"
+            f"gates on the percentage bound alone. Add an override in "
+            f"{THRESHOLDS} with "
+            f"max_abs_increase_{unit} = {hi_next - value!r} (rule B)"
         )
         return failures
 
+    if not isinstance(rule, dict):
+        return [
+            f"{key}: threshold override {rule!r} is not an object carrying "
+            f"max_abs_increase_{unit} and max_pct_increase -- fix it in {THRESHOLDS}"
+        ]
+
     bound = rule.get("max_abs_increase_ms", rule.get("max_abs_increase_us"))
+    if bound is not None and not is_number(bound):
+        return [
+            f"{key}: max_abs_increase_{unit} is {bound!r}, not a number, so it "
+            f"cannot be compared with the derived bound -- fix it in {THRESHOLDS}"
+        ]
     expected = hi_next - value
     if bound is None or abs(bound - expected) > REL_TOLERANCE * expected:
         failures.append(
@@ -354,6 +382,11 @@ def check_key(key, entry, thresholds, ladders):
     pct = rule.get("max_pct_increase")
     if pct is None:
         failures.append(f"{key}: no max_pct_increase, so the metric never gates")
+    elif not is_number(pct):
+        failures.append(
+            f"{key}: max_pct_increase is {pct!r}, not a number -- fix it in "
+            f"{THRESHOLDS}"
+        )
     elif bound is not None and pct >= 100.0 * bound / value:
         failures.append(
             f"{key}: max_pct_increase {pct:g}% is at or above the absolute bound's "
@@ -391,6 +424,13 @@ def main():
 
     ladders = {"ms": span_edges_ms(), "us": microsecond_edges()}
     gated = baseline["metrics"]
+    # A list or a string is truthy, so it survives the placeholder test above
+    # and then either crashes or reports its characters as gated keys.
+    if not isinstance(gated, dict):
+        sys.exit(
+            f"{BASELINE}: 'metrics' is {type(gated).__name__}, not an object of "
+            f"key -> {{value, unit}} -- recapture the baseline from a CI run"
+        )
     failures = []
 
     declared = declared_keys(metrics_cfg)

.github/scripts/telemetry/test_check_regression_bounds.py

@@ -156,6 +156,45 @@ class TestInputHandling(CheckerCase):
         self.assertEqual(code, 1, out)
         self.assertIn("valid JSON", out)
 
+    def test_non_object_top_level_fails_naming_the_input(self):
+        """Valid JSON of the wrong shape must be named, not raise a traceback.
+
+        A top-level null, list or number parses, so it reaches the first .get
+        and dies pointing at a line in the checker rather than at the file the
+        operator has to fix.
+        """
+        for rel, text in (
+            (BASELINE, "null"),
+            (THRESHOLDS, "[]"),
+            (METRICS, "5"),
+        ):
+            with self.subTest(input=rel):
+                self.setUp()
+                (self.tree / rel).write_text(text)
+                code, out = self.run_checker()
+                self.assertEqual(code, 1, out)
+                self.assertIn(rel, out)
+                self.assertIn("not an object", out)
+                self.assertNotIn("Traceback", out)
+
+    def test_non_object_metrics_map_fails_naming_the_input(self):
+        """A string 'metrics' is truthy, so it slips past the placeholder test.
+
+        Left unchecked it reports the string's own characters as gated keys,
+        which is worse than a crash: the advice is wrong rather than absent.
+        An empty map still has to pass, because that is the bootstrap state.
+        """
+        self.edit_json(BASELINE, lambda d: d.update(metrics="span.tx.process.p99"))
+        code, out = self.run_checker()
+        self.assertEqual(code, 1, out)
+        self.assertIn("'metrics' is str", out)
+        self.assertNotIn("Traceback", out)
+
+        self.setUp()
+        self.edit_json(BASELINE, lambda d: d.update(metrics={}))
+        code, out = self.run_checker()
+        self.assertEqual(code, 0, out)
+
 
 class TestRules(CheckerCase):
     """One case per rule, so a rule that stops flagging is caught."""
@@ -183,6 +222,40 @@ class TestRules(CheckerCase):
         self.assertEqual(code, 1, out)
         self.assertIn("(rule B)", out)
 
+    def test_rule_b_names_the_unit_suffixed_key(self):
+        """The key it tells the operator to add must be the key the code reads.
+
+        The bound is stored as max_abs_increase_ms or _us. A message naming a
+        bare max_abs_increase sends the operator to add a key nothing reads, so
+        the gate keeps failing with no explanation. Both suffixes are covered,
+        because a test on the ms side alone passes on a hard-coded "_ms".
+        """
+        for group, suffix in (
+            ("span.ledger.build", "max_abs_increase_ms"),
+            ("job.transaction.queued", "max_abs_increase_us"),
+        ):
+            with self.subTest(group=group):
+                self.setUp()
+                self.edit_json(THRESHOLDS, lambda d: d["overrides"].pop(group))
+                code, out = self.run_checker()
+                self.assertEqual(code, 1, out)
+                self.assertIn("(rule B)", out)
+                self.assertIn(suffix, out)
+                self.assertNotIn("max_abs_increase =", out)
+
+    def test_non_numeric_threshold_is_reported_not_crashed(self):
+        """A hand-edited bound that is a string must be named, not raise."""
+        self.edit_json(
+            THRESHOLDS,
+            lambda d: d["overrides"]["span.ledger.build"]["p99"].update(
+                max_abs_increase_ms="5.5"
+            ),
+        )
+        code, out = self.run_checker()
+        self.assertEqual(code, 1, out)
+        self.assertIn("not a number", out)
+        self.assertNotIn("Traceback", out)
+
     def test_rule_c_flags_rounded_bound(self):
         """A bound rounded for readability is still not the derived bound."""
         _, exact = self.gated("span.tx.process.p99")