diff --git a/.github/scripts/telemetry/test_check_regression_bounds.py b/.github/scripts/telemetry/test_check_regression_bounds.py index 44222e6576..896829d03b 100644 --- a/.github/scripts/telemetry/test_check_regression_bounds.py +++ b/.github/scripts/telemetry/test_check_regression_bounds.py @@ -247,7 +247,7 @@ class TestRules(CheckerCase): """A hand-edited bound that is a string must be named, not raise.""" self.edit_json( THRESHOLDS, - lambda d: d["overrides"]["span.ledger.build"]["p99"].update( + lambda d: d["overrides"]["span.ledger.build"]["p95"].update( max_abs_increase_ms="5.5" ), ) diff --git a/docker/telemetry/workload/baselines/baseline-timings.json b/docker/telemetry/workload/baselines/baseline-timings.json index f7060cbbd5..1ce5c004a4 100644 --- a/docker/telemetry/workload/baselines/baseline-timings.json +++ b/docker/telemetry/workload/baselines/baseline-timings.json @@ -1,89 +1,87 @@ { - "captured_at": "2026-08-26T11:54:41Z", - "git_sha": "8418d474a7a63aa981a4854aa94388a556517b0d", + "captured_at": "2026-09-10T18:05:00Z", + "git_sha": "a0385c53cb8f1e9ab77379b5584367a92d0a9014", "metrics": { "job.acceptLedger.queued.p95": { "unit": "us", - "value": 166.13636363636323 + "value": 95.58333333333331 }, "job.acceptLedger.running.p95": { "unit": "us", - "value": 6142.857142857149 + "value": 15967.741935483866 }, "job.transaction.queued.p95": { "unit": "us", - "value": 426.19047619047586 + "value": 403.74177631578937 }, "job.transaction.running.p95": { "unit": "us", - "value": 599.9999999999986 + "value": 376.7512077294688 }, "span.consensus.accept.p50": { "unit": "ms", - "value": 0.5287356321839081 + "value": 1.4363636363636365 }, "span.consensus.accept.p95": { "unit": "ms", - "value": 8.969696969696969 + "value": 8.811111111111114 }, "span.consensus.accept.p99": { "unit": "ms", - "value": 20.800000000000026 + "value": 20.928571428571445 }, "span.consensus.ledger_close.p95": { "unit": "ms", - "value": 0.7829999999999997 + "value": 0.6749999999999998 }, "span.consensus.ledger_close.p99": { "unit": "ms", - "value": 2.0299999999999896 + "value": 3.049999999999991 }, "span.ledger.build.p95": { "unit": "ms", - "value": 4.53333333333333 - }, - "span.ledger.build.p99": { - "unit": "ms", - "value": 9.109090909090913 + "value": 4.7714285714285705 }, "span.ledger.validate.p50": { "unit": "ms", - "value": 0.06471894002114831 + "value": 0.059761904761904766 }, "span.rpc.ws_message.p50": { "unit": "ms", - "value": 0.16003451676528602 + "value": 0.16282758747645082 }, "span.rpc.ws_message.p95": { "unit": "ms", - "value": 0.6977397260273974 + "value": 0.8122454466253443 }, "span.rpc.ws_message.p99": { "unit": "ms", - "value": 0.9757123287671235 + "value": 0.9872660098522166 }, "span.tx.apply.p95": { "unit": "ms", - "value": 3.524999999999999 + "value": 4.639716312056738 }, "span.tx.apply.p99": { "unit": "ms", - "value": 5.066666666666704 + "value": 4.9733333333333345 }, "span.tx.process.p50": { "unit": "ms", - "value": 0.20062219789579117 + "value": 0.21390674968918655 }, "span.tx.process.p95": { "unit": "ms", - "value": 0.6100467289719625 + "value": 0.49949970576841685 }, "span.tx.process.p99": { "unit": "ms", - "value": 2.758787878787877 + "value": 0.9939697679594013 } }, "profile": "full-validation", "schema_version": 1, + "source_runs": [34495527952, 34505215266, 34507425933], + "statistic": "median of three clean full-validation runs", "window": "3m" } diff --git a/docker/telemetry/workload/regression-metrics.json b/docker/telemetry/workload/regression-metrics.json index 70508085ad..f68c3f027c 100644 --- a/docker/telemetry/workload/regression-metrics.json +++ b/docker/telemetry/workload/regression-metrics.json @@ -1,21 +1,30 @@ { "_description": "Metric surface for the OTel-driven regression gate. Each entry names a metric, the quantiles to capture, and how to query Prometheus. The comparator compares current run against baseline-timings.json under these exact keys.", - "_key_format": "{category}.{name}.p{quantile} (e.g. span.tx.process.p99, job.transaction.queued.p95). Only the categories defined below are captured; there is no rpc_methods group, so no rpc.* key is produced or gated (FU-4).", - "_excluded_spans": "rpc.process is deliberately absent from spans.names. It is created only in ServerHandler::processRequest() on the HTTP/JSON-RPC path, which the workload load generators, being WebSocket-only, never reach, so its quantiles were captured as null every run and could never gate. (The harness shell scripts do issue a few HTTP JSON-RPC health polls, far too few to produce a meaningful quantile.) See baselines/README.md.", - "_excluded_ledger_store": "ledger.store is deliberately absent from spans.names too, for a different reason: it is below the ladder's resolution. The 2026-08-24 capture returned p50/p95/p99 of exactly 0.005/0.0095/0.0099 ms, which is 0.5/0.95/0.99 x the ladder's first edge of 0.01 ms — the signature of every sample landing in the first bucket, so the numbers are interpolation arithmetic on the bucket floor rather than latencies. That is physically plausible: LedgerMaster.cpp:470 wraps an in-memory ledgerHistory_.insert, which completes in single-digit microseconds. While all mass stays under 10 us the reported quantile cannot move materially, so NO absolute bound can gate it — every ledger.store slowing from 2 us to 9 us, 4.5x, leaves the reported value unchanged. Three keys that read as covered but cannot fire are worse than no keys (the same argument that excluded rpc.process), so they were removed rather than left in with a bound that looks derived. Restoring the key needs sub-10us edges on the collector's spanmetrics ladder (for example 0.001ms and 0.005ms) plus the matching entries in HistogramBuckets.h — that is the ladder's branch, not this file. ledger.store presence is still asserted by expected_spans.json and docker/telemetry/integration-test.sh, and its rate is still on the ledger-operations dashboard; only the latency gate drops it.", + "_excluded_ledger_store": "ledger.store is deliberately absent from spans.names too, for a different reason: it is below the ladder's resolution. The 2026-08-24 capture returned p50/p95/p99 of exactly 0.005/0.0095/0.0099 ms, which is 0.5/0.95/0.99 x the ladder's first edge of 0.01 ms \u2014 the signature of every sample landing in the first bucket, so the numbers are interpolation arithmetic on the bucket floor rather than latencies. That is physically plausible: LedgerMaster.cpp:470 wraps an in-memory ledgerHistory_.insert, which completes in single-digit microseconds. While all mass stays under 10 us the reported quantile cannot move materially, so NO absolute bound can gate it \u2014 every ledger.store slowing from 2 us to 9 us, 4.5x, leaves the reported value unchanged. Three keys that read as covered but cannot fire are worse than no keys (the same argument that excluded rpc.process), so they were removed rather than left in with a bound that looks derived. Restoring the key needs sub-10us edges on the collector's spanmetrics ladder (for example 0.001ms and 0.005ms) plus the matching entries in HistogramBuckets.h \u2014 that is the ladder's branch, not this file. ledger.store presence is still asserted by expected_spans.json and docker/telemetry/integration-test.sh, and its rate is still on the ledger-operations dashboard; only the latency gate drops it.", "_excluded_quantiles": "A THIRD KIND OF EXCLUSION, and the only one that deleting a name cannot express. spans.names lists span NAMES while _quantiles is shared across all of them, so the declared surface is the names x quantiles product and dropping ONE quantile of ONE span needs a subtraction. excluded_keys below is that subtraction: a flat {category}.{name}.p{quantile} key, exactly as _key_format defines it, mapped to the reason it is not gated. It can only ever remove a key, never add one, so a typo cannot silently start gating something new -- and check_regression_bounds.py rule F rejects an entry that would not otherwise be declared, an entry with an empty reason, and an entry that still carries a threshold override or a baseline value, so the exclusion cannot rot into dead config. Both prom_queries.py (which builds the capture plan) and check_regression_bounds.py (rule A) subtract it, so an excluded key is not queried, never reaches timings.json, and is not expected in the baseline. NOTHING ELSE CHANGES: the quantile is still computable from Prometheus with the _query_template above, the span is still asserted by expected_spans.json, and its rate is still on the ledger-operations dashboard. Only the latency gate drops it.", "_excluded_shape": "ALL FIVE ENTRIES BELOW SHARE ONE SHAPE, and it is worth naming because it will recur: the observed maximum across CI runs exceeds (baseline + bound), so an ordinary run clears the trip point with nothing having regressed. Two mechanisms produce that, and both are visible here. (1) A baseline that lands in the ladder's LOW buckets gets a tiny derived bound, because the bound IS the distance to the next edge up -- span.tx.apply.p50 at 0.0060 ms sits in the first bucket (0, 0.01] and gets 0.0440 ms of headroom, against a metric that has been measured at 2.3378 ms. (2) A spread so large that no bucket of headroom could absorb it -- span.ledger.validate.p99's 66.8x range reaches 25.8750 ms against a 10 ms trip point even though its bound is a comparatively generous 8.94 ms. The first mechanism is the one that excludes three keys here, and it is a property of WHERE THE CAPTURED RUN LANDED rather than of the metric: the same span.tx.apply.p50 has read 0.7917 ms, mid-distribution, where the identical rule produces a 4.21 ms bound that absorbs the whole range. Whether the gate functioned was therefore decided by luck of the draw. THE FOLLOW-UP THAT WOULD RESTORE COVERAGE, stated so it is not left implied: a baseline captured from a SINGLE run cannot support these keys, because one sample carries no information about spread and the bound is derived from that one sample alone. What would let them be gated again is a multi-run baseline -- or a spread measurement captured alongside the baseline, so a bound can be sized against observed variance instead of against the ladder only. That is not implemented; it is the design change these five exclusions are waiting on. Until then, do NOT re-gate any of them by re-baselining until a run happens to land favourably, which is the failure this note exists to prevent.", + "_excluded_spans": "rpc.process is deliberately absent from spans.names. It is created only in ServerHandler::processRequest() on the HTTP/JSON-RPC path, which the workload load generators, being WebSocket-only, never reach, so its quantiles were captured as null every run and could never gate. (The harness shell scripts do issue a few HTTP JSON-RPC health polls, far too few to produce a meaningful quantile.) See baselines/README.md.", + "_key_format": "{category}.{name}.p{quantile} (e.g. span.tx.process.p99, job.transaction.queued.p95). Only the categories defined below are captured; there is no rpc_methods group, so no rpc.* key is produced or gated (FU-4).", "excluded_keys": { "span.consensus.ledger_close.p50": "Run-to-run variance exceeds the bound this ladder can derive, the same limit as the ledger.validate pair and the same mechanism as the two sibling p50 keys excluded alongside it. Baseline 0.0387 ms sits in the low bucket (0.01, 0.05], so hi_next is 0.1 ms and the derived bound is 0.0613 ms -- a 2.58x trip point. Measured across three CI runs the value spans 0.0387 to 0.2377 ms, a 6.1x spread (5.9x over four runs), and run 32867433073 read 0.2377 ms, 2.38x the trip point, on the SAME post-path-finding-removal workload as this baseline. So a healthy run reddens CI. This is a variance limit, not a defect and not a missing bound: widening is unavailable, because a bound tolerating 0.2377 ms would reach past the 0.25 ms edge and gate almost nothing. Do NOT re-gate by widening, and do NOT re-baseline until a run lands higher -- see _excluded_shape.", "span.ledger.build.p50": "The same mechanism as span.consensus.ledger_close.p50, one bucket up. Baseline 0.1151 ms sits in (0.1, 0.25], so hi_next is 0.5 ms and the bound is 0.3849 ms -- a 4.34x trip point. Across three CI runs the value spans 0.1151 to 2.3826 ms, a 20.7x spread (25.3x over four runs), and the observed maximum is 4.77x the trip point. Note what the previous baseline hid: at 1.0612 ms the same rule gave a 8.94 ms bound and a 10 ms trip point, which absorbed the entire range, so this key read as gated purely because that capture landed mid-distribution. Ledger construction is the hot path this gate most wants to guard, which makes the loss real and worth fixing properly -- with a baseline that carries spread information, not with a wider bound.", - "span.tx.apply.p50": "The most extreme case of the low-bucket mechanism, and the clearest evidence that a single-run baseline cannot size a bound for these keys. Baseline 0.00597 ms lands in the ladder's FIRST bucket (0, 0.01], so hi_next is 0.05 ms and the bound is 0.0440 ms. Across three CI runs the value spans 0.00597 to 2.3378 ms, a 391.8x spread (364x over four runs), putting the observed maximum at 46.76x its trip point -- by far the worst of the five. The previous baseline read 0.7917 ms for the same key on the same workload, a 132x difference between two runs, and at that value the identical rule produced a 4.21 ms bound whose 5 ms trip point absorbed the full range. Nothing about the metric changed between those two captures; only where the sampled run fell in its own distribution did. Separately, a baseline inside the first bucket means the reported figure is interpolation across that bucket and tracks the FRACTION of applies finishing under 10 us rather than a latency, which is the ledger.store problem in embryo -- so restoring this key needs a finer low-end ladder as well as a spread-aware baseline. Rule E does not flag it because the value is not quantile x first_edge exactly.", + "span.ledger.build.p99": "Run-to-run variance exceeds the bound this ladder can derive, measured on the first three runs after the account-funding race was fixed. Baseline 8.944 ms (median of CI runs 34495527952, 34505215266, 34507425933) sits in (5, 10], so hi_next is 25 ms, the bound is 16.056 ms and the trip point is 25 ms. Across those three runs the value read 29.000, 7.060 and 8.944 ms -- a 4.11x spread, and the maximum is 1.16x the trip point, so a healthy run reddens CI. Run 34495527952 is that run: it turned the workload gate red on this key alone while all 200 span and metric checks passed and every phase reported 0 errors. Two corroborating details. First, its 29.000 ms reading is shared to the last digit with span.consensus.accept.p99 in the same run, which is the signature of histogram_quantile interpolating a thin tail inside one bucket rather than of ledger construction slowing down. Second, the previous baseline hid this: at 9.109 ms the same rule also gave a 25 ms trip point, and the key read as gated only because both the capture and the comparison runs happened to land low -- the 2026-08-26 capture predated the funding fix, so its phases submitted little or no traffic. Widening is not available: a bound tolerating 29.000 ms would be 20.06 ms and reach into the (25, 50] bucket, restoring exactly the single-crossing false positive the derivation rule exists to remove. Ledger construction is not left unguarded -- span.ledger.build.p95 stays gated and sits at 0.48 of its trip point. Do NOT re-gate this key by widening the bound; it needs a spread-aware baseline, or a finer ladder edge between 10 ms and 25 ms.", "span.ledger.validate.p95": "Run-to-run variance is larger than the bound this ladder can derive. Measured across four CI runs the value spans 0.1281 to 0.7500 ms, a 5.9x spread, against a baseline of 0.2404 ms whose trip point is the next ladder edge at 0.5 ms -- so an ordinary run clears the trip point with nothing having regressed. Run 32867433073 read 0.7500 ms, +212%, and turned CI red. The derived bound models QUANTIZATION noise only (hi_next - baseline is one bucket of headroom); the dominant noise term for this span is peer-validation arrival timing in a 5-node cluster, and that term was never measured before the key was gated. Widening is not available: a bound that tolerated 0.7500 ms would reach past the 1 ms edge and leave the key gating nothing. This is a variance limit, not a defect and not a missing bound -- do NOT re-gate it by widening.", - "span.ledger.validate.p99": "The same mechanism as p95, two orders of magnitude worse. Across the same four runs the value spans 0.3875 to 25.8750 ms, a 66.8x spread, against a baseline of 1.0600 ms and a 10 ms trip point; run 32862589645 read 25.8750 ms, +2341%. The span opens only once a quorum-completing validation arrives (LedgerMaster.cpp:1003, inside checkAccept, past the tvc < minVal early return) and wraps the promotion work that follows -- setValidated, setFull, setValidLedger, pendSaveValidated -- so its duration tracks peer-validation arrival timing and what promotion then triggers. One slow consensus round therefore dominates the tail of a 3m rate window, and which round that is differs every run. A bound tolerating 25.8750 ms would be ~24.8 ms against a 1.0600 ms baseline, which gates nothing at all. Note that the two CI failures landed on DIFFERENT quantiles in different runs while the other quantile stayed well inside its bound in the same run: that asymmetry is the signature of variance, not of a regression." + "span.ledger.validate.p99": "The same mechanism as p95, two orders of magnitude worse. Across the same four runs the value spans 0.3875 to 25.8750 ms, a 66.8x spread, against a baseline of 1.0600 ms and a 10 ms trip point; run 32862589645 read 25.8750 ms, +2341%. The span opens only once a quorum-completing validation arrives (LedgerMaster.cpp:1003, inside checkAccept, past the tvc < minVal early return) and wraps the promotion work that follows -- setValidated, setFull, setValidLedger, pendSaveValidated -- so its duration tracks peer-validation arrival timing and what promotion then triggers. One slow consensus round therefore dominates the tail of a 3m rate window, and which round that is differs every run. A bound tolerating 25.8750 ms would be ~24.8 ms against a 1.0600 ms baseline, which gates nothing at all. Note that the two CI failures landed on DIFFERENT quantiles in different runs while the other quantile stayed well inside its bound in the same run: that asymmetry is the signature of variance, not of a regression.", + "span.tx.apply.p50": "The most extreme case of the low-bucket mechanism, and the clearest evidence that a single-run baseline cannot size a bound for these keys. Baseline 0.00597 ms lands in the ladder's FIRST bucket (0, 0.01], so hi_next is 0.05 ms and the bound is 0.0440 ms. Across three CI runs the value spans 0.00597 to 2.3378 ms, a 391.8x spread (364x over four runs), putting the observed maximum at 46.76x its trip point -- by far the worst of the five. The previous baseline read 0.7917 ms for the same key on the same workload, a 132x difference between two runs, and at that value the identical rule produced a 4.21 ms bound whose 5 ms trip point absorbed the full range. Nothing about the metric changed between those two captures; only where the sampled run fell in its own distribution did. Separately, a baseline inside the first bucket means the reported figure is interpolation across that bucket and tracks the FRACTION of applies finishing under 10 us rather than a latency, which is the ledger.store problem in embryo -- so restoring this key needs a finer low-end ladder as well as a spread-aware baseline. Rule E does not flag it because the value is not quantile x first_edge exactly." + }, + "job_queue": { + "_phases": ["queued", "running"], + "_quantiles": [0.95], + "_queued_template": "histogram_quantile({quantile}, sum by (le) (rate(job_queued_us_bucket{job_type=\"{name}\"}[{window}])))", + "_running_template": "histogram_quantile({quantile}, sum by (le) (rate(job_running_us_bucket{job_type=\"{name}\"}[{window}])))", + "_unit": "us", + "names": ["transaction", "acceptLedger"] }, "spans": { + "_quantiles": [0.5, 0.95, 0.99], "_query_template": "histogram_quantile({quantile}, sum by (le) (rate(span_duration_milliseconds_bucket{span_name=\"{name}\"}[{window}])))", "_unit": "ms", - "_quantiles": [0.5, 0.95, 0.99], "names": [ "rpc.ws_message", "tx.process", @@ -25,13 +34,5 @@ "consensus.ledger_close", "consensus.accept" ] - }, - "job_queue": { - "_queued_template": "histogram_quantile({quantile}, sum by (le) (rate(job_queued_us_bucket{job_type=\"{name}\"}[{window}])))", - "_running_template": "histogram_quantile({quantile}, sum by (le) (rate(job_running_us_bucket{job_type=\"{name}\"}[{window}])))", - "_unit": "us", - "_quantiles": [0.95], - "_phases": ["queued", "running"], - "names": ["transaction", "acceptLedger"] } } diff --git a/docker/telemetry/workload/regression-thresholds.json b/docker/telemetry/workload/regression-thresholds.json index ff50363116..76c2b9850c 100644 --- a/docker/telemetry/workload/regression-thresholds.json +++ b/docker/telemetry/workload/regression-thresholds.json @@ -1,146 +1,142 @@ { - "_description": "Per-metric regression thresholds. A metric regresses when current - baseline exceeds BOTH the percentage and absolute bounds (AND, not OR \u2014 this tolerates small-value noise). Defaults apply unless a per-metric override exists.", + "_absolute_bound_derivation": "HOW EVERY max_abs_increase_* NUMBER BELOW WAS OBTAINED. Rule: locate the baseline value in the half-open bucket (lo, hi] of its own ladder, take hi_next = the next edge above hi, and set the bound to (hi_next - baseline). The trip point is therefore exactly hi_next: the gate fires only when the reported value EXCEEDS the top of the bucket above the baseline's own bucket. WHY THAT AND NOT A MULTIPLE OF THE BUCKET WIDTH: histogram_quantile returns a value interpolated inside whichever bucket the true quantile falls in, so a reading taken while the true quantile sits anywhere in the baseline's bucket OR anywhere in the one immediately above is at most hi_next and cannot fire. Firing requires the true quantile to have moved at least two buckets up. A multiple of the ENCLOSING width cannot deliver that, because once the quantile crosses hi the interpolation happens across the NEXT bucket, which on this ladder is up to 8x wider \u2014 (0.5,1] has width 0.5 and (1,5] has width 4 \u2014 so the reading's excursion is not bounded by any multiple of the enclosing width. Worked example: span.tx.process.p99 has baseline 2.7588ms in bucket (1, 5], hi_next = 10, so its bound is 7.2412ms and the gate fires only above 10ms. Bounds are stored as exact doubles rather than rounded figures so that rounding cannot break the guarantee and so check_regression_bounds.py can assert each one against the ladder to within a 1e-12 relative tolerance -- tight enough that a bound rounded for readability, such as 7.2412 for 7.241212121212123, is rejected; _derivation_table below shows the arithmetic for each one. Measured over the committed baseline this rule yields a detection floor of 2.21x to 16.28x of baseline, per key. WHAT THIS RULE DOES NOT COVER, AND THE ONE CHECK TO RUN BEFORE GATING ANY KEY: hi_next - baseline is derived from the LADDER, so it budgets for QUANTIZATION noise -- one bucket of interpolation headroom -- and for nothing else. It knows nothing about how much the metric itself moves between runs on identical code. Where run-to-run workload variance is the larger term the bound is simply the wrong size, and the gate reddens on a healthy run. So before adding a key here, capture it over several runs and check its OBSERVED MAXIMUM against its trip point (baseline + bound); gate it only if the observed maximum stays below that trip point with margin. Spread on its own proves nothing -- it is spread RELATIVE TO THE TRIP POINT that decides, and a baseline that lands at the LOW end of a metric's own range shrinks that trip point even though nothing about the metric changed. THREE KEYS FAILED THIS TEST ON THE 2026-08-26 BASELINE AND ARE NOW EXCLUDED, all of them p50: span.tx.apply.p50 (bound 0.0440ms, trips at 0.05ms, observed max 2.3378ms = 46.76x its trip point), span.ledger.build.p50 (bound 0.3849ms, trips at 0.5ms, observed max 2.3826ms = 4.77x) and span.consensus.ledger_close.p50 (bound 0.0613ms, trips at 0.1ms, observed max 0.2377ms = 2.38x). Their spreads across three runs are 391.8x, 20.7x and 6.1x. This is the general rule above being APPLIED, not a new exception: a key is gateable only when its run-to-run spread fits inside its bound, and these three do not. The evidence that settles it is span.tx.apply.p50's own history -- it read 0.7917ms in the previous baseline and 0.00597ms in this one, a 132x difference between two runs of the SAME workload. At the old value the identical rule produced a 4.21ms bound whose 5ms trip point absorbed the whole range; at the new one it produces 0.0440ms and cannot. Whether the gate functioned was therefore decided by where in its distribution the captured run happened to land, which is not a threshold needing tuning but a key that cannot be gated from a single-run baseline at all. Before the exclusion, replaying the two preceding CI runs 32862589645 and 32867433073 against this baseline reported exactly those three and nothing else on BOTH runs, and 32867433073 carries the same post-path-finding-removal workload as the baseline itself -- so the movement was metric variance, not a workload difference. After it, both runs replay clean. The remaining 20 keys sit at or below 0.58 of their trip points, the worst being span.consensus.accept.p50. See _excluded_shape in regression-metrics.json for what all five excluded keys have in common and for the multi-run-baseline work that would let them be gated again. A key that fails this test is not fixed by widening its bound: see excluded_keys in regression-metrics.json. WHAT THIS REPLACED, IN TWO GENERATIONS: (1) a single flat pair of bounds (10ms for span p50/p95, 15ms for span p99, 20000us for job_queue p95) justified as 'roughly two bucket widths in the 5-25ms band where most span quantiles actually sit'. The 2026-08-24 capture falsifies that premise \u2014 18 of the 28 quantiles gated at that time sat below 1ms \u2014 so the absolute bound sat 1.15x to 2000x above the metric it guarded and, because the rule is an AND, the percentage bound could never carry a regression on its own; a 10x regression injected into each key in turn was caught on only 5 of 28, and a 100x regression injected into span.ledger.store.p95 produced 0 regressions and exit 0. (2) a first correction to 2 \u00d7 the ENCLOSING bucket width, which caught 10x on 28 of 28 but placed the trip point INSIDE the adjacent bucket -- and so left a single-crossing false positive reachable -- on 21 of the 25 keys gated at the time, 4 of them tripping on a tail-mass shift under 1.5% of samples. That is the assumption this rule removes. RE-DERIVE THESE NUMBERS whenever baseline-timings.json is refreshed or either ladder changes: a refreshed baseline can land in a different bucket, which changes hi_next. .github/scripts/telemetry/check_regression_bounds.py enforces the rule in CI so a stale bound cannot survive a baseline refresh. LIMITATION \u2014 WHICH KEYS ARE ONLY WEAKLY GUARDED: the guarantee costs sensitivity wherever the ladder is coarse, and the detection floor is hi_next/baseline, so a baseline sitting just above an edge is guarded loosely. job.acceptLedger.running.p95 (baseline 6142.86us, fires at 100000us, 16.28x) is NOT meaningfully guarded, and it is now the one key a 10x regression does NOT catch: measured, 10x reaches 61429us and passes, and the gate first fires at 16.28x. It sits just above the 5000us edge while hi_next is 100000us, two steps up. Its floor moved there in this refresh, from 5.74x, because its baseline fell 17428.57us to 6142.86us while hi_next stayed at 100000us -- it does NOT fire on any observed run, so it stays gated, but the weak floor is recorded here so it is visible rather than surprising. span.consensus.accept.p50 (9.46x), job.transaction.running.p95 (8.33x), span.tx.process.p95 (8.20x), span.rpc.ws_message.p95 (7.17x), span.consensus.ledger_close.p95 (6.39x) and span.rpc.ws_message.p99 (5.12x) are also weak. Four of the seven are limited by the 1ms\u21925ms step; the rest by 1000us\u21925000us (job.transaction.running.p95) and 25000us\u2192100000us (job.acceptLedger.running.p95). The fix is a 2ms edge (and ideally 3ms) in the collector's spanmetrics ladder plus the matching edges in kMillisecondBuckets, and 2000us plus 50000us edges in kMicrosecondBuckets \u2014 that work belongs to the branch that owns the ladders, not here. Until then do not read these keys as guarded. span.ledger.store is absent from the overrides below because it is excluded from the gated surface entirely: its quantiles are the ladder floor times the quantile, so no bound can gate it. See _excluded_ledger_store in regression-metrics.json. REFRESHED 2026-09-10 from the median of CI runs 34495527952, 34505215266 and 34507425933, the first three runs with the account-funding race fixed. The 2026-08-26 baseline predated that fix, so the phases that lost their traffic captured artificially low ledger and transaction timings; job.transaction.queued.p95 and job.transaction.running.p95 could not be captured at all. Applying the observed-maximum test to the refreshed numbers leaves 19 of 20 keys between 0.17 and 0.76 of their trip points, and disqualifies span.ledger.build.p99 -- see excluded_keys in regression-metrics.json. span.tx.process.p95 is the tightest survivor at 0.76 and is the key to re-measure first if the gate reddens again.", "_bucket_note": "SpanMetrics latency histograms use explicit buckets [0.01,0.05,0.1,0.25,0.5,1,5,10,25,50,100,250,500]ms then [1,2,3,4,5,10,30]s (20 edges; docker/telemetry/otel-collector-config.yaml is the authoritative list). Second-scale consensus spans have 2s/3s/4s boundaries, so their quantiles quantize to ~1s widths there \u2014 the ladder is NOT uniformly 2x-or-coarser, which matters for _percentage_bound_note. The native job_queue histograms are microsecond-valued on the ladder [1,2,5,10,25,50,100,250,500,1000,5000,25000,100000,500000]us then [1,5,10,30,60]s (19 edges; include/xrpl/telemetry/HistogramBuckets.h is authoritative). NOTE: BOTH ladders were re-cut, and a baseline captured before its own ladder changed is an interpolation artefact, not a latency. The job_queue floor moved 100us \u2192 1us. The span floor is 0.01ms; a span baseline captured against a 1ms floor is void below 1ms \u2014 a p95 reading 0.95ms there is 0.95 \u00d7 that 1ms first edge, not a measurement. Do not assume a surviving span baseline is unaffected by ladder work: every span quantile below 1ms is affected. Only the band from 1ms to 1s is safe: those edges are byte-identical across the two ladders. The re-cut also ADDED edges above 1s (2s/3s/4s/10s/30s), so a span whose quantiles land in the second-scale range \u2014 consensus.round ~3.9s, consensus.establish ~1.9s, the ledger.acquire tail \u2014 is distorted just as much, and any pre-2026-08-04 baseline for it is equally void. Do not read this note as licensing a stale second-scale baseline.", - "_absolute_bound_derivation": "HOW EVERY max_abs_increase_* NUMBER BELOW WAS OBTAINED. Rule: locate the baseline value in the half-open bucket (lo, hi] of its own ladder, take hi_next = the next edge above hi, and set the bound to (hi_next - baseline). The trip point is therefore exactly hi_next: the gate fires only when the reported value EXCEEDS the top of the bucket above the baseline's own bucket. WHY THAT AND NOT A MULTIPLE OF THE BUCKET WIDTH: histogram_quantile returns a value interpolated inside whichever bucket the true quantile falls in, so a reading taken while the true quantile sits anywhere in the baseline's bucket OR anywhere in the one immediately above is at most hi_next and cannot fire. Firing requires the true quantile to have moved at least two buckets up. A multiple of the ENCLOSING width cannot deliver that, because once the quantile crosses hi the interpolation happens across the NEXT bucket, which on this ladder is up to 8x wider \u2014 (0.5,1] has width 0.5 and (1,5] has width 4 \u2014 so the reading's excursion is not bounded by any multiple of the enclosing width. Worked example: span.tx.process.p99 has baseline 2.7588ms in bucket (1, 5], hi_next = 10, so its bound is 7.2412ms and the gate fires only above 10ms. Bounds are stored as exact doubles rather than rounded figures so that rounding cannot break the guarantee and so check_regression_bounds.py can assert each one against the ladder to within a 1e-12 relative tolerance -- tight enough that a bound rounded for readability, such as 7.2412 for 7.241212121212123, is rejected; _derivation_table below shows the arithmetic for each one. Measured over the committed baseline this rule yields a detection floor of 2.21x to 16.28x of baseline, per key. WHAT THIS RULE DOES NOT COVER, AND THE ONE CHECK TO RUN BEFORE GATING ANY KEY: hi_next - baseline is derived from the LADDER, so it budgets for QUANTIZATION noise -- one bucket of interpolation headroom -- and for nothing else. It knows nothing about how much the metric itself moves between runs on identical code. Where run-to-run workload variance is the larger term the bound is simply the wrong size, and the gate reddens on a healthy run. So before adding a key here, capture it over several runs and check its OBSERVED MAXIMUM against its trip point (baseline + bound); gate it only if the observed maximum stays below that trip point with margin. Spread on its own proves nothing -- it is spread RELATIVE TO THE TRIP POINT that decides, and a baseline that lands at the LOW end of a metric's own range shrinks that trip point even though nothing about the metric changed. THREE KEYS FAILED THIS TEST ON THE 2026-08-26 BASELINE AND ARE NOW EXCLUDED, all of them p50: span.tx.apply.p50 (bound 0.0440ms, trips at 0.05ms, observed max 2.3378ms = 46.76x its trip point), span.ledger.build.p50 (bound 0.3849ms, trips at 0.5ms, observed max 2.3826ms = 4.77x) and span.consensus.ledger_close.p50 (bound 0.0613ms, trips at 0.1ms, observed max 0.2377ms = 2.38x). Their spreads across three runs are 391.8x, 20.7x and 6.1x. This is the general rule above being APPLIED, not a new exception: a key is gateable only when its run-to-run spread fits inside its bound, and these three do not. The evidence that settles it is span.tx.apply.p50's own history -- it read 0.7917ms in the previous baseline and 0.00597ms in this one, a 132x difference between two runs of the SAME workload. At the old value the identical rule produced a 4.21ms bound whose 5ms trip point absorbed the whole range; at the new one it produces 0.0440ms and cannot. Whether the gate functioned was therefore decided by where in its distribution the captured run happened to land, which is not a threshold needing tuning but a key that cannot be gated from a single-run baseline at all. Before the exclusion, replaying the two preceding CI runs 32862589645 and 32867433073 against this baseline reported exactly those three and nothing else on BOTH runs, and 32867433073 carries the same post-path-finding-removal workload as the baseline itself -- so the movement was metric variance, not a workload difference. After it, both runs replay clean. The remaining 20 keys sit at or below 0.58 of their trip points, the worst being span.consensus.accept.p50. See _excluded_shape in regression-metrics.json for what all five excluded keys have in common and for the multi-run-baseline work that would let them be gated again. A key that fails this test is not fixed by widening its bound: see excluded_keys in regression-metrics.json. WHAT THIS REPLACED, IN TWO GENERATIONS: (1) a single flat pair of bounds (10ms for span p50/p95, 15ms for span p99, 20000us for job_queue p95) justified as 'roughly two bucket widths in the 5-25ms band where most span quantiles actually sit'. The 2026-08-24 capture falsifies that premise \u2014 18 of the 28 quantiles gated at that time sat below 1ms \u2014 so the absolute bound sat 1.15x to 2000x above the metric it guarded and, because the rule is an AND, the percentage bound could never carry a regression on its own; a 10x regression injected into each key in turn was caught on only 5 of 28, and a 100x regression injected into span.ledger.store.p95 produced 0 regressions and exit 0. (2) a first correction to 2 \u00d7 the ENCLOSING bucket width, which caught 10x on 28 of 28 but placed the trip point INSIDE the adjacent bucket -- and so left a single-crossing false positive reachable -- on 21 of the 25 keys gated at the time, 4 of them tripping on a tail-mass shift under 1.5% of samples. That is the assumption this rule removes. RE-DERIVE THESE NUMBERS whenever baseline-timings.json is refreshed or either ladder changes: a refreshed baseline can land in a different bucket, which changes hi_next. .github/scripts/telemetry/check_regression_bounds.py enforces the rule in CI so a stale bound cannot survive a baseline refresh. LIMITATION \u2014 WHICH KEYS ARE ONLY WEAKLY GUARDED: the guarantee costs sensitivity wherever the ladder is coarse, and the detection floor is hi_next/baseline, so a baseline sitting just above an edge is guarded loosely. job.acceptLedger.running.p95 (baseline 6142.86us, fires at 100000us, 16.28x) is NOT meaningfully guarded, and it is now the one key a 10x regression does NOT catch: measured, 10x reaches 61429us and passes, and the gate first fires at 16.28x. It sits just above the 5000us edge while hi_next is 100000us, two steps up. Its floor moved there in this refresh, from 5.74x, because its baseline fell 17428.57us to 6142.86us while hi_next stayed at 100000us -- it does NOT fire on any observed run, so it stays gated, but the weak floor is recorded here so it is visible rather than surprising. span.consensus.accept.p50 (9.46x), job.transaction.running.p95 (8.33x), span.tx.process.p95 (8.20x), span.rpc.ws_message.p95 (7.17x), span.consensus.ledger_close.p95 (6.39x) and span.rpc.ws_message.p99 (5.12x) are also weak. Four of the seven are limited by the 1ms\u21925ms step; the rest by 1000us\u21925000us (job.transaction.running.p95) and 25000us\u2192100000us (job.acceptLedger.running.p95). The fix is a 2ms edge (and ideally 3ms) in the collector's spanmetrics ladder plus the matching edges in kMillisecondBuckets, and 2000us plus 50000us edges in kMicrosecondBuckets \u2014 that work belongs to the branch that owns the ladders, not here. Until then do not read these keys as guarded. span.ledger.store is absent from the overrides below because it is excluded from the gated surface entirely: its quantiles are the ladder floor times the quantile, so no bound can gate it. See _excluded_ledger_store in regression-metrics.json.", - "_percentage_bound_note": "For every key gated today the absolute bound is the binding half of the AND and the percentage bound never decides the outcome: measured, (bound / baseline) ranges from 121% (span.ledger.build.p95) to 1528% (job.acceptLedger.running.p95), all above the 50% and 5% percentage bounds configured here, and the minimum trip multiple of all 20 keys is set by the absolute bound. THIS IS NOT A GENERAL GUARANTEE. Do not reason from 'every step of both ladders is at least a factor of 2' -- that premise is false. The span ladder breaks it three times at the top: 2s->3s is 1.5x, 3s->4s is 1.33x, 4s->5s is 1.25x, so second-scale consensus quantiles quantize to ~1s widths there. Because the bound is (hi_next - baseline), a baseline between about 2667ms and 3000ms, or between about 3334ms and 4000ms, gets an absolute bound worth less than 50% of itself and the PERCENTAGE bound becomes the operative one -- at which point the metric fires on a 50% move that is smaller than one bucket width, and the single-crossing guarantee in _absolute_bound_derivation is lost. That band is not hypothetical: the collector config names consensus.round (~3.9s) as a reason those edges exist, and 3900ms sits in the second sub-band with an absolute bound of 5000 - 3900 = 1100, only 28.2% of baseline. Whoever gates a key whose baseline lands in either sub-band MUST lower its max_pct_increase below (bound / baseline) for that key, or state explicitly that the metric is percentage-gated and the bucket guarantee does not hold for it. check_regression_bounds.py enforces this as rule D so the trap cannot be walked into silently. The percentage entries are required and still meaningful regardless: compare_to_baseline.py treats a missing max_pct_increase as 'no threshold configured' and would stop gating the metric entirely; they record the intended relative tolerance (consensus spans 5%, everything else 50%); and they are the operative bound on the defaults path (see _defaults_note).", "_defaults_note": "A MISSING OVERRIDE IS DETECTED BY CI, NOT BY THESE DEFAULTS. .github/scripts/telemetry/check_regression_bounds.py fails the build at lint time, naming the key and the exact value its bound should have, before the workload ever runs. That is the mechanism; the defaults below are only a runtime backstop for the case where that check is bypassed. The defaults carry the FLOOR of each ladder as their absolute bound \u2014 0.01ms for spans, 1us for job_queue \u2014 deliberately too small to bind for any real metric, which leaves max_pct_increase (50%) as the operative bound on this path. Measured: a metric with no override and a baseline of 3900ms passes at +49% and fires at +51%; a job metric with a baseline of 5000us behaves the same. The backstop is honestly imperfect and should not be oversold. At 50% relative it CAN false-fire: a metric whose baseline is 1.06ms inside the 4ms-wide (1,5] bucket fires on a single-bucket-width move (measured: 1.06 \u2192 5.06ms, +377%, regressed). That false fire is NOT to be read as 'the intended signal that the override is missing' \u2014 CI prints REGRESSION and a reader cannot tell it from a real one, and rejecting a tighter alternative for exactly that cries-wolf risk while shipping it here would be inconsistent. The check is what makes the signal legible. The backstop is kept only because a metric silently not gated at all is the worse of the two failures.", "_derivation_table": { "_format": "override key: in -> hi_next - baseline = ", - "job.acceptLedger.queued": "p95 166.13636363636323 in (100,250] -> hi_next 500 - baseline = 333.8636363636368", - "job.acceptLedger.running": "p95 6142.857142857149 in (5000,25000] -> hi_next 100000 - baseline = 93857.14285714286", - "job.transaction.queued": "p95 426.19047619047586 in (250,500] -> hi_next 1000 - baseline = 573.8095238095241", - "job.transaction.running": "p95 599.9999999999986 in (500,1000] -> hi_next 5000 - baseline = 4400.000000000002", - "span.consensus.accept": "p50 0.5287356321839081 in (0.5,1] -> hi_next 5 - baseline = 4.471264367816092 | p95 8.969696969696969 in (5,10] -> hi_next 25 - baseline = 16.03030303030303 | p99 20.800000000000026 in (10,25] -> hi_next 50 - baseline = 29.199999999999974", - "span.consensus.ledger_close": "p95 0.7829999999999997 in (0.5,1] -> hi_next 5 - baseline = 4.2170000000000005 | p99 2.0299999999999896 in (1,5] -> hi_next 10 - baseline = 7.97000000000001 (p50 is NOT gated -- see excluded_keys in regression-metrics.json)", - "span.ledger.build": "p95 4.53333333333333 in (1,5] -> hi_next 10 - baseline = 5.46666666666667 | p99 9.109090909090913 in (5,10] -> hi_next 25 - baseline = 15.890909090909087 (p50 is NOT gated -- see excluded_keys in regression-metrics.json)", - "span.ledger.validate": "p50 0.06471894002114831 in (0.05,0.1] -> hi_next 0.25 - baseline = 0.1852810599788517 (p95 and p99 are NOT gated -- see excluded_keys in regression-metrics.json: their run-to-run spread, 5.9x and 66.8x over four CI runs, exceeds any bound this rule can derive)", - "span.rpc.ws_message": "p50 0.16003451676528602 in (0.1,0.25] -> hi_next 0.5 - baseline = 0.339965483234714 | p95 0.6977397260273974 in (0.5,1] -> hi_next 5 - baseline = 4.302260273972602 | p99 0.9757123287671235 in (0.5,1] -> hi_next 5 - baseline = 4.024287671232877", - "span.tx.apply": "p95 3.524999999999999 in (1,5] -> hi_next 10 - baseline = 6.475000000000001 | p99 5.066666666666704 in (5,10] -> hi_next 25 - baseline = 19.933333333333294 (p50 is NOT gated -- see excluded_keys in regression-metrics.json)", - "span.tx.process": "p50 0.20062219789579117 in (0.1,0.25] -> hi_next 0.5 - baseline = 0.29937780210420883 | p95 0.6100467289719625 in (0.5,1] -> hi_next 5 - baseline = 4.389953271028038 | p99 2.758787878787877 in (1,5] -> hi_next 10 - baseline = 7.241212121212123" + "job.acceptLedger.queued": "p95 95.58333333333331 in (50,100] -> hi_next 250 - baseline = 154.41666666666669", + "job.acceptLedger.running": "p95 15967.741935483866 in (5000,25000] -> hi_next 100000 - baseline = 84032.25806451614", + "job.transaction.queued": "p95 403.74177631578937 in (250,500] -> hi_next 1000 - baseline = 596.2582236842106", + "job.transaction.running": "p95 376.7512077294688 in (250,500] -> hi_next 1000 - baseline = 623.2487922705312", + "span.consensus.accept": "p50 1.4363636363636365 in (1,5] -> hi_next 10 - baseline = 8.563636363636363 | p95 8.811111111111114 in (5,10] -> hi_next 25 - baseline = 16.188888888888886 | p99 20.928571428571445 in (10,25] -> hi_next 50 - baseline = 29.071428571428555", + "span.consensus.ledger_close": "p95 0.6749999999999998 in (0.5,1] -> hi_next 5 - baseline = 4.325 | p99 3.049999999999991 in (1,5] -> hi_next 10 - baseline = 6.950000000000009", + "span.ledger.build": "p95 4.7714285714285705 in (1,5] -> hi_next 10 - baseline = 5.2285714285714295", + "span.ledger.validate": "p50 0.059761904761904766 in (0.05,0.1] -> hi_next 0.25 - baseline = 0.19023809523809523", + "span.rpc.ws_message": "p50 0.16282758747645082 in (0.1,0.25] -> hi_next 0.5 - baseline = 0.3371724125235492 | p95 0.8122454466253443 in (0.5,1] -> hi_next 5 - baseline = 4.187754553374655 | p99 0.9872660098522166 in (0.5,1] -> hi_next 5 - baseline = 4.012733990147783", + "span.tx.apply": "p95 4.639716312056738 in (1,5] -> hi_next 10 - baseline = 5.360283687943262 | p99 4.9733333333333345 in (1,5] -> hi_next 10 - baseline = 5.0266666666666655", + "span.tx.process": "p50 0.21390674968918655 in (0.1,0.25] -> hi_next 0.5 - baseline = 0.28609325031081345 | p95 0.49949970576841685 in (0.25,0.5] -> hi_next 1 - baseline = 0.5005002942315832 | p99 0.9939697679594013 in (0.5,1] -> hi_next 5 - baseline = 4.006030232040599" }, + "_description": "Per-metric regression thresholds. A metric regresses when current - baseline exceeds BOTH the percentage and absolute bounds (AND, not OR \u2014 this tolerates small-value noise). Defaults apply unless a per-metric override exists.", + "_percentage_bound_note": "For every key gated today the absolute bound is the binding half of the AND and the percentage bound never decides the outcome: measured, (bound / baseline) ranges from 121% (span.ledger.build.p95) to 1528% (job.acceptLedger.running.p95), all above the 50% and 5% percentage bounds configured here, and the minimum trip multiple of all 20 keys is set by the absolute bound. THIS IS NOT A GENERAL GUARANTEE. Do not reason from 'every step of both ladders is at least a factor of 2' -- that premise is false. The span ladder breaks it three times at the top: 2s->3s is 1.5x, 3s->4s is 1.33x, 4s->5s is 1.25x, so second-scale consensus quantiles quantize to ~1s widths there. Because the bound is (hi_next - baseline), a baseline between about 2667ms and 3000ms, or between about 3334ms and 4000ms, gets an absolute bound worth less than 50% of itself and the PERCENTAGE bound becomes the operative one -- at which point the metric fires on a 50% move that is smaller than one bucket width, and the single-crossing guarantee in _absolute_bound_derivation is lost. That band is not hypothetical: the collector config names consensus.round (~3.9s) as a reason those edges exist, and 3900ms sits in the second sub-band with an absolute bound of 5000 - 3900 = 1100, only 28.2% of baseline. Whoever gates a key whose baseline lands in either sub-band MUST lower its max_pct_increase below (bound / baseline) for that key, or state explicitly that the metric is percentage-gated and the bucket guarantee does not hold for it. check_regression_bounds.py enforces this as rule D so the trap cannot be walked into silently. The percentage entries are required and still meaningful regardless: compare_to_baseline.py treats a missing max_pct_increase as 'no threshold configured' and would stop gating the metric entirely; they record the intended relative tolerance (consensus spans 5%, everything else 50%); and they are the operative bound on the defaults path (see _defaults_note).", "defaults": { - "span": { - "p50": { - "max_pct_increase": 50.0, - "max_abs_increase_ms": 0.01 - }, - "p95": { - "max_pct_increase": 50.0, - "max_abs_increase_ms": 0.01 - }, - "p99": { - "max_pct_increase": 50.0, - "max_abs_increase_ms": 0.01 - } - }, "job_queue": { "p95": { - "max_pct_increase": 50.0, - "max_abs_increase_us": 1.0 + "max_abs_increase_us": 1.0, + "max_pct_increase": 50.0 + } + }, + "span": { + "p50": { + "max_abs_increase_ms": 0.01, + "max_pct_increase": 50.0 + }, + "p95": { + "max_abs_increase_ms": 0.01, + "max_pct_increase": 50.0 + }, + "p99": { + "max_abs_increase_ms": 0.01, + "max_pct_increase": 50.0 } } }, "overrides": { "job.acceptLedger.queued": { "p95": { - "max_pct_increase": 50.0, - "max_abs_increase_us": 333.8636363636368 + "max_abs_increase_us": 154.41666666666669, + "max_pct_increase": 50.0 } }, "job.acceptLedger.running": { "p95": { - "max_pct_increase": 50.0, - "max_abs_increase_us": 93857.14285714286 + "max_abs_increase_us": 84032.25806451614, + "max_pct_increase": 50.0 } }, "job.transaction.queued": { "p95": { - "max_pct_increase": 50.0, - "max_abs_increase_us": 573.8095238095241 + "max_abs_increase_us": 596.2582236842106, + "max_pct_increase": 50.0 } }, "job.transaction.running": { "p95": { - "max_pct_increase": 50.0, - "max_abs_increase_us": 4400.000000000002 + "max_abs_increase_us": 623.2487922705312, + "max_pct_increase": 50.0 } }, "span.consensus.accept": { "p50": { - "max_pct_increase": 5.0, - "max_abs_increase_ms": 4.471264367816092 + "max_abs_increase_ms": 8.563636363636363, + "max_pct_increase": 5.0 }, "p95": { - "max_pct_increase": 5.0, - "max_abs_increase_ms": 16.03030303030303 + "max_abs_increase_ms": 16.188888888888886, + "max_pct_increase": 5.0 }, "p99": { - "max_pct_increase": 5.0, - "max_abs_increase_ms": 29.199999999999974 + "max_abs_increase_ms": 29.071428571428555, + "max_pct_increase": 5.0 } }, "span.consensus.ledger_close": { "p95": { - "max_pct_increase": 5.0, - "max_abs_increase_ms": 4.2170000000000005 + "max_abs_increase_ms": 4.325, + "max_pct_increase": 5.0 }, "p99": { - "max_pct_increase": 5.0, - "max_abs_increase_ms": 7.97000000000001 + "max_abs_increase_ms": 6.950000000000009, + "max_pct_increase": 5.0 } }, "span.ledger.build": { "p95": { - "max_pct_increase": 50.0, - "max_abs_increase_ms": 5.46666666666667 - }, - "p99": { - "max_pct_increase": 50.0, - "max_abs_increase_ms": 15.890909090909087 + "max_abs_increase_ms": 5.2285714285714295, + "max_pct_increase": 50.0 } }, "span.ledger.validate": { "p50": { - "max_pct_increase": 50.0, - "max_abs_increase_ms": 0.1852810599788517 + "max_abs_increase_ms": 0.19023809523809523, + "max_pct_increase": 50.0 } }, "span.rpc.ws_message": { "p50": { - "max_pct_increase": 50.0, - "max_abs_increase_ms": 0.339965483234714 + "max_abs_increase_ms": 0.3371724125235492, + "max_pct_increase": 50.0 }, "p95": { - "max_pct_increase": 50.0, - "max_abs_increase_ms": 4.302260273972602 + "max_abs_increase_ms": 4.187754553374655, + "max_pct_increase": 50.0 }, "p99": { - "max_pct_increase": 50.0, - "max_abs_increase_ms": 4.024287671232877 + "max_abs_increase_ms": 4.012733990147783, + "max_pct_increase": 50.0 } }, "span.tx.apply": { "p95": { - "max_pct_increase": 50.0, - "max_abs_increase_ms": 6.475000000000001 + "max_abs_increase_ms": 5.360283687943262, + "max_pct_increase": 50.0 }, "p99": { - "max_pct_increase": 50.0, - "max_abs_increase_ms": 19.933333333333294 + "max_abs_increase_ms": 5.0266666666666655, + "max_pct_increase": 50.0 } }, "span.tx.process": { "p50": { - "max_pct_increase": 50.0, - "max_abs_increase_ms": 0.29937780210420883 + "max_abs_increase_ms": 0.28609325031081345, + "max_pct_increase": 50.0 }, "p95": { - "max_pct_increase": 50.0, - "max_abs_increase_ms": 4.389953271028038 + "max_abs_increase_ms": 0.5005002942315832, + "max_pct_increase": 50.0 }, "p99": { - "max_pct_increase": 50.0, - "max_abs_increase_ms": 7.241212121212123 + "max_abs_increase_ms": 4.006030232040599, + "max_pct_increase": 50.0 } } }